Merge branch 'dev' into 'main'

dev merge

Closes #169

See merge request veilid/veilid!14

.cargo/config.toml

@@ -1,14 +1,2 @@
 [build]
 rustflags = ["--cfg", "tokio_unstable"]
-
-[target.aarch64-unknown-linux-gnu]
-linker = "aarch64-linux-gnu-gcc"
-
-[target.aarch64-linux-android]
-linker = "/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang"
-[target.armv7-linux-androideabi]
-linker = "/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi30-clang"
-[target.x86_64-linux-android]
-linker = "/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android30-clang"
-[target.i686-linux-android]
-linker = "/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android30-clang"

.gitmodules

@@ -16,9 +16,6 @@
 [submodule "external/netlink"]
 	path = external/netlink
 	url = ../netlink.git
-[submodule "external/no-std-net"]
-	path = external/no-std-net
-	url = ../no-std-net.git
 [submodule "external/libmdns"]
 	path = external/libmdns
 	url = ../libmdns.git

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 
 members = [
+    "veilid-tools",
     "veilid-core",
     "veilid-server",
     "veilid-cli",
@@ -8,7 +9,7 @@ members = [
     "veilid-wasm",
 ]
 
-exclude = [ "./external/keyring-rs", "./external/netlink", "./external/cursive", "./external/hashlink" ]
+exclude = [ "./external/keyring-manager", "./external/netlink", "./external/cursive", "./external/hashlink" ]
 
 [patch.crates-io]
 cursive = { path = "./external/cursive/cursive" }

Earthfile

@@ -52,9 +52,9 @@ deps-android:
     FROM +deps-cross
     RUN apt-get install -y openjdk-9-jdk-headless
     RUN mkdir /Android; mkdir /Android/Sdk
-    RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip
+    RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip
     RUN cd /Android; unzip /Android/cmdline-tools.zip
-    RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;30.0.3 ndk\;22.0.7026061 cmake\;3.18.1 platform-tools platforms\;android-30
+    RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33
     RUN apt-get clean
     
 # Just linux build not android
@@ -65,13 +65,16 @@ deps-linux:
 # Code + Linux deps
 code-linux:
     FROM +deps-linux
-    COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
+    COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
+    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
     WORKDIR /veilid
 
 # Code + Linux + Android deps
 code-android:
     FROM +deps-android
-    COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
+    COPY --dir .cargo external files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
+    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
+    RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml
     WORKDIR /veilid
 
 # Clippy only
@@ -93,7 +96,7 @@ build-linux-arm64:
 build-android:
     FROM +code-android
     WORKDIR /veilid/veilid-core
-    ENV PATH=$PATH:/Android/Sdk/ndk/22.0.7026061/toolchains/llvm/prebuilt/linux-x86_64/bin/
+    ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
     RUN cargo build --target aarch64-linux-android --release
     RUN cargo build --target armv7-linux-androideabi --release
     RUN cargo build --target i686-linux-android --release

README.md

@@ -27,9 +27,11 @@ method is highly recommended as you may run into path problems with the 'flutter
 command line without it. If you do so, you may skip to 
 [Run Veilid setup script](#Run Veilid setup script).
 
-* build-tools;30.0.3
+* build-tools;33.0.1
-* ndk;22.0.7026061
+* ndk;25.1.8937393
 * cmake;3.22.1
+* platform-tools
+* platforms;android-33
 
 #### Setup Dependencies using the CLI
 
@@ -39,8 +41,10 @@ instructions for `sdkmanager`
 the command line to install the requisite package versions:
 
 ```shell
-sdkmanager --install "build-tools;30.0.3"
+sdkmanager --install "platform-tools"
-sdkmanager --install "ndk;22.0.7026061"
+sdkmanager --install "platforms;android-33"
+sdkmanager --install "build-tools;33.0.1"
+sdkmanager --install "ndk;25.1.8937393"
 sdkmanager --install "cmake;3.22.1"
 ```
 
@@ -50,7 +54,7 @@ your path.
 ```shell
 cat << EOF >> ~/.profile 
 export ANDROID_SDK_ROOT=<path to sdk>
-export ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/22.0.7026061
+export ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/25.1.8937393
 export PATH=\$PATH:$ANDROID_SDK_ROOT/platform-tools
 EOF
 ```
@@ -88,9 +92,10 @@ Development requires:
 
 You will need to use Android Studio [here](https://developer.android.com/studio) 
 to maintain your Android dependencies. Use the SDK Manager in the IDE to install the following packages (use package details view to select version):
-* Android SDK Build Tools (30.0.3)
+* Android SDK Build Tools (33.0.1)
-* NDK (Side-by-side) (22.0.7026061)
+* NDK (Side-by-side) (25.1.8937393)
 * Cmake (3.22.1)
+* Android SDK 33
 * Android SDK Command Line Tools (latest) (7.0/latest)
 
 #### Setup command line environment
@@ -101,7 +106,7 @@ your path.
 ```shell
 cat << EOF >> ~/.zshenv
 export ANDROID_SDK_ROOT=$HOME/Library/Android/sdk
-export ANDROID_NDK_HOME=$HOME/Library/Android/sdk/ndk/22.0.7026061
+export ANDROID_NDK_HOME=$HOME/Library/Android/sdk/ndk/25.1.8937393
 export PATH=\$PATH:$HOME/Library/Android/sdk/platform-tools
 EOF
 ```

doc/config/sample.config

@@ -81,7 +81,6 @@ core:
             min_peer_refresh_time_ms: 2000
             validate_dial_info_receipt_time_ms: 2000
         upnp: true
-        natpmp: false
         detect_address_changes: true
         enable_local_peer_scope: false
         restricted_nat_retries: 0

doc/config/veilid-server-config.md

@@ -193,7 +193,6 @@ network:
     bootstrap: ['bootstrap.dev.veilid.net']
     bootstrap_nodes: []
     upnp: true
-    natpmp: false
     detect_address_changes: true
     enable_local_peer_scope: false
     restricted_nat_retries: 0

external/keyring-manager

@@ -1 +1 @@
-Subproject commit 1655f89cf2ec70900c520080819d76ffad90adee
+Subproject commit b127b2d3c653fea163a776dd58b3798f28aeeee3

external/keyvaluedb

@@ -1 +1 @@
-Subproject commit e30d0058defd9cfd7bd546bb177228edda8076ab
+Subproject commit 3408e0b2ae3df0088e0714bc23fb33c82a58e22c

external/no-std-net

@@ -1 +0,0 @@
-Subproject commit db4af788049b5073567a36cb2e7b0445af66ab1c

scripts/earthly/cargo-android/config.toml

@@ -0,0 +1,8 @@
+[target.aarch64-linux-android]
+linker = "/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android33-clang"
+[target.armv7-linux-androideabi]
+linker = "/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi33-clang"
+[target.x86_64-linux-android]
+linker = "/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android33-clang"
+[target.i686-linux-android]
+linker = "/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android33-clang"

scripts/earthly/cargo-linux/config.toml

@@ -0,0 +1,2 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

scripts/ios_build.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+pushd $SCRIPTDIR >/dev/null
+
+CARGO=`which cargo`
+CARGO=${CARGO:=~/.cargo/bin/cargo}
+CARGO_DIR=$(dirname $CARGO)
+
+CARGO_MANIFEST_PATH=$(python3 -c "import os; import json; print(json.loads(os.popen('$CARGO locate-project').read())['root'])")
+CARGO_WORKSPACE_PATH=$(python3 -c "import os; import json; print(json.loads(os.popen('$CARGO locate-project --workspace').read())['root'])")
+TARGET_PATH=$(python3 -c "import os; print(os.path.realpath(\"$CARGO_WORKSPACE_PATH/../target\"))")
+PACKAGE_NAME=$1
+shift
+
+if [ "$CONFIGURATION" == "Debug" ]; then 
+    EXTRA_CARGO_OPTIONS="$@"
+    BUILD_MODE="debug"
+else
+    EXTRA_CARGO_OPTIONS="$@ --release"
+    BUILD_MODE="release"
+fi
+ARCHS=${ARCHS:=arm64}
+
+if [ "$PLATFORM_NAME" == "iphonesimulator" ]; then
+    LIPO_OUT_NAME="lipo-ios-sim"
+else 
+    LIPO_OUT_NAME="lipo-ios"
+fi
+
+for arch in $ARCHS
+do
+    if [ "$arch" == "arm64" ]; then
+        echo arm64
+        if [ "$PLATFORM_NAME" == "iphonesimulator" ]; then
+            CARGO_TARGET=aarch64-apple-ios-sim
+        else
+            CARGO_TARGET=aarch64-apple-ios
+        fi
+        CARGO_TOOLCHAIN=
+    elif [ "$arch" == "x86_64" ]; then
+        echo x86_64
+        CARGO_TARGET=x86_64-apple-ios
+        CARGO_TOOLCHAIN=
+    else
+        echo Unsupported ARCH: $arch
+        continue
+    fi
+
+
+
+    # Choose arm64 brew for unit tests by default if we are on M1
+    if [ -f /opt/homebrew/bin/brew ]; then
+        HOMEBREW_DIR=/opt/homebrew/bin
+    elif [ -f /usr/local/bin/brew ]; then
+        HOMEBREW_DIR=/usr/local/bin
+    else 
+        HOMEBREW_DIR=$(dirname `which brew`)
+    fi
+
+    env -i PATH=/usr/bin:/bin:$HOMEBREW_DIR:$CARGO_DIR HOME="$HOME" USER="$USER" cargo $CARGO_TOOLCHAIN build $EXTRA_CARGO_OPTIONS --target $CARGO_TARGET --manifest-path $CARGO_MANIFEST_PATH
+
+    LIPOS="$LIPOS $TARGET_PATH/$CARGO_TARGET/$BUILD_MODE/lib$PACKAGE_NAME.a"
+
+done
+
+# Make lipo build
+mkdir -p "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/"
+lipo $LIPOS -create -output "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/lib$PACKAGE_NAME.a"
+
+# Make most recent dylib available without build mode for flutter
+cp "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/lib$PACKAGE_NAME.a" "$TARGET_PATH/$LIPO_OUT_NAME/lib$PACKAGE_NAME.a"
+
+popd >/dev/null

scripts/macos_build.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+pushd $SCRIPTDIR >/dev/null
+
+CARGO=`which cargo`
+CARGO=${CARGO:=~/.cargo/bin/cargo}
+CARGO_DIR=$(dirname $CARGO)
+
+CARGO_MANIFEST_PATH=$(python3 -c "import os; import json; print(json.loads(os.popen('$CARGO locate-project').read())['root'])")
+CARGO_WORKSPACE_PATH=$(python3 -c "import os; import json; print(json.loads(os.popen('$CARGO locate-project --workspace').read())['root'])")
+TARGET_PATH=$(python3 -c "import os; print(os.path.realpath(\"$CARGO_WORKSPACE_PATH/../target\"))")
+PACKAGE_NAME=$1
+shift
+
+if [ "$CONFIGURATION" == "Debug" ]; then 
+    EXTRA_CARGO_OPTIONS="$@"
+    BUILD_MODE="debug"
+else
+    EXTRA_CARGO_OPTIONS="$@ --release"
+    BUILD_MODE="release"
+fi
+ARCHS=${ARCHS:=arm64}
+
+LIPO_OUT_NAME="lipo-darwin"
+
+for arch in $ARCHS
+do
+    if [ "$arch" == "arm64" ]; then
+        echo arm64
+        CARGO_TARGET=aarch64-apple-darwin
+        CARGO_TOOLCHAIN=
+    elif [ "$arch" == "x86_64" ]; then
+        echo x86_64
+        CARGO_TARGET=x86_64-apple-darwin
+        CARGO_TOOLCHAIN=
+    else
+        echo Unsupported ARCH: $arch
+        continue
+    fi
+
+    # Choose arm64 brew for unit tests by default if we are on M1
+    if [ -f /opt/homebrew/bin/brew ]; then
+        HOMEBREW_DIR=/opt/homebrew/bin
+    elif [ -f /usr/local/bin/brew ]; then
+        HOMEBREW_DIR=/usr/local/bin
+    else 
+        HOMEBREW_DIR=$(dirname `which brew`)
+    fi
+
+    env -i PATH=/usr/bin:/bin:$HOMEBREW_DIR:$CARGO_DIR HOME="$HOME" USER="$USER" cargo $CARGO_TOOLCHAIN build $EXTRA_CARGO_OPTIONS --target $CARGO_TARGET --manifest-path $CARGO_MANIFEST_PATH
+
+    LIPOS="$LIPOS $TARGET_PATH/$CARGO_TARGET/$BUILD_MODE/lib$PACKAGE_NAME.dylib"
+
+done
+
+# Make lipo build
+mkdir -p "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/"
+lipo $LIPOS -create -output "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/lib$PACKAGE_NAME.dylib"
+
+# Make most recent dylib available without build mode for flutter
+cp "$TARGET_PATH/$LIPO_OUT_NAME/$BUILD_MODE/lib$PACKAGE_NAME.dylib" "$TARGET_PATH/$LIPO_OUT_NAME/lib$PACKAGE_NAME.dylib"
+
+popd > /dev/null

scripts/new_android_sim.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+UNAME_M=`uname -m`
+if [[ "$UNAME_M" == "arm64" ]]; then
+    ANDROID_ABI=arm64-v8a
+elif [[ "$UNAME_M" == "x86_64" ]]; then
+    ANDROID_ABI=x86
+else 
+    echo "Unknown platform"
+    exit 1
+fi
+AVD_NAME="testavd"
+AVD_TAG="google_atd"
+AVD_IMAGE="system-images;android-30;$AVD_TAG;$ANDROID_ABI"
+AVD_DEVICE="Nexus 10"
+# Install AVD image
+$ANDROID_SDK_ROOT/tools/bin/sdkmanager --install "$AVD_IMAGE"
+# Make AVD
+echo "no" | $ANDROID_SDK_ROOT/tools/bin/avdmanager --verbose create avd --force --name "$AVD_NAME" --package "$AVD_IMAGE" --tag "$AVD_TAG" --abi "$ANDROID_ABI" --device "$AVD_DEVICE"
+# Run emulator
+$ANDROID_SDK_ROOT/emulator/emulator -avd testavd -no-snapshot -no-boot-anim -no-window &
+( trap exit SIGINT ; read -r -d '' _ </dev/tty ) ## wait for Ctrl-C
+kill %1
+wait

scripts/new_ios_sim.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+ID=$(xcrun simctl create test-iphone com.apple.CoreSimulator.SimDeviceType.iPhone-14-Pro com.apple.CoreSimulator.SimRuntime.iOS-16-1 2>/dev/null)
+xcrun simctl boot $ID
+xcrun simctl bootstatus $ID
+echo Simulator ID is $ID
+( trap exit SIGINT ; read -r -d '' _ </dev/tty ) ## wait for Ctrl-C
+xcrun simctl delete $ID
+
+    

setup_linux.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
+set -eo pipefail
+
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+if [[ "$(uname)" != "Linux" ]]; then 
+    echo Not running Linux
+    exit 1
+fi
+
 if [ "$(lsb_release -d | grep -qEi 'debian|buntu|mint')" ]; then
     echo Not a supported Linux
     exit 1
@@ -14,6 +21,14 @@ else
     exit 1
 fi
 
+# ensure Android Command Line Tools exist
+if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
+    echo '[X] Android command line tools are installed' 
+else
+    echo 'Android command line tools are not installed'
+    exit 1
+fi
+
 # ensure ANDROID_NDK_HOME is defined and exists
 if [ -d "$ANDROID_NDK_HOME" ]; then
     echo '[X] $ANDROID_NDK_HOME is defined and exists' 
@@ -79,6 +94,9 @@ cargo install wasm-bindgen-cli wasm-pack
 # Ensure packages are installed
 sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk llvm wabt checkinstall
 
+# Ensure android sdk packages are installed
+$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33
+
 # Install capnproto using the same mechanism as our earthly build
 $SCRIPTDIR/scripts/earthly/install_capnproto.sh
 # Install protoc using the same mechanism as our earthly build

setup_macos.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+set -eo pipefail
+
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 if [ ! "$(uname)" == "Darwin" ]; then
@@ -14,6 +16,14 @@ else
     exit 1
 fi
 
+# ensure Android Command Line Tools exist
+if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
+    echo '[X] Android command line tools are installed' 
+else
+    echo 'Android command line tools are not installed'
+    exit 1
+fi
+
 # ensure ANDROID_NDK_HOME is defined and exists
 if [ -d "$ANDROID_NDK_HOME" ]; then
     echo '[X] $ANDROID_NDK_HOME is defined and exists' 
@@ -86,6 +96,10 @@ else
     exit 1
 fi
 
+
+# Ensure android sdk packages are installed
+$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33
+
 # install targets
 rustup target add aarch64-apple-darwin aarch64-apple-ios x86_64-apple-darwin x86_64-apple-ios wasm32-unknown-unknown aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android
 
@@ -108,5 +122,5 @@ if [ "$BREW_USER" == "" ]; then
         BREW_USER=`whoami`
     fi
 fi
-sudo -H -u $BREW_USER brew install capnp cmake wabt llvm protobuf
+sudo -H -u $BREW_USER brew install capnp cmake wabt llvm protobuf openjdk@11
-
+sudo gem install cocoapods

setup_windows.bat

@@ -0,0 +1,38 @@
+@echo off
+setlocal
+
+REM #############################################
+
+PUSHD %~dp0
+SET ROOTDIR=%CD%
+POPD
+
+IF NOT DEFINED ProgramFiles(x86) (
+    echo This script requires a 64-bit Windows Installation. Exiting.
+    goto end
+)
+
+FOR %%X IN (protoc.exe) DO (SET PROTOC_FOUND=%%~$PATH:X)
+IF NOT DEFINED PROTOC_FOUND (
+    echo protobuf compiler ^(protoc^) is required but it's not installed. Install protoc 21.10 or higher. Ensure it is in your path. Aborting.
+    echo protoc is available here: https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-win64.zip
+    goto end
+)
+
+FOR %%X IN (capnp.exe) DO (SET CAPNP_FOUND=%%~$PATH:X)
+IF NOT DEFINED CAPNP_FOUND (
+    echo capnproto compiler ^(capnp^) is required but it's not installed. Install capnp 0.10.3 or higher. Ensure it is in your path. Aborting.
+    echo capnp is available here: https://capnproto.org/capnproto-c++-win32-0.10.3.zip
+    goto end
+)
+
+FOR %%X IN (cargo.exe) DO (SET CARGO_FOUND=%%~$PATH:X)
+IF NOT DEFINED CARGO_FOUND (
+    echo rust ^(cargo^) is required but it's not installed. Install rust 1.65 or higher. Ensure it is in your path. Aborting.
+    echo install rust via rustup here: https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe
+    goto ends
+)
+
+echo Setup successful
+:end
+ENDLOCAL

veilid-cli/Cargo.toml

@@ -16,7 +16,7 @@ rt-async-std = [ "async-std", "veilid-core/rt-async-std", "cursive/rt-async-std"
 rt-tokio = [ "tokio", "tokio-util", "veilid-core/rt-tokio", "cursive/rt-tokio" ]
 
 [dependencies]
-cursive = { path = "../external/cursive/cursive", default-features = false, features = [ "crossterm", "toml"]}
+cursive = { path = "../external/cursive/cursive", default-features = false, features = [ "crossterm", "toml", "ansi" ]}
 async-std = { version = "^1.9", features = ["unstable", "attributes"], optional = true }
 tokio = { version = "^1", features = ["full"], optional = true }
 tokio-util = { version = "^0", features = ["compat"], optional = true}

veilid-cli/src/client_api_connection.rs

@@ -8,7 +8,7 @@ use serde::de::DeserializeOwned;
 use std::cell::RefCell;
 use std::net::SocketAddr;
 use std::rc::Rc;
-use veilid_core::xx::*;
+use veilid_core::tools::*;
 use veilid_core::*;
 
 macro_rules! capnp_failed {
@@ -92,6 +92,9 @@ impl veilid_client::Server for VeilidClientImpl {
             VeilidUpdate::Config(config) => {
                 self.comproc.update_config(config);
             }
+            VeilidUpdate::Route(route) => {
+                self.comproc.update_route(route);
+            }
             VeilidUpdate::Shutdown => self.comproc.update_shutdown(),
         }
 
@@ -226,8 +229,8 @@ impl ClientApiConnection {
 
         // Wait until rpc system completion or disconnect was requested
         let res = rpc_jh.await;
-        #[cfg(feature = "rt-tokio")]
+        // #[cfg(feature = "rt-tokio")]
-        let res = res.map_err(|e| format!("join error: {}", e))?;
+        // let res = res.map_err(|e| format!("join error: {}", e))?;
         res.map_err(|e| format!("client RPC system error: {}", e))
     }
 
@@ -441,7 +444,11 @@ impl ClientApiConnection {
         res.map_err(map_to_string)
     }
 
-    pub async fn server_appcall_reply(&mut self, id: u64, msg: Vec<u8>) -> Result<(), String> {
+    pub async fn server_appcall_reply(
+        &mut self,
+        id: OperationId,
+        msg: Vec<u8>,
+    ) -> Result<(), String> {
         trace!("ClientApiConnection::appcall_reply");
         let server = {
             let inner = self.inner.borrow();
@@ -452,7 +459,7 @@ impl ClientApiConnection {
                 .clone()
         };
         let mut request = server.borrow().app_call_reply_request();
-        request.get().set_id(id);
+        request.get().set_id(id.as_u64());
         request.get().set_message(&msg);
         let response = self
             .cancellable(request.send().promise)

veilid-cli/src/command_processor.rs

@@ -1,13 +1,11 @@
 use crate::client_api_connection::*;
 use crate::settings::Settings;
-use crate::tools::*;
 use crate::ui::*;
-use log::*;
 use std::cell::*;
 use std::net::SocketAddr;
 use std::rc::Rc;
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
-use veilid_core::xx::{Eventual, EventualCommon};
+use veilid_core::tools::*;
 use veilid_core::*;
 
 pub fn convert_loglevel(s: &str) -> Result<VeilidConfigLogLevel, String> {
@@ -49,7 +47,7 @@ struct CommandProcessorInner {
     autoreconnect: bool,
     server_addr: Option<SocketAddr>,
     connection_waker: Eventual,
-    last_call_id: Option<u64>,
+    last_call_id: Option<OperationId>,
 }
 
 type Handle<T> = Rc<RefCell<T>>;
@@ -251,7 +249,7 @@ reply               - reply to an AppCall not handled directly by the server
                     }
                     Ok(v) => v,
                 };
-                (id, second)
+                (OperationId::new(id), second)
             } else {
                 let id = match some_last_id {
                     None => {
@@ -366,7 +364,7 @@ reply               - reply to an AppCall not handled directly by the server
                 debug!("Connection lost, retrying in 2 seconds");
                 {
                     let waker = self.inner_mut().connection_waker.instance_clone(());
-                    let _ = timeout(Duration::from_millis(2000), waker).await;
+                    let _ = timeout(2000, waker).await;
                 }
                 self.inner_mut().connection_waker.reset();
                 first = false;
@@ -390,20 +388,42 @@ reply               - reply to an AppCall not handled directly by the server
     ////////////////////////////////////////////
 
     pub fn update_attachment(&mut self, attachment: veilid_core::VeilidStateAttachment) {
-        self.inner_mut().ui.set_attachment_state(attachment.state);
+        self.inner_mut().ui.set_attachment_state(
+            attachment.state,
+            attachment.public_internet_ready,
+            attachment.local_network_ready,
+        );
     }
 
     pub fn update_network_status(&mut self, network: veilid_core::VeilidStateNetwork) {
         self.inner_mut().ui.set_network_status(
             network.started,
-            network.bps_down,
+            network.bps_down.as_u64(),
-            network.bps_up,
+            network.bps_up.as_u64(),
             network.peers,
         );
     }
     pub fn update_config(&mut self, config: veilid_core::VeilidStateConfig) {
         self.inner_mut().ui.set_config(config.config)
     }
+    pub fn update_route(&mut self, route: veilid_core::VeilidStateRoute) {
+        let mut out = String::new();
+        if !route.dead_routes.is_empty() {
+            out.push_str(&format!("Dead routes: {:?}", route.dead_routes));
+        }
+        if !route.dead_remote_routes.is_empty() {
+            if !out.is_empty() {
+                out.push_str("\n");
+            }
+            out.push_str(&format!(
+                "Dead remote routes: {:?}",
+                route.dead_remote_routes
+            ));
+        }
+        if !out.is_empty() {
+            self.inner().ui.add_node_event(out);
+        }
+    }
 
     pub fn update_log(&mut self, log: veilid_core::VeilidLog) {
         self.inner().ui.add_node_event(format!(
@@ -455,7 +475,9 @@ reply               - reply to an AppCall not handled directly by the server
 
         self.inner().ui.add_node_event(format!(
             "AppCall ({:?}) id = {:016x} : {}",
-            call.sender, call.id, strmsg
+            call.sender,
+            call.id.as_u64(),
+            strmsg
         ));
 
         self.inner_mut().last_call_id = Some(call.id);

veilid-cli/src/main.rs

@@ -1,14 +1,15 @@
 #![deny(clippy::all)]
 #![deny(unused_must_use)]
+#![recursion_limit = "256"]
 
-use veilid_core::xx::*;
+use crate::tools::*;
+use veilid_core::tools::*;
 
 use clap::{Arg, ColorChoice, Command};
 use flexi_logger::*;
 use std::ffi::OsStr;
 use std::net::ToSocketAddrs;
 use std::path::Path;
-use tools::*;
 
 mod client_api_connection;
 mod command_processor;

veilid-cli/src/peers_table_view.rs

@@ -1,7 +1,7 @@
 use super::*;
 use cursive_table_view::*;
 use std::cmp::Ordering;
-use veilid_core::PeerTableData;
+use veilid_core::*;
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub enum PeerTableColumn {
@@ -24,7 +24,8 @@ pub enum PeerTableColumn {
 //     }
 // }
 
-fn format_ts(ts: u64) -> String {
+fn format_ts(ts: Timestamp) -> String {
+    let ts = ts.as_u64();
     let secs = timestamp_to_secs(ts);
     if secs >= 1.0 {
         format!("{:.2}s", timestamp_to_secs(ts))
@@ -33,7 +34,8 @@ fn format_ts(ts: u64) -> String {
     }
 }
 
-fn format_bps(bps: u64) -> String {
+fn format_bps(bps: ByteCount) -> String {
+    let bps = bps.as_u64();
     if bps >= 1024u64 * 1024u64 * 1024u64 {
         format!("{:.2}GB/s", (bps / (1024u64 * 1024u64)) as f64 / 1024.0)
     } else if bps >= 1024u64 * 1024u64 {

veilid-cli/src/tools.rs

@@ -3,32 +3,12 @@ use core::future::Future;
 
 cfg_if! {
     if #[cfg(feature="rt-async-std")] {
-        pub use async_std::task::JoinHandle;
         pub use async_std::net::TcpStream;
-        pub use async_std::future::TimeoutError;
-        pub fn spawn_local<F: Future<Output = T> + 'static, T: 'static>(f: F) -> JoinHandle<T> {
-            async_std::task::spawn_local(f)
-        }
-        pub fn spawn_detached_local<F: Future<Output = T> + 'static, T: 'static>(f: F) {
-            let _ = async_std::task::spawn_local(f);
-        }
-        pub use async_std::task::sleep;
-        pub use async_std::future::timeout;
         pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
             async_std::task::block_on(f)
         }
     } else if #[cfg(feature="rt-tokio")] {
-        pub use tokio::task::JoinHandle;
         pub use tokio::net::TcpStream;
-        pub use tokio::time::error::Elapsed as TimeoutError;
-        pub fn spawn_local<F: Future<Output = T> + 'static, T: 'static>(f: F) -> JoinHandle<T> {
-            tokio::task::spawn_local(f)
-        }
-        pub fn spawn_detached_local<F: Future<Output = T> + 'static, T: 'static>(f: F) {
-            let _ = tokio::task::spawn_local(f);
-        }
-        pub use tokio::time::sleep;
-        pub use tokio::time::timeout;
         pub fn block_on<F: Future<Output = T>, T>(f: F) -> T {
             let rt = tokio::runtime::Runtime::new().unwrap();
             let local = tokio::task::LocalSet::new();

veilid-cli/src/ui.rs

@@ -51,6 +51,8 @@ pub type UICallback = Box<dyn Fn(&mut Cursive) + Send>;
 
 struct UIState {
     attachment_state: Dirty<AttachmentState>,
+    public_internet_ready: Dirty<bool>,
+    local_network_ready: Dirty<bool>,
     network_started: Dirty<bool>,
     network_down_up: Dirty<(f32, f32)>,
     connection_state: Dirty<ConnectionState>,
@@ -62,6 +64,8 @@ impl UIState {
     pub fn new() -> Self {
         Self {
             attachment_state: Dirty::new(AttachmentState::Detached),
+            public_internet_ready: Dirty::new(false),
+            local_network_ready: Dirty::new(false),
             network_started: Dirty::new(false),
             network_down_up: Dirty::new((0.0, 0.0)),
             connection_state: Dirty::new(ConnectionState::Disconnected),
@@ -234,17 +238,28 @@ impl UI {
     fn peers(s: &mut Cursive) -> ViewRef<PeersTableView> {
         s.find_name("peers").unwrap()
     }
-    fn render_attachment_state<'a>(inner: &mut UIInner) -> &'a str {
+    fn render_attachment_state(inner: &mut UIInner) -> String {
-        match inner.ui_state.attachment_state.get() {
+        let att = match inner.ui_state.attachment_state.get() {
-            AttachmentState::Detached => " Detached [----]",
+            AttachmentState::Detached => "[----]",
-            AttachmentState::Attaching => "Attaching [/   ]",
+            AttachmentState::Attaching => "[/   ]",
-            AttachmentState::AttachedWeak => " Attached [|   ]",
+            AttachmentState::AttachedWeak => "[|   ]",
-            AttachmentState::AttachedGood => " Attached [||  ]",
+            AttachmentState::AttachedGood => "[||  ]",
-            AttachmentState::AttachedStrong => " Attached [||| ]",
+            AttachmentState::AttachedStrong => "[||| ]",
-            AttachmentState::FullyAttached => " Attached [||||]",
+            AttachmentState::FullyAttached => "[||||]",
-            AttachmentState::OverAttached => " Attached [++++]",
+            AttachmentState::OverAttached => "[++++]",
-            AttachmentState::Detaching => "Detaching [////]",
+            AttachmentState::Detaching => "[////]",
-        }
+        };
+        let pi = if *inner.ui_state.public_internet_ready.get() {
+            "+P"
+        } else {
+            "-p"
+        };
+        let ln = if *inner.ui_state.local_network_ready.get() {
+            "+L"
+        } else {
+            "-l"
+        };
+        format!("{}{}{}", att, pi, ln)
     }
     fn render_network_status(inner: &mut UIInner) -> String {
         match inner.ui_state.network_started.get() {
@@ -344,16 +359,10 @@ impl UI {
             Ok(_) => {}
             Err(e) => {
                 let color = *Self::inner_mut(s).log_colors.get(&Level::Error).unwrap();
-
-                cursive_flexi_logger_view::push_to_log(StyledString::styled(
-                    format!("> {}", text),
-                    color,
-                ));
                 cursive_flexi_logger_view::push_to_log(StyledString::styled(
                     format!("  Error: {}", e),
                     color,
                 ));
-                return;
             }
         }
         // save to history unless it's a duplicate
@@ -838,9 +847,20 @@ impl UI {
         inner.cmdproc = Some(cmdproc);
         let _ = inner.cb_sink.send(Box::new(UI::update_cb));
     }
-    pub fn set_attachment_state(&mut self, state: AttachmentState) {
+    pub fn set_attachment_state(
+        &mut self,
+        state: AttachmentState,
+        public_internet_ready: bool,
+        local_network_ready: bool,
+    ) {
         let mut inner = self.inner.borrow_mut();
         inner.ui_state.attachment_state.set(state);
+        inner
+            .ui_state
+            .public_internet_ready
+            .set(public_internet_ready);
+        inner.ui_state.local_network_ready.set(local_network_ready);
+
         let _ = inner.cb_sink.send(Box::new(UI::update_cb));
     }
     pub fn set_network_status(
@@ -878,8 +898,12 @@ impl UI {
     pub fn add_node_event(&self, event: String) {
         let inner = self.inner.borrow();
         let color = *inner.log_colors.get(&Level::Info).unwrap();
+        let mut starting_style: Style = color.into();
         for line in event.lines() {
-            cursive_flexi_logger_view::push_to_log(StyledString::styled(line, color));
+            let (spanned_string, end_style) =
+                cursive::utils::markup::ansi::parse_with_starting_style(starting_style, line);
+            cursive_flexi_logger_view::push_to_log(spanned_string);
+            starting_style = end_style;
         }
         let _ = inner.cb_sink.send(Box::new(UI::update_cb));
     }

veilid-core/Cargo.toml

@@ -11,20 +11,20 @@ crate-type = ["cdylib", "staticlib", "rlib"]
 
 [features]
 default = []
-rt-async-std = [ "async-std", "async-std-resolver", "async_executors/async_std", "rtnetlink?/smol_socket" ]
+rt-async-std = [ "async-std", "async-std-resolver", "async_executors/async_std", "rtnetlink?/smol_socket", "veilid-tools/rt-async-std" ]
-rt-tokio = [ "tokio", "tokio-util", "tokio-stream", "trust-dns-resolver/tokio-runtime", "async_executors/tokio_tp", "async_executors/tokio_io", "async_executors/tokio_timer", "rtnetlink?/tokio_socket" ]
+rt-tokio = [ "tokio", "tokio-util", "tokio-stream", "trust-dns-resolver/tokio-runtime", "async_executors/tokio_tp", "async_executors/tokio_io", "async_executors/tokio_timer", "rtnetlink?/tokio_socket", "veilid-tools/rt-tokio" ]
 
-android_tests = []
+veilid_core_android_tests = [ "dep:paranoid-android" ]
-ios_tests = [ "simplelog" ]
+veilid_core_ios_tests = [ "dep:tracing-oslog" ]
 tracking = []
 
 [dependencies]
+veilid-tools = { path = "../veilid-tools", features = [ "tracing" ] }
 tracing = { version = "^0", features = ["log", "attributes"] }
 tracing-subscriber = "^0"
 tracing-error = "^0"
 eyre = "^0"
 capnp = { version = "^0", default_features = false }
-rust-fsm = "^0"
 static_assertions = "^1"
 cfg-if = "^1"
 thiserror = "^1"
@@ -34,6 +34,8 @@ secrecy = "^0"
 chacha20poly1305 = "^0"
 chacha20 = "^0"
 hashlink = { path = "../external/hashlink", features = ["serde_impl"] }
+serde = { version = "^1", features = ["derive" ] }
+serde_json = { version = "^1" }
 serde-big-array = "^0"
 futures-util = { version = "^0", default_features = false, features = ["alloc"] }
 parking_lot = "^0"
@@ -59,10 +61,10 @@ rtnetlink = { version = "^0", default-features = false, optional = true }
 async-std-resolver = { version = "^0", optional = true }
 trust-dns-resolver = { version = "^0", optional = true }
 keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" }
-serde_bytes = { version = "^0" }
+#rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_32", "validation"] }
-#rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_64", "validation"] }
+rkyv = { git = "https://github.com/rkyv/rkyv.git", rev = "57e2a8d", default_features = false, features = ["std", "alloc", "strict", "size_32", "validation"] }
-rkyv = { git = "https://github.com/crioux/rkyv.git", branch = "issue_326", default_features = false, features = ["std", "alloc", "strict", "size_64", "validation"] }
 bytecheck = "^0"
+data-encoding = { version = "^2" }
 
 # Dependencies for native builds only
 # Linux, Windows, Mac, iOS, Android
@@ -84,9 +86,6 @@ rustls = "^0.19"
 rustls-pemfile = "^0.2"
 futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
 keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
-data-encoding = { version = "^2" }
-serde = { version = "^1", features = ["derive" ] }
-serde_json = { version = "^1" }
 socket2 = "^0"
 bugsalot = "^0"
 chrono = "^0"
@@ -98,11 +97,7 @@ nix = "^0"
 wasm-bindgen = "^0"
 js-sys = "^0"
 wasm-bindgen-futures = "^0"
-no-std-net = { path = "../external/no-std-net", features = ["serde"] }
 keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
-data-encoding = { version = "^2", default_features = false, features = ["alloc"] }
-serde = { version = "^1", default-features = false, features = ["derive", "alloc"] }
-serde_json = { version = "^1", default-features = false, features = ["alloc"] }
 getrandom = { version = "^0", features = ["js"] }
 ws_stream_wasm = "^0"
 async_executors = { version = "^0", default-features = false, features = [ "bindgen", "timer" ]}
@@ -131,9 +126,9 @@ features = [
 [target.'cfg(target_os = "android")'.dependencies]
 jni = "^0"
 jni-sys = "^0"
-ndk = { version = "^0", features = ["trace"] }
+ndk = { version = "^0.7" }
-ndk-glue = { version = "^0", features = ["logger"] }
+ndk-glue = { version = "^0.7", features = ["logger"] }
-tracing-android = { version = "^0" }
+paranoid-android = { version = "^0", optional = true }
 
 # Dependenices for all Unix (Linux, Android, MacOS, iOS)
 [target.'cfg(unix)'.dependencies]
@@ -151,7 +146,7 @@ windows-permissions = "^0"
 
 # Dependencies for iOS
 [target.'cfg(target_os = "ios")'.dependencies]
-simplelog = { version = "^0", optional = true }
+tracing-oslog = { version = "^0", optional = true }
 
 # Rusqlite configuration to ensure platforms that don't come with sqlite get it bundled
 # Except WASM which doesn't use sqlite

veilid-core/ios_build.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-CARGO_MANIFEST_PATH=$(python -c "import os; print(os.path.realpath(\"$SCRIPTDIR/Cargo.toml\"))")
-# echo CARGO_MANIFEST_PATH: $CARGO_MANIFEST_PATH 
-
-if [ "$CONFIGURATION" == "Debug" ]; then 
-    EXTRA_CARGO_OPTIONS="$@"
-else
-    EXTRA_CARGO_OPTIONS="$@ --release"
-fi
-ARCHS=${ARCHS:=arm64}
-for arch in $ARCHS
-do
-    if [ "$arch" == "arm64" ]; then
-        echo arm64
-        CARGO_TARGET=aarch64-apple-ios
-        #CARGO_TOOLCHAIN=+ios-arm64-1.57.0
-        CARGO_TOOLCHAIN=
-    elif [ "$arch" == "x86_64" ]; then
-        echo x86_64
-        CARGO_TARGET=x86_64-apple-ios
-        CARGO_TOOLCHAIN=
-    else
-        echo Unsupported ARCH: $arch
-        continue
-    fi
-
-    CARGO=`which cargo`
-    CARGO=${CARGO:=~/.cargo/bin/cargo}
-    CARGO_DIR=$(dirname $CARGO)
-
-    # Choose arm64 brew for unit tests by default if we are on M1
-    if [ -f /opt/homebrew/bin/brew ]; then
-        HOMEBREW_DIR=/opt/homebrew/bin
-    elif [ -f /usr/local/bin/brew ]; then
-        HOMEBREW_DIR=/usr/local/bin
-    else 
-        HOMEBREW_DIR=$(dirname `which brew`)
-    fi
-
-    env -i PATH=/usr/bin:/bin:$HOMEBREW_DIR:$CARGO_DIR HOME="$HOME" USER="$USER" cargo $CARGO_TOOLCHAIN build $EXTRA_CARGO_OPTIONS --target $CARGO_TARGET --manifest-path $CARGO_MANIFEST_PATH
-done
-

veilid-core/proto/veilid.capnp

@@ -156,17 +156,12 @@ using ValueSeqNum = UInt32;                             # sequence numbers for v
 
 struct ValueKey @0xe64b0992c21a0736 {
     publicKey               @0  :ValueID;               # the location of the value
-    subkey                  @1  :Text;                  # the name of the subkey (or empty if the whole key)
+    subkey                  @1  :Text;                  # the name of the subkey (or empty for the default subkey)
 }
 
-# struct ValueKeySeq {
-#    key                     @0  :ValueKey;              # the location of the value
-#    seq                     @1  :ValueSeqNum;           # the sequence number of the value subkey
-# }
-
 struct ValueData @0xb4b7416f169f2a3d {
-    data                    @0  :Data;                  # value or subvalue contents
+    seq                     @0  :ValueSeqNum;           # sequence number of value
-    seq                     @1  :ValueSeqNum;           # sequence number of value
+    data                    @1  :Data;                  # value or subvalue contents
 }
 
 # Operations
@@ -188,6 +183,12 @@ enum DialInfoClass @0x880005edfdd38b1e {
     portRestrictedNAT       @5;                         # P = Device without portmap behind address-and-port restricted NAT
 }
 
+enum Sequencing @0xb6735890f7818a1c {
+    noPreference            @0;
+    preferOrdered           @1;
+    ensureOrdered           @2;
+}
+
 struct DialInfoDetail @0x96423aa1d67b74d8 {
     dialInfo                @0  :DialInfo;
     class                   @1  :DialInfoClass;
@@ -266,9 +267,10 @@ struct PeerInfo @0xfe2d722d5d3c4bcb {
 
 struct RoutedOperation @0xcbcb8535b839e9dd {
     version                 @0  :UInt8;                 # crypto version in use for the data
-    signatures              @1  :List(Signature);       # signatures from nodes that have handled the private route
+    sequencing              @1  :Sequencing;            # sequencing preference to use to pass the message along
-    nonce                   @2  :Nonce;                 # nonce Xmsg
+    signatures              @2  :List(Signature);       # signatures from nodes that have handled the private route
-    data                    @3  :Data;                  # operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr))
+    nonce                   @3  :Nonce;                 # nonce Xmsg
+    data                    @4  :Data;                  # operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr))
 }
 
 struct OperationStatusQ @0x865d80cea70d884a {
@@ -303,11 +305,6 @@ struct OperationRoute @0x96741859ce6ac7dd {
     operation               @1  :RoutedOperation;       # The operation to be routed
 }
 
-struct OperationNodeInfoUpdate @0xc9647b32a48b66ce {
-    signedNodeInfo          @0  :SignedNodeInfo;        # Our signed node info
-}
-
-
 struct OperationAppCallQ @0xade67b9f09784507 {
     message                    @0  :Data;                  # Opaque request to application
 }
@@ -466,12 +463,12 @@ struct Question @0xd8510bc33492ef70 {
         findNodeQ           @3  :OperationFindNodeQ;
         
         # Routable operations
-        getValueQ           @4  :OperationGetValueQ;
+        appCallQ            @4  :OperationAppCallQ;
-        setValueQ           @5  :OperationSetValueQ;
+        getValueQ           @5  :OperationGetValueQ;
-        watchValueQ         @6  :OperationWatchValueQ;
+        setValueQ           @6  :OperationSetValueQ;
-        supplyBlockQ        @7  :OperationSupplyBlockQ;
+        watchValueQ         @7  :OperationWatchValueQ;
-        findBlockQ          @8  :OperationFindBlockQ;
+        supplyBlockQ        @8  :OperationSupplyBlockQ;
-        appCallQ            @9  :OperationAppCallQ;
+        findBlockQ          @9  :OperationFindBlockQ;
         
         # Tunnel operations
         startTunnelQ        @10 :OperationStartTunnelQ;
@@ -486,13 +483,12 @@ struct Statement @0x990e20828f404ae1 {
         # Direct operations
         validateDialInfo    @0  :OperationValidateDialInfo;
         route               @1  :OperationRoute;
-        nodeInfoUpdate      @2  :OperationNodeInfoUpdate;
         
         # Routable operations
-        valueChanged        @3  :OperationValueChanged;
+        signal              @2  :OperationSignal;
-        signal              @4  :OperationSignal;
+        returnReceipt       @3  :OperationReturnReceipt;
-        returnReceipt       @5  :OperationReturnReceipt;
+        appMessage          @4  :OperationAppMessage;
-        appMessage          @6  :OperationAppMessage;
+        valueChanged        @5  :OperationValueChanged;
     }
 }
 
@@ -504,12 +500,12 @@ struct Answer @0xacacb8b6988c1058 {
         findNodeA           @1  :OperationFindNodeA;
         
         # Routable operations
-        getValueA           @2  :OperationGetValueA;
+        appCallA            @2  :OperationAppCallA;
-        setValueA           @3  :OperationSetValueA;
+        getValueA           @3  :OperationGetValueA;
-        watchValueA         @4  :OperationWatchValueA;    
+        setValueA           @4  :OperationSetValueA;
-        supplyBlockA        @5  :OperationSupplyBlockA; 
+        watchValueA         @5  :OperationWatchValueA;    
-        findBlockA          @6  :OperationFindBlockA;
+        supplyBlockA        @6  :OperationSupplyBlockA; 
-        appCallA            @7  :OperationAppCallA;
+        findBlockA          @7  :OperationFindBlockA;
     
         # Tunnel operations
         startTunnelA        @8  :OperationStartTunnelA;
@@ -521,9 +517,10 @@ struct Answer @0xacacb8b6988c1058 {
 struct Operation @0xbf2811c435403c3b {
     opId                    @0  :UInt64;                # Random RPC ID. Must be random to foil reply forgery attacks. 
     senderNodeInfo          @1  :SignedNodeInfo;        # (optional) SignedNodeInfo for the sender to be cached by the receiver.
+    targetNodeInfoTs        @2  :UInt64;                # Timestamp the sender believes the target's node info to be at or zero if not sent
     kind :union {
-        question            @2  :Question;
+        question            @3  :Question;
-        statement           @3  :Statement;
+        statement           @4  :Statement;
-        answer              @4  :Answer;
+        answer              @5  :Answer;
     }
 }

veilid-core/run_tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+pushd $SCRIPTDIR 2>/dev/null
+if [[ "$1" == "wasm" ]]; then
+    WASM_BINDGEN_TEST_TIMEOUT=120 wasm-pack test --firefox --headless
+elif [[ "$1" == "ios" ]]; then
+    SYMROOT=/tmp/testout
+    APPNAME=veilidcore-tests
+    BUNDLENAME=com.veilid.veilidcore-tests
+    ID="$2"
+    if [[ "$ID" == "" ]]; then 
+        echo "No emulator ID specified"
+        exit 1
+    fi
+
+    # Build for simulator
+    xcrun xcodebuild -project src/tests/ios/$APPNAME/$APPNAME.xcodeproj/ -scheme $APPNAME -destination "generic/platform=iOS Simulator" SYMROOT=$SYMROOT
+
+    # Run in temporary simulator
+    xcrun simctl install $ID $SYMROOT/Debug-iphonesimulator/$APPNAME.app
+    xcrun simctl spawn $ID log stream --level debug --predicate "subsystem == \"$BUNDLENAME\"" &
+    xcrun simctl launch --console $ID $BUNDLENAME
+    sleep 1 # Ensure the last log lines print
+    kill -INT %1
+
+    # Clean up build output
+    rm -rf /tmp/testout
+
+elif [[ "$1" == "android" ]]; then
+    ID="$2"
+    if [[ "$ID" == "" ]]; then 
+        echo "No emulator ID specified"
+        exit 1
+    fi
+    APPNAME=veilid_core_android_tests
+    APPID=com.veilid.veilid_core_android_tests
+    ACTIVITYNAME=MainActivity
+    pushd src/tests/android/$APPNAME >/dev/null
+    # Build apk
+    ./gradlew assembleDebug
+    # Wait for boot
+    adb -s $ID wait-for-device
+    # Install app
+    adb -s $ID install -r ./app/build/outputs/apk/debug/app-debug.apk 
+    # Start activity
+    adb -s $ID shell am start-activity -W $APPID/.$ACTIVITYNAME
+    # Get the pid of the program
+    APP_PID=`adb -s $ID shell pidof -s $APPID`
+    # Print the logcat
+    adb -s $ID shell logcat --pid=$APP_PID veilid-core:V *:S &
+    # Wait for the pid to be done
+    while [ "$(adb -s $ID shell pidof -s $APPID)" != "" ]; do
+        sleep 1
+    done
+    # Terminate logcat
+    kill %1
+    # Finished
+    popd >/dev/null
+
+else
+    cargo test --features=rt-tokio
+    cargo test --features=rt-async-std
+fi
+popd 2>/dev/null

veilid-core/run_windows_tests.bat

@@ -0,0 +1,4 @@
+@echo off
+cargo test --features=rt-tokio -- --nocapture
+cargo test --features=rt-async-std -- --nocapture
+

veilid-core/src/api_tracing_layer.rs

@@ -1,6 +1,6 @@
 use crate::core_context::*;
 use crate::veilid_api::*;
-use crate::xx::*;
+use crate::*;
 use core::fmt::Write;
 use once_cell::sync::OnceCell;
 use tracing_subscriber::*;

veilid-core/src/attachment_manager.rs

@@ -1,111 +1,13 @@
-use crate::callback_state_machine::*;
 use crate::crypto::Crypto;
 use crate::network_manager::*;
 use crate::routing_table::*;
-use crate::xx::*;
 use crate::*;
-use core::convert::TryFrom;
-use core::fmt;
-use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
-use serde::*;
-
-state_machine! {
-    derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,)
-    pub Attachment(Detached)
-//---
-    Detached(AttachRequested) => Attaching [StartAttachment],
-    Attaching => {
-        AttachmentStopped => Detached,
-        WeakPeers => AttachedWeak,
-        GoodPeers => AttachedGood,
-        StrongPeers => AttachedStrong,
-        FullPeers => FullyAttached,
-        TooManyPeers => OverAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    AttachedWeak => {
-        NoPeers => Attaching,
-        GoodPeers => AttachedGood,
-        StrongPeers => AttachedStrong,
-        FullPeers => FullyAttached,
-        TooManyPeers => OverAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    AttachedGood => {
-        NoPeers => Attaching,
-        WeakPeers => AttachedWeak,
-        StrongPeers => AttachedStrong,
-        FullPeers => FullyAttached,
-        TooManyPeers => OverAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    AttachedStrong => {
-        NoPeers => Attaching,
-        WeakPeers => AttachedWeak,
-        GoodPeers => AttachedGood,
-        FullPeers => FullyAttached,
-        TooManyPeers => OverAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    FullyAttached => {
-        NoPeers => Attaching,
-        WeakPeers => AttachedWeak,
-        GoodPeers => AttachedGood,
-        StrongPeers => AttachedStrong,
-        TooManyPeers => OverAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    OverAttached => {
-        NoPeers => Attaching,
-        WeakPeers => AttachedWeak,
-        GoodPeers => AttachedGood,
-        StrongPeers => AttachedStrong,
-        FullPeers => FullyAttached,
-        DetachRequested => Detaching [StopAttachment]
-    },
-    Detaching => {
-        AttachmentStopped => Detached,
-    },
-}
-
-impl fmt::Display for AttachmentState {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        let out = match self {
-            AttachmentState::Attaching => "attaching".to_owned(),
-            AttachmentState::AttachedWeak => "attached_weak".to_owned(),
-            AttachmentState::AttachedGood => "attached_good".to_owned(),
-            AttachmentState::AttachedStrong => "attached_strong".to_owned(),
-            AttachmentState::FullyAttached => "fully_attached".to_owned(),
-            AttachmentState::OverAttached => "over_attached".to_owned(),
-            AttachmentState::Detaching => "detaching".to_owned(),
-            AttachmentState::Detached => "detached".to_owned(),
-        };
-        write!(f, "{}", out)
-    }
-}
-
-impl TryFrom<String> for AttachmentState {
-    type Error = ();
-
-    fn try_from(s: String) -> Result<Self, Self::Error> {
-        Ok(match s.as_str() {
-            "attaching" => AttachmentState::Attaching,
-            "attached_weak" => AttachmentState::AttachedWeak,
-            "attached_good" => AttachmentState::AttachedGood,
-            "attached_strong" => AttachmentState::AttachedStrong,
-            "fully_attached" => AttachmentState::FullyAttached,
-            "over_attached" => AttachmentState::OverAttached,
-            "detaching" => AttachmentState::Detaching,
-            "detached" => AttachmentState::Detached,
-            _ => return Err(()),
-        })
-    }
-}
 
 pub struct AttachmentManagerInner {
-    attachment_machine: CallbackStateMachine<Attachment>,
+    last_attachment_state: AttachmentState,
+    last_routing_table_health: Option<RoutingTableHealth>,
     maintain_peers: bool,
-    attach_timestamp: Option<u64>,
+    attach_ts: Option<Timestamp>,
     update_callback: Option<UpdateCallback>,
     attachment_maintainer_jh: Option<MustJoinHandle<()>>,
 }
@@ -142,9 +44,10 @@ impl AttachmentManager {
     }
     fn new_inner() -> AttachmentManagerInner {
         AttachmentManagerInner {
-            attachment_machine: CallbackStateMachine::new(),
+            last_attachment_state: AttachmentState::Detached,
+            last_routing_table_health: None,
             maintain_peers: false,
-            attach_timestamp: None,
+            attach_ts: None,
             update_callback: None,
             attachment_maintainer_jh: None,
         }
@@ -177,84 +80,107 @@ impl AttachmentManager {
     }
 
     pub fn is_attached(&self) -> bool {
-        let s = self.inner.lock().attachment_machine.state();
+        let s = self.inner.lock().last_attachment_state;
         !matches!(s, AttachmentState::Detached | AttachmentState::Detaching)
     }
     pub fn is_detached(&self) -> bool {
-        let s = self.inner.lock().attachment_machine.state();
+        let s = self.inner.lock().last_attachment_state;
         matches!(s, AttachmentState::Detached)
     }
 
-    pub fn get_attach_timestamp(&self) -> Option<u64> {
+    pub fn get_attach_timestamp(&self) -> Option<Timestamp> {
-        self.inner.lock().attach_timestamp
+        self.inner.lock().attach_ts
     }
 
     fn translate_routing_table_health(
-        health: RoutingTableHealth,
+        health: &RoutingTableHealth,
         config: &VeilidConfigRoutingTable,
-    ) -> AttachmentInput {
+    ) -> AttachmentState {
         if health.reliable_entry_count >= config.limit_over_attached.try_into().unwrap() {
-            return AttachmentInput::TooManyPeers;
+            return AttachmentState::OverAttached;
         }
         if health.reliable_entry_count >= config.limit_fully_attached.try_into().unwrap() {
-            return AttachmentInput::FullPeers;
+            return AttachmentState::FullyAttached;
         }
         if health.reliable_entry_count >= config.limit_attached_strong.try_into().unwrap() {
-            return AttachmentInput::StrongPeers;
+            return AttachmentState::AttachedStrong;
         }
         if health.reliable_entry_count >= config.limit_attached_good.try_into().unwrap() {
-            return AttachmentInput::GoodPeers;
+            return AttachmentState::AttachedGood;
         }
         if health.reliable_entry_count >= config.limit_attached_weak.try_into().unwrap()
             || health.unreliable_entry_count >= config.limit_attached_weak.try_into().unwrap()
         {
-            return AttachmentInput::WeakPeers;
+            return AttachmentState::AttachedWeak;
-        }
-        AttachmentInput::NoPeers
-    }
-    fn translate_attachment_state(state: &AttachmentState) -> AttachmentInput {
-        match state {
-            AttachmentState::OverAttached => AttachmentInput::TooManyPeers,
-            AttachmentState::FullyAttached => AttachmentInput::FullPeers,
-            AttachmentState::AttachedStrong => AttachmentInput::StrongPeers,
-            AttachmentState::AttachedGood => AttachmentInput::GoodPeers,
-            AttachmentState::AttachedWeak => AttachmentInput::WeakPeers,
-            AttachmentState::Attaching => AttachmentInput::NoPeers,
-            _ => panic!("Invalid state"),
         }
+        AttachmentState::Attaching
     }
 
-    async fn update_attachment(&self) {
+    /// Update attachment and network readiness state
-        let new_peer_state_input = {
+    /// and possibly send a VeilidUpdate::Attachment
-            let inner = self.inner.lock();
+    fn update_attachment(&self) {
+        // update the routing table health
+        let routing_table = self.network_manager().routing_table();
+        let health = routing_table.get_routing_table_health();
+        let opt_update = {
+            let mut inner = self.inner.lock();
 
-            let old_peer_state_input =
+            // Check if the routing table health is different
-                AttachmentManager::translate_attachment_state(&inner.attachment_machine.state());
+            if let Some(last_routing_table_health) = &inner.last_routing_table_health {
+                // If things are the same, just return
+                if last_routing_table_health == &health {
+                    return;
+                }
+            }
 
-            // get reliable peer count from routing table
+            // Swap in new health numbers
-            let routing_table = self.network_manager().routing_table();
+            let opt_previous_health = inner.last_routing_table_health.take();
-            let health = routing_table.get_routing_table_health();
+            inner.last_routing_table_health = Some(health.clone());
+
+            // Calculate new attachment state
             let config = self.config();
             let routing_table_config = &config.get().network.routing_table;
+            let previous_attachment_state = inner.last_attachment_state;
+            inner.last_attachment_state =
+                AttachmentManager::translate_routing_table_health(&health, routing_table_config);
 
-            let new_peer_state_input =
+            // If we don't have an update callback yet for some reason, just return now
-                AttachmentManager::translate_routing_table_health(health, routing_table_config);
+            let Some(update_callback) = inner.update_callback.clone() else {
+                return;
+            };
 
-            if old_peer_state_input == new_peer_state_input {
+            // Send update if one of:
-                None
+            // * the attachment state has changed
+            // * routing domain readiness has changed
+            // * this is our first routing table health check
+            let send_update = previous_attachment_state != inner.last_attachment_state
+                || opt_previous_health
+                    .map(|x| {
+                        x.public_internet_ready != health.public_internet_ready
+                            || x.local_network_ready != health.local_network_ready
+                    })
+                    .unwrap_or(true);
+            if send_update {
+                Some((update_callback, Self::get_veilid_state_inner(&*inner)))
             } else {
-                Some(new_peer_state_input)
+                None
             }
         };
-        if let Some(next_input) = new_peer_state_input {
+
-            let _ = self.process_input(&next_input).await;
+        // Send the update outside of the lock
+        if let Some(update) = opt_update {
+            (update.0)(VeilidUpdate::Attachment(update.1));
         }
     }
 
     #[instrument(level = "debug", skip(self))]
     async fn attachment_maintainer(self) {
-        debug!("attachment starting");
+        {
-        self.inner.lock().attach_timestamp = Some(intf::get_timestamp());
+            let mut inner = self.inner.lock();
+            inner.last_attachment_state = AttachmentState::Attaching;
+            inner.attach_ts = Some(get_aligned_timestamp());
+            debug!("attachment starting");
+        }
         let netman = self.network_manager();
 
         let mut restart;
@@ -283,13 +209,21 @@ impl AttachmentManager {
                     break;
                 }
 
-                self.update_attachment().await;
+                // Update attachment and network readiness state
+                // and possibly send a VeilidUpdate::Attachment
+                self.update_attachment();
 
                 // sleep should be at the end in case maintain_peers changes state
-                intf::sleep(1000).await;
+                sleep(1000).await;
             }
             debug!("stopped maintaining peers");
 
+            if !restart {
+                let mut inner = self.inner.lock();
+                inner.last_attachment_state = AttachmentState::Detaching;
+                debug!("attachment stopping");
+            }
+
             debug!("stopping network");
             netman.shutdown().await;
 
@@ -299,16 +233,15 @@ impl AttachmentManager {
 
             debug!("completely restarting attachment");
             // chill out for a second first, give network stack time to settle out
-            intf::sleep(1000).await;
+            sleep(1000).await;
         }
 
-        trace!("stopping attachment");
+        {
-        let attachment_machine = self.inner.lock().attachment_machine.clone();
+            let mut inner = self.inner.lock();
-        let _output = attachment_machine
+            inner.last_attachment_state = AttachmentState::Detached;
-            .consume(&AttachmentInput::AttachmentStopped)
+            inner.attach_ts = None;
-            .await;
+            debug!("attachment stopped");
-        debug!("attachment stopped");
+        }
-        self.inner.lock().attach_timestamp = None;
     }
 
     #[instrument(level = "debug", skip_all, err)]
@@ -317,15 +250,7 @@ impl AttachmentManager {
         {
             let mut inner = self.inner.lock();
             inner.update_callback = Some(update_callback.clone());
-            let update_callback2 = update_callback.clone();
+        }
-            inner.attachment_machine.set_state_change_callback(Arc::new(
-                move |_old_state: AttachmentState, new_state: AttachmentState| {
-                    update_callback2(VeilidUpdate::Attachment(VeilidStateAttachment {
-                        state: new_state,
-                    }))
-                },
-            ));
-        };
 
         self.network_manager().init(update_callback).await?;
 
@@ -341,18 +266,20 @@ impl AttachmentManager {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn attach(&self) {
+    pub async fn attach(&self) -> bool {
         // Create long-running connection maintenance routine
         let mut inner = self.inner.lock();
         if inner.attachment_maintainer_jh.is_some() {
-            return;
+            return false;
         }
         inner.maintain_peers = true;
-        inner.attachment_maintainer_jh = Some(intf::spawn(self.clone().attachment_maintainer()));
+        inner.attachment_maintainer_jh = Some(spawn(self.clone().attachment_maintainer()));
+
+        true
     }
 
     #[instrument(level = "trace", skip(self))]
-    async fn detach(&self) {
+    pub async fn detach(&self) -> bool {
         let attachment_maintainer_jh = {
             let mut inner = self.inner.lock();
             let attachment_maintainer_jh = inner.attachment_maintainer_jh.take();
@@ -364,57 +291,34 @@ impl AttachmentManager {
         };
         if let Some(jh) = attachment_maintainer_jh {
             jh.await;
+            true
+        } else {
+            false
         }
     }
 
-    async fn handle_output(&self, output: &AttachmentOutput) {
+    pub fn get_attachment_state(&self) -> AttachmentState {
-        match output {
+        self.inner.lock().last_attachment_state
-            AttachmentOutput::StartAttachment => self.attach(),
+    }
-            AttachmentOutput::StopAttachment => self.detach().await,
+
+    fn get_veilid_state_inner(inner: &AttachmentManagerInner) -> VeilidStateAttachment {
+        VeilidStateAttachment {
+            state: inner.last_attachment_state,
+            public_internet_ready: inner
+                .last_routing_table_health
+                .as_ref()
+                .map(|x| x.public_internet_ready)
+                .unwrap_or(false),
+            local_network_ready: inner
+                .last_routing_table_health
+                .as_ref()
+                .map(|x| x.local_network_ready)
+                .unwrap_or(false),
         }
     }
 
-    async fn process_input(&self, input: &AttachmentInput) -> EyreResult<()> {
-        let attachment_machine = self.inner.lock().attachment_machine.clone();
-        let output = attachment_machine.consume(input).await;
-        match output {
-            Err(e) => Err(eyre!(
-                "invalid input '{:?}' for state machine in state '{:?}': {:?}",
-                input,
-                attachment_machine.state(),
-                e
-            )),
-            Ok(v) => {
-                if let Some(o) = v {
-                    self.handle_output(&o).await;
-                }
-                Ok(())
-            }
-        }
-    }
-
-    #[instrument(level = "trace", skip(self), err)]
-    pub async fn request_attach(&self) -> EyreResult<()> {
-        self.process_input(&AttachmentInput::AttachRequested)
-            .await
-            .map_err(|e| eyre!("Attach request failed: {}", e))
-    }
-
-    #[instrument(level = "trace", skip(self), err)]
-    pub async fn request_detach(&self) -> EyreResult<()> {
-        self.process_input(&AttachmentInput::DetachRequested)
-            .await
-            .map_err(|e| eyre!("Detach request failed: {}", e))
-    }
-
-    pub fn get_state(&self) -> AttachmentState {
-        let attachment_machine = self.inner.lock().attachment_machine.clone();
-        attachment_machine.state()
-    }
-
     pub fn get_veilid_state(&self) -> VeilidStateAttachment {
-        VeilidStateAttachment {
+        let inner = self.inner.lock();
-            state: self.get_state(),
+        Self::get_veilid_state_inner(&*inner)
-        }
     }
 }

veilid-core/src/core_context.rs

@@ -3,7 +3,7 @@ use crate::attachment_manager::*;
 use crate::crypto::Crypto;
 use crate::veilid_api::*;
 use crate::veilid_config::*;
-use crate::xx::*;
+use crate::*;
 
 pub type UpdateCallback = Arc<dyn Fn(VeilidUpdate) + Send + Sync>;
 
@@ -201,9 +201,8 @@ impl VeilidCoreContext {
     ) -> Result<VeilidCoreContext, VeilidAPIError> {
         cfg_if! {
             if #[cfg(target_os = "android")] {
-                if crate::intf::utils::android::ANDROID_GLOBALS.lock().is_none() {
+                if !crate::intf::android::is_android_ready() {
-                    error!("Android globals are not set up");
+                    apibail_internal!("Android globals are not set up");
-                    return Err(VeilidAPIError::Internal { message: "Android globals are not set up".to_owned() });
                 }
             }
         }
@@ -251,7 +250,7 @@ pub async fn api_startup(
     // See if we have an API started up already
     let mut initialized_lock = INITIALIZED.lock().await;
     if *initialized_lock {
-        return Err(VeilidAPIError::AlreadyInitialized);
+        apibail_already_initialized!();
     }
 
     // Create core context
@@ -274,7 +273,7 @@ pub async fn api_startup_json(
     // See if we have an API started up already
     let mut initialized_lock = INITIALIZED.lock().await;
     if *initialized_lock {
-        return Err(VeilidAPIError::AlreadyInitialized);
+        apibail_already_initialized!();
     }
 
     // Create core context

veilid-core/src/crypto/envelope.rs

@@ -2,7 +2,6 @@
 #![allow(clippy::absurd_extreme_comparisons)]
 use super::*;
 use crate::routing_table::VersionRange;
-use crate::xx::*;
 use crate::*;
 use core::convert::TryInto;
 
@@ -45,7 +44,7 @@ pub struct Envelope {
     version: u8,
     min_version: u8,
     max_version: u8,
-    timestamp: u64,
+    timestamp: Timestamp,
     nonce: EnvelopeNonce,
     sender_id: DHTKey,
     recipient_id: DHTKey,
@@ -54,7 +53,7 @@ pub struct Envelope {
 impl Envelope {
     pub fn new(
         version: u8,
-        timestamp: u64,
+        timestamp: Timestamp,
         nonce: EnvelopeNonce,
         sender_id: DHTKey,
         recipient_id: DHTKey,
@@ -76,7 +75,7 @@ impl Envelope {
         // Ensure we are at least the length of the envelope
         // Silent drop here, as we use zero length packets as part of the protocol for hole punching
         if data.len() < MIN_ENVELOPE_SIZE {
-            return Err(VeilidAPIError::generic("envelope data too small"));
+            apibail_generic!("envelope data too small");
         }
 
         // Verify magic number
@@ -84,31 +83,28 @@ impl Envelope {
             .try_into()
             .map_err(VeilidAPIError::internal)?;
         if magic != *ENVELOPE_MAGIC {
-            return Err(VeilidAPIError::generic("bad magic number"));
+            apibail_generic!("bad magic number");
         }
 
         // Check version
         let version = data[0x04];
         if version > MAX_CRYPTO_VERSION || version < MIN_CRYPTO_VERSION {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!("unsupported cryptography version", version);
-                "unsupported cryptography version",
-                version,
-            ));
         }
 
         // Get min version
         let min_version = data[0x05];
         if min_version > version {
-            return Err(VeilidAPIError::parse_error("version too low", version));
+            apibail_parse_error!("version too low", version);
         }
 
         // Get max version
         let max_version = data[0x06];
         if version > max_version {
-            return Err(VeilidAPIError::parse_error("version too high", version));
+            apibail_parse_error!("version too high", version);
         }
         if min_version > max_version {
-            return Err(VeilidAPIError::generic("version information invalid"));
+            apibail_generic!("version information invalid");
         }
 
         // Get size and ensure it matches the size of the envelope and is less than the maximum message size
@@ -118,25 +114,26 @@ impl Envelope {
                 .map_err(VeilidAPIError::internal)?,
         );
         if (size as usize) > MAX_ENVELOPE_SIZE {
-            return Err(VeilidAPIError::parse_error("envelope too large", size));
+            apibail_parse_error!("envelope too large", size);
         }
         if (size as usize) != data.len() {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!(
                 "size doesn't match envelope size",
                 format!(
                     "size doesn't match envelope size: size={} data.len()={}",
                     size,
                     data.len()
-                ),
+                )
-            ));
+            );
         }
 
         // Get the timestamp
-        let timestamp: u64 = u64::from_le_bytes(
+        let timestamp: Timestamp = u64::from_le_bytes(
             data[0x0A..0x12]
                 .try_into()
                 .map_err(VeilidAPIError::internal)?,
-        );
+        )
+        .into();
 
         // Get nonce and sender node id
         let nonce: EnvelopeNonce = data[0x12..0x2A]
@@ -153,10 +150,10 @@ impl Envelope {
 
         // Ensure sender_id and recipient_id are not the same
         if sender_id == recipient_id {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!(
                 "sender_id should not be same as recipient_id",
-                recipient_id.encode(),
+                recipient_id.encode()
-            ));
+            );
         }
 
         // Get signature
@@ -206,10 +203,7 @@ impl Envelope {
         // Ensure body isn't too long
         let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE;
         if envelope_size > MAX_ENVELOPE_SIZE {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!("envelope size is too large", envelope_size);
-                "envelope size is too large",
-                envelope_size,
-            ));
         }
         let mut data = vec![0u8; envelope_size];
 
@@ -224,7 +218,7 @@ impl Envelope {
         // Write size
         data[0x08..0x0A].copy_from_slice(&(envelope_size as u16).to_le_bytes());
         // Write timestamp
-        data[0x0A..0x12].copy_from_slice(&self.timestamp.to_le_bytes());
+        data[0x0A..0x12].copy_from_slice(&self.timestamp.as_u64().to_le_bytes());
         // Write nonce
         data[0x12..0x2A].copy_from_slice(&self.nonce);
         // Write sender node id
@@ -267,7 +261,7 @@ impl Envelope {
         }
     }
 
-    pub fn get_timestamp(&self) -> u64 {
+    pub fn get_timestamp(&self) -> Timestamp {
         self.timestamp
     }
 

veilid-core/src/crypto/key.rs

@@ -1,5 +1,3 @@
-use crate::veilid_rng::*;
-use crate::xx::*;
 use crate::*;
 
 use core::cmp::{Eq, Ord, PartialEq, PartialOrd};

veilid-core/src/crypto/mod.rs

@@ -13,7 +13,6 @@ pub use value::*;
 pub const MIN_CRYPTO_VERSION: u8 = 0u8;
 pub const MAX_CRYPTO_VERSION: u8 = 0u8;
 
-use crate::xx::*;
 use crate::*;
 use chacha20::cipher::{KeyIvInit, StreamCipher};
 use chacha20::XChaCha20;
@@ -25,6 +24,7 @@ use ed25519_dalek as ed;
 use hashlink::linked_hash_map::Entry;
 use hashlink::LruCache;
 use serde::{Deserialize, Serialize};
+
 use x25519_dalek as xd;
 
 pub type SharedSecret = [u8; 32];
@@ -132,12 +132,12 @@ impl Crypto {
             drop(db);
             table_store.delete("crypto_caches").await?;
             db = table_store.open("crypto_caches", 1).await?;
-            db.store(0, b"node_id", &node_id.unwrap().bytes)?;
+            db.store(0, b"node_id", &node_id.unwrap().bytes).await?;
         }
 
         // Schedule flushing
         let this = self.clone();
-        let flush_future = intf::interval(60000, move || {
+        let flush_future = interval(60000, move || {
             let this = this.clone();
             async move {
                 if let Err(e) = this.flush().await {
@@ -159,7 +159,7 @@ impl Crypto {
         };
 
         let db = table_store.open("crypto_caches", 1).await?;
-        db.store(0, b"dh_cache", &cache_bytes)?;
+        db.store(0, b"dh_cache", &cache_bytes).await?;
         Ok(())
     }
 
@@ -229,13 +229,13 @@ impl Crypto {
 
     pub fn get_random_nonce() -> Nonce {
         let mut nonce = [0u8; 24];
-        intf::random_bytes(&mut nonce).unwrap();
+        random_bytes(&mut nonce).unwrap();
         nonce
     }
 
     pub fn get_random_secret() -> SharedSecret {
         let mut s = [0u8; 32];
-        intf::random_bytes(&mut s).unwrap();
+        random_bytes(&mut s).unwrap();
         s
     }
 

veilid-core/src/crypto/receipt.rs

@@ -1,7 +1,6 @@
 #![allow(dead_code)]
 #![allow(clippy::absurd_extreme_comparisons)]
 use super::*;
-use crate::xx::*;
 use crate::*;
 use core::convert::TryInto;
 use data_encoding::BASE64URL_NOPAD;
@@ -59,10 +58,10 @@ impl Receipt {
         extra_data: D,
     ) -> Result<Self, VeilidAPIError> {
         if extra_data.as_ref().len() > MAX_EXTRA_DATA_SIZE {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!(
                 "extra data too large for receipt",
-                extra_data.as_ref().len(),
+                extra_data.as_ref().len()
-            ));
+            );
         }
         Ok(Self {
             version,
@@ -75,7 +74,7 @@ impl Receipt {
     pub fn from_signed_data(data: &[u8]) -> Result<Receipt, VeilidAPIError> {
         // Ensure we are at least the length of the envelope
         if data.len() < MIN_RECEIPT_SIZE {
-            return Err(VeilidAPIError::parse_error("receipt too small", data.len()));
+            apibail_parse_error!("receipt too small", data.len());
         }
 
         // Verify magic number
@@ -83,16 +82,13 @@ impl Receipt {
             .try_into()
             .map_err(VeilidAPIError::internal)?;
         if magic != *RECEIPT_MAGIC {
-            return Err(VeilidAPIError::generic("bad magic number"));
+            apibail_generic!("bad magic number");
         }
 
         // Check version
         let version = data[0x04];
         if version > MAX_CRYPTO_VERSION || version < MIN_CRYPTO_VERSION {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!("unsupported cryptography version", version);
-                "unsupported cryptography version",
-                version,
-            ));
         }
 
         // Get size and ensure it matches the size of the envelope and is less than the maximum message size
@@ -102,16 +98,13 @@ impl Receipt {
                 .map_err(VeilidAPIError::internal)?,
         );
         if (size as usize) > MAX_RECEIPT_SIZE {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!("receipt size is too large", size);
-                "receipt size is too large",
-                size,
-            ));
         }
         if (size as usize) != data.len() {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!(
                 "size doesn't match receipt size",
-                format!("size={} data.len()={}", size, data.len()),
+                format!("size={} data.len()={}", size, data.len())
-            ));
+            );
         }
 
         // Get sender id
@@ -153,10 +146,7 @@ impl Receipt {
         // Ensure extra data isn't too long
         let receipt_size: usize = self.extra_data.len() + MIN_RECEIPT_SIZE;
         if receipt_size > MAX_RECEIPT_SIZE {
-            return Err(VeilidAPIError::parse_error(
+            apibail_parse_error!("receipt too large", receipt_size);
-                "receipt too large",
-                receipt_size,
-            ));
         }
         let mut data: Vec<u8> = vec![0u8; receipt_size];
 

veilid-core/src/crypto/tests/test_crypto.rs

@@ -1,7 +1,5 @@
 use super::*;
 use crate::tests::common::test_veilid_config::*;
-use crate::xx::*;
-use crate::*;
 
 static LOREM_IPSUM:&[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ";
 

veilid-core/src/crypto/tests/test_dht_key.rs

@@ -1,7 +1,6 @@
 #![allow(clippy::bool_assert_comparison)]
 
 use super::*;
-use crate::xx::*;
 use core::convert::TryFrom;
 
 static LOREM_IPSUM:&str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ";

veilid-core/src/crypto/tests/test_envelope_receipt.rs

@@ -1,7 +1,5 @@
 use super::*;
 use crate::tests::common::test_veilid_config::*;
-use crate::xx::*;
-use crate::*;
 
 pub async fn test_envelope_round_trip() {
     info!("--- test envelope round trip ---");
@@ -14,7 +12,7 @@ pub async fn test_envelope_round_trip() {
     let crypto = api.crypto().unwrap();
 
     // Create envelope
-    let ts = 0x12345678ABCDEF69u64;
+    let ts = Timestamp::from(0x12345678ABCDEF69u64);
     let nonce = Crypto::get_random_nonce();
     let (sender_id, sender_secret) = generate_secret();
     let (recipient_id, recipient_secret) = generate_secret();

veilid-core/src/intf/native/android/get_directories.rs

@@ -1,6 +1,7 @@
 use super::*;
-use crate::xx::*;
+use jni::objects::JString;
 
+#[allow(dead_code)]
 pub fn get_files_dir() -> String {
     let aglock = ANDROID_GLOBALS.lock();
     let ag = aglock.as_ref().unwrap();
@@ -25,6 +26,7 @@ pub fn get_files_dir() -> String {
     .unwrap()
 }
 
+#[allow(dead_code)]
 pub fn get_cache_dir() -> String {
     let aglock = ANDROID_GLOBALS.lock();
     let ag = aglock.as_ref().unwrap();

veilid-core/src/intf/native/android/mod.rs

@@ -0,0 +1,53 @@
+mod get_directories;
+pub use get_directories::*;
+
+use crate::*;
+use jni::errors::Result as JniResult;
+use jni::{objects::GlobalRef, objects::JObject, JNIEnv, JavaVM};
+use lazy_static::*;
+
+pub struct AndroidGlobals {
+    pub vm: JavaVM,
+    pub ctx: GlobalRef,
+}
+
+impl Drop for AndroidGlobals {
+    fn drop(&mut self) {
+        // Ensure we're attached before dropping GlobalRef
+        self.vm.attach_current_thread_as_daemon().unwrap();
+    }
+}
+
+lazy_static! {
+    pub static ref ANDROID_GLOBALS: Arc<Mutex<Option<AndroidGlobals>>> = Arc::new(Mutex::new(None));
+}
+
+pub fn veilid_core_setup_android(env: JNIEnv, ctx: JObject) {
+    *ANDROID_GLOBALS.lock() = Some(AndroidGlobals {
+        vm: env.get_java_vm().unwrap(),
+        ctx: env.new_global_ref(ctx).unwrap(),
+    });
+}
+
+pub fn is_android_ready() -> bool {
+    ANDROID_GLOBALS.lock().is_some()
+}
+
+pub fn get_android_globals() -> (JavaVM, GlobalRef) {
+    let globals_locked = ANDROID_GLOBALS.lock();
+    let globals = globals_locked.as_ref().unwrap();
+    let env = globals.vm.attach_current_thread_as_daemon().unwrap();
+    let vm = env.get_java_vm().unwrap();
+    let ctx = globals.ctx.clone();
+    (vm, ctx)
+}
+
+pub fn with_null_local_frame<'b, T, F>(env: JNIEnv<'b>, s: i32, f: F) -> JniResult<T>
+where
+    F: FnOnce() -> JniResult<T>,
+{
+    env.push_local_frame(s)?;
+    let out = f();
+    env.pop_local_frame(JObject::null())?;
+    out
+}

veilid-core/src/intf/native/block_store.rs

@@ -1,4 +1,3 @@
-use crate::xx::*;
 use crate::*;
 
 struct BlockStoreInner {

veilid-core/src/intf/native/mod.rs

@@ -2,9 +2,12 @@ mod block_store;
 mod protected_store;
 mod system;
 mod table_store;
-pub mod utils;
 
 pub use block_store::*;
 pub use protected_store::*;
 pub use system::*;
 pub use table_store::*;
+
+#[cfg(target_os = "android")]
+pub mod android;
+pub mod network_interfaces;

veilid-core/src/intf/native/network_interfaces/apple.rs

@@ -1,5 +1,5 @@
 use super::*;
-use crate::*;
+
 use libc::{
     close, freeifaddrs, getifaddrs, if_nametoindex, ifaddrs, ioctl, pid_t, sockaddr, sockaddr_in6,
     socket, sysctl, time_t, AF_INET6, CTL_NET, IFF_BROADCAST, IFF_LOOPBACK, IFF_RUNNING, IFNAMSIZ,

veilid-core/src/intf/native/network_interfaces/mod.rs

@@ -1,7 +1,7 @@
-use crate::xx::*;
-use core::fmt;
 mod tools;
 
+use crate::*;
+
 cfg_if::cfg_if! {
     if #[cfg(any(target_os = "linux", target_os = "android"))] {
         mod netlink;

veilid-core/src/intf/native/network_interfaces/netlink.rs

@@ -1,5 +1,4 @@
 use super::*;
-use crate::*;
 
 use alloc::collections::btree_map::Entry;
 use futures_util::stream::TryStreamExt;
@@ -322,7 +321,7 @@ impl PlatformSupportNetlink {
             .wrap_err("failed to create rtnetlink socket")?;
 
         // Spawn a connection handler
-        let connection_jh = intf::spawn(connection);
+        let connection_jh = spawn(connection);
 
         // Save the connection
         self.connection_jh = Some(connection_jh);

veilid-core/src/intf/native/network_interfaces/windows.rs

@@ -63,7 +63,8 @@ impl PlatformSupportWindows {
         // }
 
         // Iterate all the interfaces
-        let windows_interfaces = WindowsInterfaces::new().wrap_err("failed to get windows interfaces")?;
+        let windows_interfaces =
+            WindowsInterfaces::new().wrap_err("failed to get windows interfaces")?;
         for windows_interface in windows_interfaces.iter() {
             // Get name
             let intf_name = windows_interface.name();

veilid-core/src/intf/native/protected_store.rs

@@ -1,4 +1,3 @@
-use crate::xx::*;
 use crate::*;
 use data_encoding::BASE64URL_NOPAD;
 use keyring_manager::*;
@@ -56,7 +55,7 @@ impl ProtectedStore {
                 // Attempt to open the secure keyring
                 cfg_if! {
                     if #[cfg(target_os = "android")] {
-                        inner.keyring_manager = KeyringManager::new_secure(&c.program_name, intf::native::utils::android::get_android_globals()).ok();
+                        inner.keyring_manager = KeyringManager::new_secure(&c.program_name, crate::intf::android::get_android_globals()).ok();
                     } else {
                         inner.keyring_manager = KeyringManager::new_secure(&c.program_name).ok();
                     }

veilid-core/src/intf/native/system.rs

@@ -1,201 +1,11 @@
 #![allow(dead_code)]
 
-use crate::xx::*;
+use crate::*;
-use rand::prelude::*;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-pub fn get_timestamp() -> u64 {
-    match SystemTime::now().duration_since(UNIX_EPOCH) {
-        Ok(n) => n.as_micros() as u64,
-        Err(_) => panic!("SystemTime before UNIX_EPOCH!"),
-    }
-}
-
-// pub fn get_timestamp_string() -> String {
-//     let dt = chrono::Utc::now();
-//     dt.time().format("%H:%M:%S.3f").to_string()
-// }
-
-pub fn random_bytes(dest: &mut [u8]) -> EyreResult<()> {
-    let mut rng = rand::thread_rng();
-    rng.try_fill_bytes(dest).wrap_err("failed to fill bytes")
-}
-
-pub fn get_random_u32() -> u32 {
-    let mut rng = rand::thread_rng();
-    rng.next_u32()
-}
-
-pub fn get_random_u64() -> u64 {
-    let mut rng = rand::thread_rng();
-    rng.next_u64()
-}
-
-pub async fn sleep(millis: u32) {
-    if millis == 0 {
-        cfg_if! {
-            if #[cfg(feature="rt-async-std")] {
-                async_std::task::yield_now().await;
-            } else if #[cfg(feature="rt-tokio")] {
-                tokio::task::yield_now().await;
-            }
-        }
-    } else {
-        cfg_if! {
-            if #[cfg(feature="rt-async-std")] {
-                async_std::task::sleep(Duration::from_millis(u64::from(millis))).await;
-            } else if #[cfg(feature="rt-tokio")] {
-                tokio::time::sleep(Duration::from_millis(u64::from(millis))).await;
-            }
-        }
-    }
-}
-
-pub fn system_boxed<'a, Out>(
-    future: impl Future<Output = Out> + Send + 'a,
-) -> SendPinBoxFutureLifetime<'a, Out> {
-    Box::pin(future)
-}
-
-pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out>
-where
-    Out: Send + 'static,
-{
-    cfg_if! {
-        if #[cfg(feature="rt-async-std")] {
-            MustJoinHandle::new(async_std::task::spawn(future))
-        } else if #[cfg(feature="rt-tokio")] {
-            MustJoinHandle::new(tokio::task::spawn(future))
-        }
-    }
-}
-
-pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out>
-where
-    Out: 'static,
-{
-    cfg_if! {
-        if #[cfg(feature="rt-async-std")] {
-            MustJoinHandle::new(async_std::task::spawn_local(future))
-        } else if #[cfg(feature="rt-tokio")] {
-            MustJoinHandle::new(tokio::task::spawn_local(future))
-        }
-    }
-}
-
-// pub fn spawn_with_local_set<Out>(
-//     future: impl Future<Output = Out> + Send + 'static,
-// ) -> MustJoinHandle<Out>
-// where
-//     Out: Send + 'static,
-// {
-//     cfg_if! {
-//         if #[cfg(feature="rt-async-std")] {
-//             spawn(future)
-//         } else if #[cfg(feature="rt-tokio")] {
-//             MustJoinHandle::new(tokio::task::spawn_blocking(move || {
-//                 let rt = tokio::runtime::Handle::current();
-//                 rt.block_on(async {
-//                     let local = tokio::task::LocalSet::new();
-//                     local.run_until(future).await
-//                 })
-//             }))
-//         }
-//     }
-// }
-
-pub fn spawn_detached<Out>(future: impl Future<Output = Out> + Send + 'static)
-where
-    Out: Send + 'static,
-{
-    cfg_if! {
-        if #[cfg(feature="rt-async-std")] {
-            drop(async_std::task::spawn(future));
-        } else if #[cfg(feature="rt-tokio")] {
-            drop(tokio::task::spawn(future));
-        }
-    }
-}
-
-pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SendPinBoxFuture<()>
-where
-    F: Fn() -> FUT + Send + Sync + 'static,
-    FUT: Future<Output = ()> + Send,
-{
-    let e = Eventual::new();
-
-    let ie = e.clone();
-    let jh = spawn(async move {
-        while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
-            callback().await;
-        }
-    });
-
-    Box::pin(async move {
-        e.resolve().await;
-        jh.await;
-    })
-}
-
-pub async fn timeout<F, T>(dur_ms: u32, f: F) -> Result<T, TimeoutError>
-where
-    F: Future<Output = T>,
-{
-    cfg_if! {
-        if #[cfg(feature="rt-async-std")] {
-            async_std::future::timeout(Duration::from_millis(dur_ms as u64), f).await.map_err(|e| e.into())
-        } else if #[cfg(feature="rt-tokio")] {
-            tokio::time::timeout(Duration::from_millis(dur_ms as u64), f).await.map_err(|e| e.into())
-        }
-    }
-}
-
-pub async fn blocking_wrapper<F, R>(blocking_task: F, err_result: R) -> R
-where
-    F: FnOnce() -> R + Send + 'static,
-    R: Send + 'static,
-{
-    // run blocking stuff in blocking thread
-    cfg_if! {
-        if #[cfg(feature="rt-async-std")] {
-            async_std::task::spawn_blocking(blocking_task).await
-        } else if #[cfg(feature="rt-tokio")] {
-            tokio::task::spawn_blocking(blocking_task).await.unwrap_or(err_result)
-        } else {
-            #[compile_error("must use an executor")]
-        }
-    }
-}
-
-pub fn get_concurrency() -> u32 {
-    std::thread::available_parallelism()
-        .map(|x| x.get())
-        .unwrap_or_else(|e| {
-            warn!("unable to get concurrency defaulting to single core: {}", e);
-            1
-        }) as u32
-}
 
 pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
     panic!("Native Veilid should never require an outbound relay");
 }
 
-/*
-pub fn async_callback<F, OF, EF, T, E>(fut: F, ok_fn: OF, err_fn: EF)
-where
-    F: Future<Output = Result<T, E>> + Send + 'static,
-    OF: FnOnce(T) + Send + 'static,
-    EF: FnOnce(E) + Send + 'static,
-{
-    spawn(Box::pin(async move {
-        match fut.await {
-            Ok(v) => ok_fn(v),
-            Err(e) => err_fn(e),
-        };
-    }));
-}
-*/
-
 /////////////////////////////////////////////////////////////////////////////////
 // Resolver
 //

veilid-core/src/intf/native/table_store.rs

@@ -1,5 +1,5 @@
-use crate::intf::table_db::*;
+use crate::intf::table_db::TableDBInner;
-use crate::xx::*;
+pub use crate::intf::table_db::{TableDB, TableDBTransaction};
 use crate::*;
 use keyvaluedb_sqlite::*;
 use std::path::PathBuf;

veilid-core/src/intf/native/utils/android/mod.rs

@@ -1,109 +0,0 @@
-// xxx : support for android older than API 24, if we need it someday
-//mod android_get_if_addrs;
-//pub use android_get_if_addrs::*;
-
-mod get_directories;
-pub use get_directories::*;
-
-use crate::veilid_config::VeilidConfigLogLevel;
-use crate::xx::*;
-use crate::*;
-use backtrace::Backtrace;
-use jni::errors::Result as JniResult;
-use jni::{objects::GlobalRef, objects::JObject, objects::JString, JNIEnv, JavaVM};
-use lazy_static::*;
-use std::panic;
-use tracing::*;
-use tracing_subscriber::prelude::*;
-use tracing_subscriber::*;
-
-pub struct AndroidGlobals {
-    pub vm: JavaVM,
-    pub ctx: GlobalRef,
-}
-
-impl Drop for AndroidGlobals {
-    fn drop(&mut self) {
-        // Ensure we're attached before dropping GlobalRef
-        self.vm.attach_current_thread_as_daemon().unwrap();
-    }
-}
-
-lazy_static! {
-    pub static ref ANDROID_GLOBALS: Arc<Mutex<Option<AndroidGlobals>>> = Arc::new(Mutex::new(None));
-}
-
-pub fn veilid_core_setup_android_no_log<'a>(env: JNIEnv<'a>, ctx: JObject<'a>) {
-    *ANDROID_GLOBALS.lock() = Some(AndroidGlobals {
-        vm: env.get_java_vm().unwrap(),
-        ctx: env.new_global_ref(ctx).unwrap(),
-    });
-}
-
-pub fn veilid_core_setup_android<'a>(
-    env: JNIEnv<'a>,
-    ctx: JObject<'a>,
-    log_tag: &'a str,
-    log_level: VeilidConfigLogLevel,
-) {
-    // Set up subscriber and layers
-    let subscriber = Registry::default();
-    let mut layers = Vec::new();
-    let mut filters = BTreeMap::new();
-    let filter = VeilidLayerFilter::new(log_level, None);
-    let layer = tracing_android::layer(log_tag)
-        .expect("failed to set up android logging")
-        .with_filter(filter.clone());
-    filters.insert("system", filter);
-    layers.push(layer.boxed());
-
-    let subscriber = subscriber.with(layers);
-    subscriber
-        .try_init()
-        .expect("failed to init android tracing");
-
-    // Set up panic hook for backtraces
-    panic::set_hook(Box::new(|panic_info| {
-        let bt = Backtrace::new();
-        if let Some(location) = panic_info.location() {
-            error!(
-                "panic occurred in file '{}' at line {}",
-                location.file(),
-                location.line(),
-            );
-        } else {
-            error!("panic occurred but can't get location information...");
-        }
-        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
-            error!("panic payload: {:?}", s);
-        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
-            error!("panic payload: {:?}", s);
-        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
-            error!("panic payload: {:?}", a);
-        } else {
-            error!("no panic payload");
-        }
-        error!("Backtrace:\n{:?}", bt);
-    }));
-
-    veilid_core_setup_android_no_log(env, ctx);
-}
-
-pub fn get_android_globals() -> (JavaVM, GlobalRef) {
-    let globals_locked = ANDROID_GLOBALS.lock();
-    let globals = globals_locked.as_ref().unwrap();
-    let env = globals.vm.attach_current_thread_as_daemon().unwrap();
-    let vm = env.get_java_vm().unwrap();
-    let ctx = globals.ctx.clone();
-    (vm, ctx)
-}
-
-pub fn with_null_local_frame<'b, T, F>(env: JNIEnv<'b>, s: i32, f: F) -> JniResult<T>
-where
-    F: FnOnce() -> JniResult<T>,
-{
-    env.push_local_frame(s)?;
-    let out = f();
-    env.pop_local_frame(JObject::null())?;
-    out
-}

veilid-core/src/intf/native/utils/ios_test_setup.rs

@@ -1,88 +0,0 @@
-use crate::xx::*;
-use backtrace::Backtrace;
-use log::*;
-use simplelog::*;
-use std::fs::OpenOptions;
-use std::panic;
-use std::path::{Path, PathBuf};
-
-pub fn veilid_core_setup<'a>(
-    log_tag: &'a str,
-    terminal_log: Option<Level>,
-    file_log: Option<(Level, &Path)>,
-) {
-    if let Err(e) = veilid_core_setup_internal(log_tag, terminal_log, file_log) {
-        panic!("failed to set up veilid-core: {}", e);
-    }
-}
-
-fn veilid_core_setup_internal<'a>(
-    _log_tag: &'a str,
-    terminal_log: Option<Level>,
-    file_log: Option<(Level, &Path)>,
-) -> Result<(), String> {
-    let mut logs: Vec<Box<dyn SharedLogger>> = Vec::new();
-
-    let mut cb = ConfigBuilder::new();
-    for ig in veilid_core::DEFAULT_LOG_IGNORE_LIST {
-        cb.add_filter_ignore_str(ig);
-    }
-
-    if let Some(level) = terminal_log {
-        logs.push(TermLogger::new(
-            level.to_level_filter(),
-            cb.build(),
-            TerminalMode::Mixed,
-            ColorChoice::Auto,
-        ))
-    }
-    if let Some((level, log_path)) = file_log {
-        let logfile = OpenOptions::new()
-            .truncate(true)
-            .create(true)
-            .write(true)
-            .open(log_path)
-            .map_err(|e| {
-                format!(
-                    "log open error: {} path={:?} all_dirs={:?}",
-                    e,
-                    log_path,
-                    std::fs::read_dir(std::env::var("HOME").unwrap())
-                        .unwrap()
-                        .map(|d| d.unwrap().path())
-                        .collect::<Vec<PathBuf>>()
-                )
-            })?;
-        logs.push(WriteLogger::new(
-            level.to_level_filter(),
-            cb.build(),
-            logfile,
-        ))
-    }
-    CombinedLogger::init(logs).map_err(|e| format!("logger init error: {}", e))?;
-
-    panic::set_hook(Box::new(|panic_info| {
-        let bt = Backtrace::new();
-        if let Some(location) = panic_info.location() {
-            error!(
-                "panic occurred in file '{}' at line {}",
-                location.file(),
-                location.line(),
-            );
-        } else {
-            error!("panic occurred but can't get location information...");
-        }
-        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
-            error!("panic payload: {:?}", s);
-        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
-            error!("panic payload: {:?}", s);
-        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
-            error!("panic payload: {:?}", a);
-        } else {
-            error!("no panic payload");
-        }
-        error!("Backtrace:\n{:?}", bt);
-    }));
-
-    Ok(())
-}

veilid-core/src/intf/native/utils/mod.rs

@@ -1,5 +0,0 @@
-#[cfg(target_os = "android")]
-pub mod android;
-#[cfg(all(target_os = "ios", feature = "ios_tests"))]
-pub mod ios_test_setup;
-pub mod network_interfaces;

veilid-core/src/intf/table_db.rs

@@ -1,4 +1,3 @@
-use crate::xx::*;
 use crate::*;
 use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 
@@ -18,13 +17,19 @@ pub struct TableDBInner {
     database: Database,
 }
 
+impl fmt::Debug for TableDBInner {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "TableDBInner(table={})", self.table)
+    }
+}
+
 impl Drop for TableDBInner {
     fn drop(&mut self) {
         self.table_store.on_table_db_drop(self.table.clone());
     }
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct TableDB {
     inner: Arc<Mutex<TableDBInner>>,
 }
@@ -69,51 +74,51 @@ impl TableDB {
     }
 
     /// Start a TableDB write transaction. The transaction object must be committed or rolled back before dropping.
-    pub fn transact<'a>(&'a self) -> TableDBTransaction<'a> {
+    pub fn transact(&self) -> TableDBTransaction {
         let dbt = {
             let db = &self.inner.lock().database;
             db.transaction()
         };
-        TableDBTransaction::new(self, dbt)
+        TableDBTransaction::new(self.clone(), dbt)
     }
 
     /// Store a key with a value in a column in the TableDB. Performs a single transaction immediately.
-    pub fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> {
+    pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> {
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let mut dbt = db.transaction();
         dbt.put(col, key, value);
-        db.write(dbt).wrap_err("failed to store key")
+        db.write(dbt).await.wrap_err("failed to store key")
     }
 
     /// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately.
-    pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
+    pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
     where
         T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
     {
         let v = to_rkyv(value)?;
 
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let mut dbt = db.transaction();
         dbt.put(col, key, v.as_slice());
-        db.write(dbt).wrap_err("failed to store key")
+        db.write(dbt).await.wrap_err("failed to store key")
     }
 
     /// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately.
-    pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
+    pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
     where
         T: serde::Serialize,
     {
         let v = serde_json::to_vec(value)?;
 
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let mut dbt = db.transaction();
         dbt.put(col, key, v.as_slice());
-        db.write(dbt).wrap_err("failed to store key")
+        db.write(dbt).await.wrap_err("failed to store key")
     }
 
     /// Read a key from a column in the TableDB immediately.
     pub fn load(&self, col: u32, key: &[u8]) -> EyreResult<Option<Vec<u8>>> {
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         db.get(col, key).wrap_err("failed to get key")
     }
 
@@ -126,7 +131,7 @@ impl TableDB {
         <T as RkyvArchive>::Archived:
             RkyvDeserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
     {
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let out = db.get(col, key).wrap_err("failed to get key")?;
         let b = match out {
             Some(v) => v,
@@ -143,7 +148,7 @@ impl TableDB {
     where
         T: for<'de> serde::Deserialize<'de>,
     {
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let out = db.get(col, key).wrap_err("failed to get key")?;
         let b = match out {
             Some(v) => v,
@@ -156,15 +161,15 @@ impl TableDB {
     }
 
     /// Delete key with from a column in the TableDB
-    pub fn delete(&self, col: u32, key: &[u8]) -> EyreResult<bool> {
+    pub async fn delete(&self, col: u32, key: &[u8]) -> EyreResult<bool> {
-        let db = &self.inner.lock().database;
+        let db = self.inner.lock().database.clone();
         let found = db.get(col, key).wrap_err("failed to get key")?;
         match found {
             None => Ok(false),
             Some(_) => {
                 let mut dbt = db.transaction();
                 dbt.delete(col, key);
-                db.write(dbt).wrap_err("failed to delete key")?;
+                db.write(dbt).await.wrap_err("failed to delete key")?;
                 Ok(true)
             }
         }
@@ -173,70 +178,96 @@ impl TableDB {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-/// A TableDB transaction
+struct TableDBTransactionInner {
-/// Atomically commits a group of writes or deletes to the TableDB
-pub struct TableDBTransaction<'a> {
-    db: &'a TableDB,
     dbt: Option<DBTransaction>,
-    _phantom: core::marker::PhantomData<&'a ()>,
 }
 
-impl<'a> TableDBTransaction<'a> {
+impl fmt::Debug for TableDBTransactionInner {
-    fn new(db: &'a TableDB, dbt: DBTransaction) -> Self {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "TableDBTransactionInner({})",
+            match &self.dbt {
+                Some(dbt) => format!("len={}", dbt.ops.len()),
+                None => "".to_owned(),
+            }
+        )
+    }
+}
+
+/// A TableDB transaction
+/// Atomically commits a group of writes or deletes to the TableDB
+#[derive(Debug, Clone)]
+pub struct TableDBTransaction {
+    db: TableDB,
+    inner: Arc<Mutex<TableDBTransactionInner>>,
+}
+
+impl TableDBTransaction {
+    fn new(db: TableDB, dbt: DBTransaction) -> Self {
         Self {
             db,
-            dbt: Some(dbt),
+            inner: Arc::new(Mutex::new(TableDBTransactionInner { dbt: Some(dbt) })),
-            _phantom: Default::default(),
         }
     }
 
     /// Commit the transaction. Performs all actions atomically.
-    pub fn commit(mut self) -> EyreResult<()> {
+    pub async fn commit(self) -> EyreResult<()> {
-        self.db
+        let dbt = {
-            .inner
+            let mut inner = self.inner.lock();
-            .lock()
+            inner
-            .database
+                .dbt
-            .write(self.dbt.take().unwrap())
+                .take()
-            .wrap_err("commit failed")
+                .ok_or_else(|| eyre!("transaction already completed"))?
+        };
+        let db = self.db.inner.lock().database.clone();
+        db.write(dbt)
+            .await
+            .wrap_err("commit failed, transaction lost")
     }
 
     /// Rollback the transaction. Does nothing to the TableDB.
-    pub fn rollback(mut self) {
+    pub fn rollback(self) {
-        self.dbt = None;
+        let mut inner = self.inner.lock();
+        inner.dbt = None;
     }
 
     /// Store a key with a value in a column in the TableDB
-    pub fn store(&mut self, col: u32, key: &[u8], value: &[u8]) {
+    pub fn store(&self, col: u32, key: &[u8], value: &[u8]) {
-        self.dbt.as_mut().unwrap().put(col, key, value);
+        let mut inner = self.inner.lock();
+        inner.dbt.as_mut().unwrap().put(col, key, value);
     }
 
     /// Store a key in rkyv format with a value in a column in the TableDB
-    pub fn store_rkyv<T>(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
+    pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
     where
         T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
     {
         let v = to_rkyv(value)?;
-        self.dbt.as_mut().unwrap().put(col, key, v.as_slice());
+        let mut inner = self.inner.lock();
+        inner.dbt.as_mut().unwrap().put(col, key, v.as_slice());
         Ok(())
     }
 
     /// Store a key in rkyv format with a value in a column in the TableDB
-    pub fn store_json<T>(&mut self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
+    pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
     where
         T: serde::Serialize,
     {
         let v = serde_json::to_vec(value)?;
-        self.dbt.as_mut().unwrap().put(col, key, v.as_slice());
+        let mut inner = self.inner.lock();
+        inner.dbt.as_mut().unwrap().put(col, key, v.as_slice());
         Ok(())
     }
 
     /// Delete key with from a column in the TableDB
-    pub fn delete(&mut self, col: u32, key: &[u8]) {
+    pub fn delete(&self, col: u32, key: &[u8]) {
-        self.dbt.as_mut().unwrap().delete(col, key);
+        let mut inner = self.inner.lock();
+        inner.dbt.as_mut().unwrap().delete(col, key);
     }
 }
 
-impl<'a> Drop for TableDBTransaction<'a> {
+impl Drop for TableDBTransactionInner {
     fn drop(&mut self) {
         if self.dbt.is_some() {
             warn!("Dropped transaction without commit or rollback");

veilid-core/src/intf/wasm/block_store.rs

@@ -1,5 +1,3 @@
-
-use crate::xx::*;
 use crate::*;
 
 struct BlockStoreInner {

veilid-core/src/intf/wasm/mod.rs

@@ -3,10 +3,7 @@ mod protected_store;
 mod system;
 mod table_store;
 
-pub mod utils;
-
 pub use block_store::*;
 pub use protected_store::*;
 pub use system::*;
 pub use table_store::*;
-use utils::*;

veilid-core/src/intf/wasm/protected_store.rs

@@ -1,11 +1,7 @@
-use super::*;
-use crate::xx::*;
 use crate::*;
 use data_encoding::BASE64URL_NOPAD;
-use js_sys::*;
+use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
-use send_wrapper::*;
+
-use serde::{Deserialize, Serialize};
-use wasm_bindgen_futures::*;
 use web_sys::*;
 
 #[derive(Clone)]
@@ -44,15 +40,6 @@ impl ProtectedStore {
     #[instrument(level = "debug", skip(self))]
     pub async fn terminate(&self) {}
 
-    fn keyring_name(&self) -> String {
-        let c = self.config.get();
-        if c.namespace.is_empty() {
-            "veilid_protected_store".to_owned()
-        } else {
-            format!("veilid_protected_store_{}", c.namespace)
-        }
-    }
-
     fn browser_key_name(&self, key: &str) -> String {
         let c = self.config.get();
         if c.namespace.is_empty() {
@@ -136,22 +123,31 @@ impl ProtectedStore {
     }
 
     #[instrument(level = "trace", skip(self, value))]
-    pub async fn save_user_secret_frozen<T>(&self, key: &str, value: &T) -> EyreResult<bool>
+    pub async fn save_user_secret_rkyv<T>(&self, key: &str, value: &T) -> EyreResult<bool>
     where
         T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
     {
-        let v = to_frozen(value)?;
+        let v = to_rkyv(value)?;
+        self.save_user_secret(&key, &v).await
+    }
+
+    #[instrument(level = "trace", skip(self, value))]
+    pub async fn save_user_secret_json<T>(&self, key: &str, value: &T) -> EyreResult<bool>
+    where
+        T: serde::Serialize,
+    {
+        let v = serde_json::to_vec(value)?;
         self.save_user_secret(&key, &v).await
     }
 
     #[instrument(level = "trace", skip(self))]
-    pub async fn load_user_secret_frozen<T>(&self, key: &str) -> EyreResult<Option<T>>
+    pub async fn load_user_secret_rkyv<T>(&self, key: &str) -> EyreResult<Option<T>>
     where
         T: RkyvArchive,
         <T as RkyvArchive>::Archived:
             for<'t> bytecheck::CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
         <T as RkyvArchive>::Archived:
-            rkyv::Deserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
+            RkyvDeserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
     {
         let out = self.load_user_secret(key).await?;
         let b = match out {
@@ -161,7 +157,24 @@ impl ProtectedStore {
             }
         };
 
-        let obj = from_frozen(&b)?;
+        let obj = from_rkyv(b)?;
+        Ok(Some(obj))
+    }
+
+    #[instrument(level = "trace", skip(self))]
+    pub async fn load_user_secret_json<T>(&self, key: &str) -> EyreResult<Option<T>>
+    where
+        T: for<'de> serde::de::Deserialize<'de>,
+    {
+        let out = self.load_user_secret(key).await?;
+        let b = match out {
+            Some(v) => v,
+            None => {
+                return Ok(None);
+            }
+        };
+
+        let obj = serde_json::from_slice(&b)?;
         Ok(Some(obj))
     }
 

veilid-core/src/intf/wasm/system.rs

@@ -1,159 +1,6 @@
-use super::utils;
-use crate::xx::*;
 use crate::*;
-use async_executors::{Bindgen, LocalSpawnHandleExt, SpawnHandleExt, Timer};
-use futures_util::future::{select, Either};
-use js_sys::*;
-//use wasm_bindgen_futures::*;
-//use web_sys::*;
 
-#[wasm_bindgen]
+//use js_sys::*;
-extern "C" {
-    #[wasm_bindgen(catch, structural, js_namespace = global, js_name = setTimeout)]
-    fn nodejs_global_set_timeout_with_callback_and_timeout_and_arguments_0(
-        handler: &::js_sys::Function,
-        timeout: u32,
-    ) -> Result<JsValue, JsValue>;
-}
-
-pub fn get_timestamp() -> u64 {
-    if utils::is_browser() {
-        return (Date::now() * 1000.0f64) as u64;
-    } else {
-        panic!("WASM requires browser environment");
-    }
-}
-
-// pub fn get_timestamp_string() -> String {
-//     let date = Date::new_0();
-//     let hours = Date::get_utc_hours(&date);
-//     let minutes = Date::get_utc_minutes(&date);
-//     let seconds = Date::get_utc_seconds(&date);
-//     let milliseconds = Date::get_utc_milliseconds(&date);
-//     format!(
-//         "{:02}:{:02}:{:02}.{}",
-//         hours, minutes, seconds, milliseconds
-//     )
-// }
-
-pub fn random_bytes(dest: &mut [u8]) -> EyreResult<()> {
-    let len = dest.len();
-    let u32len = len / 4;
-    let remlen = len % 4;
-
-    for n in 0..u32len {
-        let r = (Math::random() * (u32::max_value() as f64)) as u32;
-
-        dest[n * 4 + 0] = (r & 0xFF) as u8;
-        dest[n * 4 + 1] = ((r >> 8) & 0xFF) as u8;
-        dest[n * 4 + 2] = ((r >> 16) & 0xFF) as u8;
-        dest[n * 4 + 3] = ((r >> 24) & 0xFF) as u8;
-    }
-    if remlen > 0 {
-        let r = (Math::random() * (u32::max_value() as f64)) as u32;
-        for n in 0..remlen {
-            dest[u32len * 4 + n] = ((r >> (n * 8)) & 0xFF) as u8;
-        }
-    }
-
-    Ok(())
-}
-
-pub fn get_random_u32() -> u32 {
-    (Math::random() * (u32::max_value() as f64)) as u32
-}
-
-pub fn get_random_u64() -> u64 {
-    let v1: u32 = get_random_u32();
-    let v2: u32 = get_random_u32();
-    ((v1 as u64) << 32) | ((v2 as u32) as u64)
-}
-
-pub async fn sleep(millis: u32) {
-    Bindgen.sleep(Duration::from_millis(millis.into())).await
-}
-
-pub fn system_boxed<'a, Out>(
-    future: impl Future<Output = Out> + Send + 'a,
-) -> SendPinBoxFutureLifetime<'a, Out> {
-    Box::pin(future)
-}
-
-pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out>
-where
-    Out: Send + 'static,
-{
-    MustJoinHandle::new(
-        Bindgen
-            .spawn_handle(future)
-            .expect("wasm-bindgen-futures spawn should never error out"),
-    )
-}
-
-pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out>
-where
-    Out: 'static,
-{
-    MustJoinHandle::new(
-        Bindgen
-            .spawn_handle_local(future)
-            .expect("wasm-bindgen-futures spawn_local should never error out"),
-    )
-}
-
-// pub fn spawn_with_local_set<Out>(
-//     future: impl Future<Output = Out> + Send + 'static,
-// ) -> MustJoinHandle<Out>
-// where
-//     Out: Send + 'static,
-// {
-//     spawn(future)
-// }
-
-pub fn spawn_detached<Out>(future: impl Future<Output = Out> + Send + 'static)
-where
-    Out: Send + 'static,
-{
-    Bindgen
-        .spawn_handle_local(future)
-        .expect("wasm-bindgen-futures spawn_local should never error out")
-        .detach()
-}
-
-pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SendPinBoxFuture<()>
-where
-    F: Fn() -> FUT + Send + Sync + 'static,
-    FUT: Future<Output = ()> + Send,
-{
-    let e = Eventual::new();
-
-    let ie = e.clone();
-    let jh = spawn(Box::pin(async move {
-        while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
-            callback().await;
-        }
-    }));
-
-    Box::pin(async move {
-        e.resolve().await;
-        jh.await;
-    })
-}
-
-pub async fn timeout<F, T>(dur_ms: u32, f: F) -> Result<T, TimeoutError>
-where
-    F: Future<Output = T>,
-{
-    match select(Box::pin(intf::sleep(dur_ms)), Box::pin(f)).await {
-        Either::Left((_x, _b)) => Err(TimeoutError()),
-        Either::Right((y, _a)) => Ok(y),
-    }
-}
-
-// xxx: for now until wasm threads are more stable, and/or we bother with web workers
-pub fn get_concurrency() -> u32 {
-    1
-}
 
 pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
     // unimplemented!
@@ -161,7 +8,7 @@ pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
 }
 
 // pub async fn get_pwa_web_server_config() -> {
-//     if utils::is_browser() {
+//     if is_browser() {
 
 //         let win = window().unwrap();
 //         let doc = win.document().unwrap();

veilid-core/src/intf/wasm/table_store.rs

@@ -1,7 +1,5 @@
-use super::*;
+use crate::intf::table_db::TableDBInner;
-
+pub use crate::intf::table_db::{TableDB, TableDBTransaction};
-use crate::intf::table_db::*;
-use crate::xx::*;
 use crate::*;
 use keyvaluedb_web::*;
 
@@ -104,9 +102,11 @@ impl TableStore {
         let db = Database::open(table_name.clone(), column_count)
             .await
             .wrap_err("failed to open tabledb")?;
-        info!(
+        trace!(
             "opened table store '{}' with table name '{:?}' with {} columns",
-            name, table_name, column_count
+            name,
+            table_name,
+            column_count
         );
 
         let table_db = TableDB::new(table_name.clone(), self.clone(), db);
@@ -136,7 +136,7 @@ impl TableStore {
             }
         }
 
-        if utils::is_browser() {
+        if is_browser() {
             let out = match Database::delete(table_name.clone()).await {
                 Ok(_) => true,
                 Err(_) => false,

veilid-core/src/lib.rs

@@ -1,5 +1,6 @@
 #![deny(clippy::all)]
 #![deny(unused_must_use)]
+#![recursion_limit = "256"]
 
 cfg_if::cfg_if! {
     if #[cfg(target_arch = "wasm32")] {
@@ -20,7 +21,6 @@ extern crate alloc;
 
 mod api_tracing_layer;
 mod attachment_manager;
-mod callback_state_machine;
 mod core_context;
 mod crypto;
 mod intf;
@@ -32,17 +32,13 @@ mod veilid_api;
 #[macro_use]
 mod veilid_config;
 mod veilid_layer_filter;
-mod veilid_rng;
-
-#[macro_use]
-pub mod xx;
 
 pub use self::api_tracing_layer::ApiTracingLayer;
-pub use self::attachment_manager::AttachmentState;
 pub use self::core_context::{api_startup, api_startup_json, UpdateCallback};
 pub use self::veilid_api::*;
 pub use self::veilid_config::*;
 pub use self::veilid_layer_filter::*;
+pub use veilid_tools as tools;
 
 pub mod veilid_capnp {
     include!(concat!(env!("OUT_DIR"), "/proto/veilid_capnp.rs"));
@@ -62,7 +58,7 @@ pub fn veilid_version() -> (u32, u32, u32) {
 }
 
 #[cfg(target_os = "android")]
-pub use intf::utils::android::{veilid_core_setup_android, veilid_core_setup_android_no_log};
+pub use intf::android::veilid_core_setup_android;
 
 pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [
     "mio",
@@ -87,3 +83,5 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [
     "trust_dns_proto",
     "attohttpc",
 ];
+
+use veilid_tools::*;

veilid-core/src/network_manager/connection_handle.rs

@@ -2,7 +2,7 @@ use super::*;
 
 #[derive(Clone, Debug)]
 pub struct ConnectionHandle {
-    id: u64,
+    id: NetworkConnectionId,
     descriptor: ConnectionDescriptor,
     channel: flume::Sender<(Option<Id>, Vec<u8>)>,
 }
@@ -15,7 +15,7 @@ pub enum ConnectionHandleSendResult {
 
 impl ConnectionHandle {
     pub(super) fn new(
-        id: u64,
+        id: NetworkConnectionId,
         descriptor: ConnectionDescriptor,
         channel: flume::Sender<(Option<Id>, Vec<u8>)>,
     ) -> Self {
@@ -26,7 +26,7 @@ impl ConnectionHandle {
         }
     }
 
-    pub fn connection_id(&self) -> u64 {
+    pub fn connection_id(&self) -> NetworkConnectionId {
         self.id
     }
 

veilid-core/src/network_manager/connection_limits.rs

@@ -21,8 +21,8 @@ pub struct ConnectionLimits {
     max_connection_frequency_per_min: usize,
     conn_count_by_ip4: BTreeMap<Ipv4Addr, usize>,
     conn_count_by_ip6_prefix: BTreeMap<Ipv6Addr, usize>,
-    conn_timestamps_by_ip4: BTreeMap<Ipv4Addr, Vec<u64>>,
+    conn_timestamps_by_ip4: BTreeMap<Ipv4Addr, Vec<Timestamp>>,
-    conn_timestamps_by_ip6_prefix: BTreeMap<Ipv6Addr, Vec<u64>>,
+    conn_timestamps_by_ip6_prefix: BTreeMap<Ipv6Addr, Vec<Timestamp>>,
 }
 
 impl ConnectionLimits {
@@ -41,14 +41,14 @@ impl ConnectionLimits {
         }
     }
 
-    fn purge_old_timestamps(&mut self, cur_ts: u64) {
+    fn purge_old_timestamps(&mut self, cur_ts: Timestamp) {
         // v4
         {
             let mut dead_keys = Vec::<Ipv4Addr>::new();
             for (key, value) in &mut self.conn_timestamps_by_ip4 {
                 value.retain(|v| {
                     // keep timestamps that are less than a minute away
-                    cur_ts.saturating_sub(*v) < 60_000_000u64
+                    cur_ts.saturating_sub(*v) < TimestampDuration::new(60_000_000u64)
                 });
                 if value.is_empty() {
                     dead_keys.push(*key);
@@ -64,7 +64,7 @@ impl ConnectionLimits {
             for (key, value) in &mut self.conn_timestamps_by_ip6_prefix {
                 value.retain(|v| {
                     // keep timestamps that are less than a minute away
-                    cur_ts.saturating_sub(*v) < 60_000_000u64
+                    cur_ts.saturating_sub(*v) < TimestampDuration::new(60_000_000u64)
                 });
                 if value.is_empty() {
                     dead_keys.push(*key);
@@ -78,7 +78,7 @@ impl ConnectionLimits {
 
     pub fn add(&mut self, addr: IpAddr) -> Result<(), AddressFilterError> {
         let ipblock = ip_to_ipblock(self.max_connections_per_ip6_prefix_size, addr);
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
 
         self.purge_old_timestamps(ts);
 
@@ -95,7 +95,7 @@ impl ConnectionLimits {
                 let tstamps = &mut self.conn_timestamps_by_ip4.entry(v4).or_default();
                 tstamps.retain(|v| {
                     // keep timestamps that are less than a minute away
-                    ts.saturating_sub(*v) < 60_000_000u64
+                    ts.saturating_sub(*v) < TimestampDuration::new(60_000_000u64)
                 });
                 assert!(tstamps.len() <= self.max_connection_frequency_per_min);
                 if tstamps.len() == self.max_connection_frequency_per_min {
@@ -134,7 +134,7 @@ impl ConnectionLimits {
     pub fn remove(&mut self, addr: IpAddr) -> Result<(), AddressNotInTableError> {
         let ipblock = ip_to_ipblock(self.max_connections_per_ip6_prefix_size, addr);
 
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
         self.purge_old_timestamps(ts);
 
         match ipblock {

veilid-core/src/network_manager/connection_manager.rs

@@ -1,5 +1,4 @@
 use super::*;
-use crate::xx::*;
 use connection_table::*;
 use network_connection::*;
 use stop_token::future::FutureExt;
@@ -49,9 +48,9 @@ impl ConnectionManager {
         async_processor_jh: MustJoinHandle<()>,
     ) -> ConnectionManagerInner {
         ConnectionManagerInner {
-            next_id: 0,
+            next_id: 0.into(),
             stop_source: Some(stop_source),
-            sender: sender,
+            sender,
             async_processor_jh: Some(async_processor_jh),
         }
     }
@@ -150,7 +149,7 @@ impl ConnectionManager {
     ) -> EyreResult<NetworkResult<ConnectionHandle>> {
         // Get next connection id to use
         let id = inner.next_id;
-        inner.next_id += 1;
+        inner.next_id += 1u64;
         log_net!(
             "on_new_protocol_network_connection: id={} prot_conn={:?}",
             id,
@@ -320,7 +319,7 @@ impl ConnectionManager {
             };
             log_net!(debug "get_or_create_connection retries left: {}", retry_count);
             retry_count -= 1;
-            intf::sleep(500).await;
+            sleep(500).await;
         });
 
         // Add to the connection table
@@ -399,7 +398,7 @@ impl ConnectionManager {
     // Callback from network connection receive loop when it exits
     // cleans up the entry in the connection table
     #[instrument(level = "trace", skip(self))]
-    pub(super) async fn report_connection_finished(&self, connection_id: u64) {
+    pub(super) async fn report_connection_finished(&self, connection_id: NetworkConnectionId) {
         // Get channel sender
         let sender = {
             let mut inner = self.arc.inner.lock();

veilid-core/src/network_manager/mod.rs

@@ -23,7 +23,7 @@ pub use network_connection::*;
 use connection_handle::*;
 use connection_limits::*;
 use crypto::*;
-use futures_util::stream::{FuturesOrdered, FuturesUnordered, StreamExt};
+use futures_util::stream::FuturesUnordered;
 use hashlink::LruCache;
 use intf::*;
 #[cfg(not(target_arch = "wasm32"))]
@@ -33,30 +33,18 @@ use routing_table::*;
 use rpc_processor::*;
 #[cfg(target_arch = "wasm32")]
 use wasm::*;
-use xx::*;
 
 ////////////////////////////////////////////////////////////////////////////////////////
 
-pub const RELAY_MANAGEMENT_INTERVAL_SECS: u32 = 1;
-pub const PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS: u32 = 1;
 pub const MAX_MESSAGE_SIZE: usize = MAX_ENVELOPE_SIZE;
 pub const IPADDR_TABLE_SIZE: usize = 1024;
-pub const IPADDR_MAX_INACTIVE_DURATION_US: u64 = 300_000_000u64; // 5 minutes
+pub const IPADDR_MAX_INACTIVE_DURATION_US: TimestampDuration = TimestampDuration::new(300_000_000u64); // 5 minutes
 pub const PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT: usize = 3;
 pub const PUBLIC_ADDRESS_CHECK_CACHE_SIZE: usize = 8;
 pub const PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS: u32 = 60;
-pub const PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US: u64 = 300_000_000u64; // 5 minutes
+pub const PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US: TimestampDuration = TimestampDuration::new(300_000_000u64); // 5 minutes
-pub const PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US: u64 = 3600_000_000u64; // 60 minutes
+pub const PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US: TimestampDuration = TimestampDuration::new(3600_000_000u64); // 60 minutes
 pub const BOOT_MAGIC: &[u8; 4] = b"BOOT";
-pub const BOOTSTRAP_TXT_VERSION: u8 = 0;
-
-#[derive(Clone, Debug)]
-pub struct BootstrapRecord {
-    min_version: u8,
-    max_version: u8,
-    dial_info_details: Vec<DialInfoDetail>,
-}
-pub type BootstrapRecordMap = BTreeMap<DHTKey, BootstrapRecord>;
 
 #[derive(Copy, Clone, Debug, Default)]
 pub struct ProtocolConfig {
@@ -79,7 +67,7 @@ struct NetworkComponents {
 // Statistics per address
 #[derive(Clone, Default)]
 pub struct PerAddressStats {
-    last_seen_ts: u64,
+    last_seen_ts: Timestamp,
     transfer_stats_accounting: TransferStatsAccounting,
     transfer_stats: TransferStatsDownUp,
 }
@@ -111,7 +99,7 @@ impl Default for NetworkManagerStats {
 
 #[derive(Debug)]
 struct ClientWhitelistEntry {
-    last_seen_ts: u64,
+    last_seen_ts: Timestamp,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -150,7 +138,7 @@ struct NetworkManagerInner {
     public_address_check_cache:
         BTreeMap<PublicAddressCheckCacheKey, LruCache<IpAddr, SocketAddress>>,
     public_address_inconsistencies_table:
-        BTreeMap<PublicAddressCheckCacheKey, HashMap<IpAddr, u64>>,
+        BTreeMap<PublicAddressCheckCacheKey, HashMap<IpAddr, Timestamp>>,
 }
 
 struct NetworkManagerUnlockedInner {
@@ -166,13 +154,7 @@ struct NetworkManagerUnlockedInner {
     update_callback: RwLock<Option<UpdateCallback>>,
     // Background processes
     rolling_transfers_task: TickTask<EyreReport>,
-    relay_management_task: TickTask<EyreReport>,
-    private_route_management_task: TickTask<EyreReport>,
-    bootstrap_task: TickTask<EyreReport>,
-    peer_minimum_refresh_task: TickTask<EyreReport>,
-    ping_validator_task: TickTask<EyreReport>,
     public_address_check_task: TickTask<EyreReport>,
-    node_info_update_single_future: MustJoinSingleFuture<()>,
 }
 
 #[derive(Clone)]
@@ -197,7 +179,6 @@ impl NetworkManager {
         block_store: BlockStore,
         crypto: Crypto,
     ) -> NetworkManagerUnlockedInner {
-        let min_peer_refresh_time_ms = config.get().network.dht.min_peer_refresh_time_ms;
         NetworkManagerUnlockedInner {
             config,
             protected_store,
@@ -208,13 +189,7 @@ impl NetworkManager {
             components: RwLock::new(None),
             update_callback: RwLock::new(None),
             rolling_transfers_task: TickTask::new(ROLLING_TRANSFERS_INTERVAL_SECS),
-            relay_management_task: TickTask::new(RELAY_MANAGEMENT_INTERVAL_SECS),
-            private_route_management_task: TickTask::new(PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS),
-            bootstrap_task: TickTask::new(1),
-            peer_minimum_refresh_task: TickTask::new_ms(min_peer_refresh_time_ms),
-            ping_validator_task: TickTask::new(1),
             public_address_check_task: TickTask::new(PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS),
-            node_info_update_single_future: MustJoinSingleFuture::new(),
         }
     }
 
@@ -235,116 +210,9 @@ impl NetworkManager {
                 crypto,
             )),
         };
-        // Set rolling transfers tick task
+
-        {
+        this.start_tasks();
-            let this2 = this.clone();
+
-            this.unlocked_inner
-                .rolling_transfers_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .rolling_transfers_task_routine(s, l, t)
-                            .instrument(trace_span!(
-                                parent: None,
-                                "NetworkManager rolling transfers task routine"
-                            )),
-                    )
-                });
-        }
-        // Set relay management tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .relay_management_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .relay_management_task_routine(s, l, t)
-                            .instrument(trace_span!(parent: None, "relay management task routine")),
-                    )
-                });
-        }
-        // Set private route management tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .private_route_management_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .private_route_management_task_routine(s, l, t)
-                            .instrument(trace_span!(
-                                parent: None,
-                                "private route management task routine"
-                            )),
-                    )
-                });
-        }
-        // Set bootstrap tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .bootstrap_task
-                .set_routine(move |s, _l, _t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .bootstrap_task_routine(s)
-                            .instrument(trace_span!(parent: None, "bootstrap task routine")),
-                    )
-                });
-        }
-        // Set peer minimum refresh tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .peer_minimum_refresh_task
-                .set_routine(move |s, _l, _t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .peer_minimum_refresh_task_routine(s)
-                            .instrument(trace_span!(
-                                parent: None,
-                                "peer minimum refresh task routine"
-                            )),
-                    )
-                });
-        }
-        // Set ping validator tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .ping_validator_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .ping_validator_task_routine(s, l, t)
-                            .instrument(trace_span!(parent: None, "ping validator task routine")),
-                    )
-                });
-        }
-        // Set public address check task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .public_address_check_task
-                .set_routine(move |s, l, t| {
-                    Box::pin(
-                        this2
-                            .clone()
-                            .public_address_check_task_routine(s, l, t)
-                            .instrument(trace_span!(
-                                parent: None,
-                                "public address check task routine"
-                            )),
-                    )
-                });
-        }
         this
     }
     pub fn config(&self) -> VeilidConfig {
@@ -412,6 +280,14 @@ impl NetworkManager {
             .connection_manager
             .clone()
     }
+    pub fn update_callback(&self) -> UpdateCallback {
+        self.unlocked_inner
+            .update_callback
+            .read()
+            .as_ref()
+            .unwrap()
+            .clone()
+    }
 
     #[instrument(level = "debug", skip_all, err)]
     pub async fn init(&self, update_callback: UpdateCallback) -> EyreResult<()> {
@@ -492,36 +368,7 @@ impl NetworkManager {
         debug!("starting network manager shutdown");
 
         // Cancel all tasks
-        debug!("stopping rolling transfers task");
+        self.stop_tasks().await;
-        if let Err(e) = self.unlocked_inner.rolling_transfers_task.stop().await {
-            warn!("rolling_transfers_task not stopped: {}", e);
-        }
-        debug!("stopping relay management task");
-        if let Err(e) = self.unlocked_inner.relay_management_task.stop().await {
-            warn!("relay_management_task not stopped: {}", e);
-        }
-        debug!("stopping bootstrap task");
-        if let Err(e) = self.unlocked_inner.bootstrap_task.stop().await {
-            error!("bootstrap_task not stopped: {}", e);
-        }
-        debug!("stopping peer minimum refresh task");
-        if let Err(e) = self.unlocked_inner.peer_minimum_refresh_task.stop().await {
-            error!("peer_minimum_refresh_task not stopped: {}", e);
-        }
-        debug!("stopping ping_validator task");
-        if let Err(e) = self.unlocked_inner.ping_validator_task.stop().await {
-            error!("ping_validator_task not stopped: {}", e);
-        }
-        debug!("stopping node info update singlefuture");
-        if self
-            .unlocked_inner
-            .node_info_update_single_future
-            .join()
-            .await
-            .is_err()
-        {
-            error!("node_info_update_single_future not stopped");
-        }
 
         // Shutdown network components if they started up
         debug!("shutting down network components");
@@ -553,11 +400,11 @@ impl NetworkManager {
         let mut inner = self.inner.lock();
         match inner.client_whitelist.entry(client) {
             hashlink::lru_cache::Entry::Occupied(mut entry) => {
-                entry.get_mut().last_seen_ts = intf::get_timestamp()
+                entry.get_mut().last_seen_ts = get_aligned_timestamp()
             }
             hashlink::lru_cache::Entry::Vacant(entry) => {
                 entry.insert(ClientWhitelistEntry {
-                    last_seen_ts: intf::get_timestamp(),
+                    last_seen_ts: get_aligned_timestamp(),
                 });
             }
         }
@@ -569,7 +416,7 @@ impl NetworkManager {
 
         match inner.client_whitelist.entry(client) {
             hashlink::lru_cache::Entry::Occupied(mut entry) => {
-                entry.get_mut().last_seen_ts = intf::get_timestamp();
+                entry.get_mut().last_seen_ts = get_aligned_timestamp();
                 true
             }
             hashlink::lru_cache::Entry::Vacant(_) => false,
@@ -579,7 +426,7 @@ impl NetworkManager {
     pub fn purge_client_whitelist(&self) {
         let timeout_ms = self.with_config(|c| c.network.client_whitelist_timeout_ms);
         let mut inner = self.inner.lock();
-        let cutoff_timestamp = intf::get_timestamp() - ((timeout_ms as u64) * 1000u64);
+        let cutoff_timestamp = get_aligned_timestamp() - TimestampDuration::new((timeout_ms as u64) * 1000u64);
         // Remove clients from the whitelist that haven't been since since our whitelist timeout
         while inner
             .client_whitelist
@@ -597,58 +444,19 @@ impl NetworkManager {
         net.needs_restart()
     }
 
-    pub async fn tick(&self) -> EyreResult<()> {
-        let routing_table = self.routing_table();
-        let net = self.net();
-        let receipt_manager = self.receipt_manager();
-
-        // Run the rolling transfers task
-        self.unlocked_inner.rolling_transfers_task.tick().await?;
-
-        // Run the relay management task
-        self.unlocked_inner.relay_management_task.tick().await?;
-
-        // See how many live PublicInternet entries we have
-        let live_public_internet_entry_count = routing_table.get_entry_count(
-            RoutingDomain::PublicInternet.into(),
-            BucketEntryState::Unreliable,
-        );
-        let min_peer_count = self.with_config(|c| c.network.dht.min_peer_count as usize);
-
-        // If none, then add the bootstrap nodes to it
-        if live_public_internet_entry_count == 0 {
-            self.unlocked_inner.bootstrap_task.tick().await?;
-        }
-        // If we still don't have enough peers, find nodes until we do
-        else if !self.unlocked_inner.bootstrap_task.is_running()
-            && live_public_internet_entry_count < min_peer_count
-        {
-            self.unlocked_inner.peer_minimum_refresh_task.tick().await?;
-        }
-
-        // Ping validate some nodes to groom the table
-        self.unlocked_inner.ping_validator_task.tick().await?;
-
-        // Run the routing table tick
-        routing_table.tick().await?;
-
-        // Run the low level network tick
-        net.tick().await?;
-
-        // Run the receipt manager tick
-        receipt_manager.tick().await?;
-
-        // Purge the client whitelist
-        self.purge_client_whitelist();
-
-        Ok(())
-    }
-
     /// Get our node's capabilities in the PublicInternet routing domain
     fn generate_public_internet_node_status(&self) -> PublicInternetNodeStatus {
-        let own_peer_info = self
+        let Some(own_peer_info) = self
             .routing_table()
-            .get_own_peer_info(RoutingDomain::PublicInternet);
+            .get_own_peer_info(RoutingDomain::PublicInternet) else {
+                return PublicInternetNodeStatus {
+                    will_route: false,
+                    will_tunnel: false,
+                    will_signal: false,
+                    will_relay: false,
+                    will_validate_dial_info: false,
+                };  
+            };
         let own_node_info = own_peer_info.signed_node_info.node_info();
 
         let will_route = own_node_info.can_inbound_relay(); // xxx: eventually this may have more criteria added
@@ -667,9 +475,14 @@ impl NetworkManager {
     }
     /// Get our node's capabilities in the LocalNetwork routing domain
     fn generate_local_network_node_status(&self) -> LocalNetworkNodeStatus {
-        let own_peer_info = self
+        let Some(own_peer_info) = self
             .routing_table()
-            .get_own_peer_info(RoutingDomain::LocalNetwork);
+            .get_own_peer_info(RoutingDomain::LocalNetwork) else {
+                return LocalNetworkNodeStatus {
+                    will_relay: false,
+                    will_validate_dial_info: false,
+                };  
+            };
 
         let own_node_info = own_peer_info.signed_node_info.node_info();
 
@@ -713,7 +526,7 @@ impl NetworkManager {
             .wrap_err("failed to generate signed receipt")?;
 
         // Record the receipt for later
-        let exp_ts = intf::get_timestamp() + expiration_us;
+        let exp_ts = get_aligned_timestamp() + expiration_us;
         receipt_manager.record_receipt(receipt, exp_ts, expected_returns, callback);
 
         Ok(out)
@@ -737,7 +550,7 @@ impl NetworkManager {
             .wrap_err("failed to generate signed receipt")?;
 
         // Record the receipt for later
-        let exp_ts = intf::get_timestamp() + expiration_us;
+        let exp_ts = get_aligned_timestamp() + expiration_us;
         let eventual = SingleShotEventual::new(Some(ReceiptEvent::Cancelled));
         let instance = eventual.instance();
         receipt_manager.record_single_shot_receipt(receipt, exp_ts, eventual);
@@ -904,7 +717,7 @@ impl NetworkManager {
                 // XXX: do we need a delay here? or another hole punch packet?
 
                 // Set the hole punch as our 'last connection' to ensure we return the receipt over the direct hole punch
-                peer_nr.set_last_connection(connection_descriptor, intf::get_timestamp());
+                peer_nr.set_last_connection(connection_descriptor, get_aligned_timestamp());
 
                 // Return the receipt using the same dial info send the receipt to it
                 rpc.rpc_call_return_receipt(Destination::direct(peer_nr), receipt)
@@ -928,7 +741,7 @@ impl NetworkManager {
         let node_id_secret = routing_table.node_id_secret();
 
         // Get timestamp, nonce
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
         let nonce = Crypto::get_random_nonce();
 
         // Encode envelope
@@ -1001,7 +814,7 @@ impl NetworkManager {
 
         // Send receipt directly
         log_net!(debug "send_out_of_band_receipt: dial_info={}", dial_info);
-        network_result_value_or_log!(debug self
+        network_result_value_or_log!(self
             .net()
             .send_data_unbound_to_dial_info(dial_info, rcpt_data)
             .await? => {
@@ -1031,10 +844,17 @@ impl NetworkManager {
         );
         let (receipt, eventual_value) = self.generate_single_shot_receipt(receipt_timeout, [])?;
 
+        // Get target routing domain
+        let Some(routing_domain) = target_nr.best_routing_domain() else {
+            return Ok(NetworkResult::no_connection_other("No routing domain for target"));
+        };
+
         // Get our peer info
-        let peer_info = self
+        let Some(peer_info) = self
             .routing_table()
-            .get_own_peer_info(RoutingDomain::PublicInternet);
+            .get_own_peer_info(routing_domain) else {
+            return Ok(NetworkResult::no_connection_other("Own peer info not available"));
+        };    
 
         // Issue the signal
         let rpc = self.rpc_processor();
@@ -1098,17 +918,11 @@ impl NetworkManager {
         data: Vec<u8>,
     ) -> EyreResult<NetworkResult<ConnectionDescriptor>> {
         // Ensure we are filtered down to UDP (the only hole punch protocol supported today)
-        // and only in the PublicInternet routing domain
         assert!(target_nr
             .filter_ref()
             .map(|nrf| nrf.dial_info_filter.protocol_type_set
                 == ProtocolTypeSet::only(ProtocolType::UDP))
             .unwrap_or_default());
-        assert!(target_nr
-            .filter_ref()
-            .map(|nrf| nrf.routing_domain_set
-                == RoutingDomainSet::only(RoutingDomain::PublicInternet))
-            .unwrap_or_default());
 
         // Build a return receipt for the signal
         let receipt_timeout = ms_to_us(
@@ -1119,10 +933,18 @@ impl NetworkManager {
                 .hole_punch_receipt_time_ms,
         );
         let (receipt, eventual_value) = self.generate_single_shot_receipt(receipt_timeout, [])?;
+
+        // Get target routing domain
+        let Some(routing_domain) = target_nr.best_routing_domain() else {
+            return Ok(NetworkResult::no_connection_other("No routing domain for target"));
+        };
+
         // Get our peer info
-        let peer_info = self
+        let Some(peer_info) = self
             .routing_table()
-            .get_own_peer_info(RoutingDomain::PublicInternet);
+            .get_own_peer_info(routing_domain) else {
+                return Ok(NetworkResult::no_connection_other("Own peer info not available"));
+            };
 
         // Get the udp direct dialinfo for the hole punch
         let hole_punch_did = target_nr
@@ -1214,7 +1036,8 @@ impl NetworkManager {
         };
 
         // Node A is our own node
-        let peer_a = routing_table.get_own_peer_info(routing_domain);
+        // Use whatever node info we've calculated so far
+        let peer_a = routing_table.get_best_effort_own_peer_info(routing_domain);
 
         // Node B is the target node
         let peer_b = match target_node_ref.make_peer_info(routing_domain) {
@@ -1313,8 +1136,7 @@ impl NetworkManager {
                             // );
 
                             // Update timestamp for this last connection since we just sent to it
-                            node_ref
+                            node_ref.set_last_connection(connection_descriptor, get_aligned_timestamp());
-                                .set_last_connection(connection_descriptor, intf::get_timestamp());
 
                             return Ok(NetworkResult::value(SendDataKind::Existing(
                                 connection_descriptor,
@@ -1346,7 +1168,7 @@ impl NetworkManager {
                             this.net().send_data_to_dial_info(dial_info, data).await?
                         );
                         // If we connected to this node directly, save off the last connection so we can use it again
-                        node_ref.set_last_connection(connection_descriptor, intf::get_timestamp());
+                        node_ref.set_last_connection(connection_descriptor, get_aligned_timestamp());
 
                         Ok(NetworkResult::value(SendDataKind::Direct(
                             connection_descriptor,
@@ -1421,7 +1243,7 @@ impl NetworkManager {
         let timeout_ms = self.with_config(|c| c.network.rpc.timeout_ms);
         // Send boot magic to requested peer address
         let data = BOOT_MAGIC.to_vec();
-        let out_data: Vec<u8> = network_result_value_or_log!(debug self
+        let out_data: Vec<u8> = network_result_value_or_log!(self
             .net()
             .send_recv_data_unbound_to_dial_info(dial_info, data, timeout_ms)
             .await? =>
@@ -1463,7 +1285,7 @@ impl NetworkManager {
         // Network accounting
         self.stats_packet_rcvd(
             connection_descriptor.remote_address().to_ip_addr(),
-            data.len() as u64,
+            ByteCount::new(data.len() as u64),
         );
 
         // If this is a zero length packet, just drop it, because these are used for hole punching
@@ -1493,13 +1315,13 @@ impl NetworkManager {
 
         // Is this a direct bootstrap request instead of an envelope?
         if data[0..4] == *BOOT_MAGIC {
-            network_result_value_or_log!(debug self.handle_boot_request(connection_descriptor).await? => {});
+            network_result_value_or_log!(self.handle_boot_request(connection_descriptor).await? => {});
             return Ok(true);
         }
 
         // Is this an out-of-band receipt instead of an envelope?
         if data[0..4] == *RECEIPT_MAGIC {
-            network_result_value_or_log!(debug self.handle_out_of_band_receipt(data).await => {});
+            network_result_value_or_log!(self.handle_out_of_band_receipt(data).await => {});
             return Ok(true);
         }
 
@@ -1515,28 +1337,28 @@ impl NetworkManager {
         // Get timestamp range
         let (tsbehind, tsahead) = self.with_config(|c| {
             (
-                c.network.rpc.max_timestamp_behind_ms.map(ms_to_us),
+                c.network.rpc.max_timestamp_behind_ms.map(ms_to_us).map(TimestampDuration::new),
-                c.network.rpc.max_timestamp_ahead_ms.map(ms_to_us),
+                c.network.rpc.max_timestamp_ahead_ms.map(ms_to_us).map(TimestampDuration::new),
             )
         });
 
         // Validate timestamp isn't too old
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
         let ets = envelope.get_timestamp();
         if let Some(tsbehind) = tsbehind {
-            if tsbehind > 0 && (ts > ets && ts - ets > tsbehind) {
+            if tsbehind.as_u64() != 0 && (ts > ets && ts.saturating_sub(ets) > tsbehind) {
                 log_net!(debug
                     "envelope time was too far in the past: {}ms ",
-                    timestamp_to_secs(ts - ets) * 1000f64
+                    timestamp_to_secs(ts.saturating_sub(ets).as_u64()) * 1000f64
                 );
                 return Ok(false);
             }
         }
         if let Some(tsahead) = tsahead {
-            if tsahead > 0 && (ts < ets && ets - ts > tsahead) {
+            if tsahead.as_u64() != 0 && (ts < ets && ets.saturating_sub(ts) > tsahead) {
                 log_net!(debug
                     "envelope time was too far in the future: {}ms",
-                    timestamp_to_secs(ets - ts) * 1000f64
+                    timestamp_to_secs(ets.saturating_sub(ts).as_u64()) * 1000f64
                 );
                 return Ok(false);
             }
@@ -1557,9 +1379,13 @@ impl NetworkManager {
 
             let some_relay_nr = if self.check_client_whitelist(sender_id) {
                 // Full relay allowed, do a full resolve_node
-                rpc.resolve_node(recipient_id).await.wrap_err(
+                match rpc.resolve_node(recipient_id).await {
-                    "failed to resolve recipient node for relay, dropping outbound relayed packet",
+                    Ok(v) => v,
-                )?
+                    Err(e) => {
+                        log_net!(debug "failed to resolve recipient node for relay, dropping outbound relayed packet: {}" ,e);
+                        return Ok(false);
+                    }
+                }
             } else {
                 // If this is not a node in the client whitelist, only allow inbound relay
                 // which only performs a lightweight lookup before passing the packet back out
@@ -1574,9 +1400,14 @@ impl NetworkManager {
             if let Some(relay_nr) = some_relay_nr {
                 // Relay the packet to the desired destination
                 log_net!("relaying {} bytes to {}", data.len(), relay_nr);
-                network_result_value_or_log!(debug self.send_data(relay_nr, data.to_vec())
+                network_result_value_or_log!(match self.send_data(relay_nr, data.to_vec())
-                    .await
+                    .await {
-                    .wrap_err("failed to forward envelope")? => {
+                        Ok(v) => v,
+                        Err(e) => {
+                            log_net!(debug "failed to forward envelope: {}" ,e);
+                            return Ok(false);    
+                        }
+                    } => {
                         return Ok(false);
                     }
                 );
@@ -1589,10 +1420,15 @@ impl NetworkManager {
         let node_id_secret = routing_table.node_id_secret();
 
         // Decrypt the envelope body
-        // xxx: punish nodes that send messages that fail to decrypt eventually
+        let body = match envelope
-        let body = envelope
+            .decrypt_body(self.crypto(), data, &node_id_secret) {
-            .decrypt_body(self.crypto(), data, &node_id_secret)
+                Ok(v) => v,
-            .wrap_err("failed to decrypt envelope body")?;
+                Err(e) => {
+                    log_net!(debug "failed to decrypt envelope body: {}",e);
+                    // xxx: punish nodes that send messages that fail to decrypt eventually
+                    return Ok(false);
+                }
+            };
 
         // Cache the envelope information in the routing table
         let source_noderef = match routing_table.register_node_with_existing_connection(
@@ -1625,7 +1461,7 @@ impl NetworkManager {
     }
 
     // Callbacks from low level network for statistics gathering
-    pub fn stats_packet_sent(&self, addr: IpAddr, bytes: u64) {
+    pub fn stats_packet_sent(&self, addr: IpAddr, bytes: ByteCount) {
         let inner = &mut *self.inner.lock();
         inner
             .stats
@@ -1641,7 +1477,7 @@ impl NetworkManager {
             .add_up(bytes);
     }
 
-    pub fn stats_packet_rcvd(&self, addr: IpAddr, bytes: u64) {
+    pub fn stats_packet_rcvd(&self, addr: IpAddr, bytes: ByteCount) {
         let inner = &mut *self.inner.lock();
         inner
             .stats
@@ -1675,9 +1511,10 @@ impl NetworkManager {
         if !has_state {
             return VeilidStateNetwork {
                 started: false,
-                bps_down: 0,
+                bps_down: 0.into(),
-                bps_up: 0,
+                bps_up: 0.into(),
                 peers: Vec::new(),
+                
             };
         }
         let routing_table = self.routing_table();
@@ -1828,7 +1665,7 @@ impl NetworkManager {
                 // public dialinfo
                 let inconsistent = if inconsistencies.len() >= PUBLIC_ADDRESS_CHANGE_DETECTION_COUNT
                 {
-                    let exp_ts = intf::get_timestamp() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US;
+                    let exp_ts = get_aligned_timestamp() + PUBLIC_ADDRESS_INCONSISTENCY_TIMEOUT_US;
                     for i in &inconsistencies {
                         pait.insert(*i, exp_ts);
                     }
@@ -1841,8 +1678,8 @@ impl NetworkManager {
                             .public_address_inconsistencies_table
                             .entry(key)
                             .or_insert_with(|| HashMap::new());
-                        let exp_ts = intf::get_timestamp()
+                        let exp_ts =
-                            + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US;
+                            get_aligned_timestamp() + PUBLIC_ADDRESS_INCONSISTENCY_PUNISHMENT_TIMEOUT_US;
                         for i in inconsistencies {
                             pait.insert(i, exp_ts);
                         }
@@ -1860,7 +1697,7 @@ impl NetworkManager {
                 // }
 
                 inconsistent
-            } else {
+            } else if matches!(public_internet_network_class, NetworkClass::OutboundOnly) {
                 // If we are currently outbound only, we don't have any public dial info
                 // but if we are starting to see consistent socket address from multiple reporting peers
                 // then we may be become inbound capable, so zap the network class so we can re-detect it and any public dial info
@@ -1888,6 +1725,10 @@ impl NetworkManager {
                     }
                 }
                 consistent
+            } else {
+                // If we are a webapp we never do this.
+                // If we have invalid network class, then public address detection is already going to happen via the network_class_discovery task
+                false
             };
 
         if needs_public_address_detection {
@@ -1910,62 +1751,4 @@ impl NetworkManager {
         }
     }
 
-    // Inform routing table entries that our dial info has changed
-    pub async fn send_node_info_updates(&self, routing_domain: RoutingDomain, all: bool) {
-        let this = self.clone();
-
-        // Run in background only once
-        let _ = self
-            .clone()
-            .unlocked_inner
-            .node_info_update_single_future
-            .single_spawn(
-                async move {
-                    // Only update if we actually have valid signed node info for this routing domain
-                    if !this.routing_table().has_valid_own_node_info(routing_domain) {
-                        trace!(
-                        "not sending node info update because our network class is not yet valid"
-                    );
-                        return;
-                    }
-
-                    // Get the list of refs to all nodes to update
-                    let cur_ts = intf::get_timestamp();
-                    let node_refs =
-                        this.routing_table()
-                            .get_nodes_needing_updates(routing_domain, cur_ts, all);
-
-                    // Send the updates
-                    log_net!(debug "Sending node info updates to {} nodes", node_refs.len());
-                    let mut unord = FuturesUnordered::new();
-                    for nr in node_refs {
-                        let rpc = this.rpc_processor();
-                        unord.push(
-                            async move {
-                                // Update the node
-                                if let Err(e) = rpc
-                                    .rpc_call_node_info_update(nr.clone(), routing_domain)
-                                    .await
-                                {
-                                    // Not fatal, but we should be able to see if this is happening
-                                    trace!("failed to send node info update to {:?}: {}", nr, e);
-                                    return;
-                                }
-
-                                // Mark the node as having seen our node info
-                                nr.set_seen_our_node_info(routing_domain);
-                            }
-                            .instrument(Span::current()),
-                        );
-                    }
-
-                    // Wait for futures to complete
-                    while unord.next().await.is_some() {}
-
-                    log_rtab!(debug "Finished sending node updates");
-                }
-                .instrument(Span::current()),
-            )
-            .await;
-    }
 }

veilid-core/src/network_manager/native/igd_manager.rs

@@ -1,12 +1,12 @@
 use super::*;
-use crate::xx::*;
 use igd::*;
 use std::net::UdpSocket;
 
+
 const UPNP_GATEWAY_DETECT_TIMEOUT_MS: u32 = 5_000;
 const UPNP_MAPPING_LIFETIME_MS: u32 = 120_000;
 const UPNP_MAPPING_ATTEMPTS: u32 = 3;
-const UPNP_MAPPING_LIFETIME_US:u64 = (UPNP_MAPPING_LIFETIME_MS as u64) * 1000u64;
+const UPNP_MAPPING_LIFETIME_US:TimestampDuration = TimestampDuration::new(UPNP_MAPPING_LIFETIME_MS as u64 * 1000u64);
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 struct PortMapKey {
@@ -19,8 +19,8 @@ struct PortMapKey {
 struct PortMapValue {
     ext_ip: IpAddr,
     mapped_port: u16,
-    timestamp: u64,
+    timestamp: Timestamp,
-    renewal_lifetime: u64,
+    renewal_lifetime: TimestampDuration,
     renewal_attempts: u32,
 }
 
@@ -177,7 +177,7 @@ impl IGDManager {
         mapped_port: u16,
     ) -> Option<()> {
         let this = self.clone();
-        intf::blocking_wrapper(move || {
+        blocking_wrapper(move || {
             let mut inner = this.inner.lock();
 
             // If we already have this port mapped, just return the existing portmap
@@ -216,7 +216,7 @@ impl IGDManager {
         expected_external_address: Option<IpAddr>,
     ) -> Option<SocketAddr> {
         let this = self.clone();
-        intf::blocking_wrapper(move || {
+        blocking_wrapper(move || {
             let mut inner = this.inner.lock();
 
             // If we already have this port mapped, just return the existing portmap
@@ -276,7 +276,7 @@ impl IGDManager {
             };
 
             // Add to mapping list to keep alive
-            let timestamp = intf::get_timestamp();
+            let timestamp = get_aligned_timestamp();
             inner.port_maps.insert(PortMapKey {
                 llpt,
                 at,
@@ -285,7 +285,7 @@ impl IGDManager {
                 ext_ip, 
                 mapped_port, 
                 timestamp, 
-                renewal_lifetime: (UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64, 
+                renewal_lifetime: ((UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64).into(), 
                 renewal_attempts: 0,
             });
 
@@ -302,7 +302,7 @@ impl IGDManager {
         let mut renews: Vec<(PortMapKey, PortMapValue)> = Vec::new();
         {
             let inner = self.inner.lock();
-            let now = intf::get_timestamp();
+            let now = get_aligned_timestamp();
 
             for (k, v) in &inner.port_maps {
                 let mapping_lifetime = now.saturating_sub(v.timestamp);
@@ -324,7 +324,7 @@ impl IGDManager {
         }
 
         let this = self.clone();
-        intf::blocking_wrapper(move || {
+        blocking_wrapper(move || {
             let mut inner = this.inner.lock();
 
             // Process full renewals
@@ -357,8 +357,8 @@ impl IGDManager {
                         inner.port_maps.insert(k, PortMapValue {
                             ext_ip: v.ext_ip,
                             mapped_port, 
-                            timestamp: intf::get_timestamp(), 
+                            timestamp: get_aligned_timestamp(), 
-                            renewal_lifetime: (UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64, 
+                            renewal_lifetime: TimestampDuration::new((UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64), 
                             renewal_attempts: 0,
                         });
                     },
@@ -398,8 +398,8 @@ impl IGDManager {
                         inner.port_maps.insert(k, PortMapValue {
                             ext_ip: v.ext_ip,
                             mapped_port: v.mapped_port, 
-                            timestamp: intf::get_timestamp(), 
+                            timestamp: get_aligned_timestamp(), 
-                            renewal_lifetime: (UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64, 
+                            renewal_lifetime: ((UPNP_MAPPING_LIFETIME_MS / 2) as u64 * 1000u64).into(), 
                             renewal_attempts: 0,
                         });
                     },
@@ -407,7 +407,7 @@ impl IGDManager {
                         log_net!(debug "failed to renew mapped port {:?} -> {:?}: {}", v, k, e);
                         
                         // Get closer to the maximum renewal timeline by a factor of two each time
-                        v.renewal_lifetime = (v.renewal_lifetime + UPNP_MAPPING_LIFETIME_US) / 2;
+                        v.renewal_lifetime = (v.renewal_lifetime + UPNP_MAPPING_LIFETIME_US) / 2u64;
                         v.renewal_attempts += 1;
                     
                         // Store new value to try again

veilid-core/src/network_manager/native/mod.rs

@@ -1,5 +1,4 @@
 mod igd_manager;
-mod natpmp_manager;
 mod network_class_discovery;
 mod network_tcp;
 mod network_udp;
@@ -9,12 +8,12 @@ mod start_protocols;
 use super::*;
 use crate::routing_table::*;
 use connection_manager::*;
+use network_interfaces::*;
 use network_tcp::*;
 use protocol::tcp::RawTcpProtocolHandler;
 use protocol::udp::RawUdpProtocolHandler;
 use protocol::ws::WebsocketProtocolHandler;
 pub use protocol::*;
-use utils::network_interfaces::*;
 
 use async_tls::TlsAcceptor;
 use futures_util::StreamExt;
@@ -94,11 +93,9 @@ struct NetworkUnlockedInner {
     update_network_class_task: TickTask<EyreReport>,
     network_interfaces_task: TickTask<EyreReport>,
     upnp_task: TickTask<EyreReport>,
-    natpmp_task: TickTask<EyreReport>,
 
     // Managers
     igd_manager: igd_manager::IGDManager,
-    natpmp_manager: natpmp_manager::NATPMPManager,
 }
 
 #[derive(Clone)]
@@ -150,9 +147,7 @@ impl Network {
             update_network_class_task: TickTask::new(1),
             network_interfaces_task: TickTask::new(5),
             upnp_task: TickTask::new(1),
-            natpmp_task: TickTask::new(1),
             igd_manager: igd_manager::IGDManager::new(config.clone()),
-            natpmp_manager: natpmp_manager::NATPMPManager::new(config),
         }
     }
 
@@ -196,13 +191,6 @@ impl Network {
                 .upnp_task
                 .set_routine(move |s, l, t| Box::pin(this2.clone().upnp_task_routine(s, l, t)));
         }
-        // Set natpmp tick task
-        {
-            let this2 = this.clone();
-            this.unlocked_inner
-                .natpmp_task
-                .set_routine(move |s, l, t| Box::pin(this2.clone().natpmp_task_routine(s, l, t)));
-        }
 
         this
     }
@@ -418,7 +406,7 @@ impl Network {
         }
         // Network accounting
         self.network_manager()
-            .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+            .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
         Ok(NetworkResult::Value(()))
     }
@@ -452,7 +440,7 @@ impl Network {
                     .await
                     .wrap_err("send message failure")?);
                 self.network_manager()
-                    .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+                    .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
                 // receive single response
                 let mut out = vec![0u8; MAX_MESSAGE_SIZE];
@@ -466,7 +454,7 @@ impl Network {
 
                 let recv_socket_addr = recv_addr.remote_address().to_socket_addr();
                 self.network_manager()
-                    .stats_packet_rcvd(recv_socket_addr.ip(), recv_len as u64);
+                    .stats_packet_rcvd(recv_socket_addr.ip(), ByteCount::new(recv_len as u64));
 
                 // if the from address is not the same as the one we sent to, then drop this
                 if recv_socket_addr != peer_socket_addr {
@@ -493,7 +481,7 @@ impl Network {
 
                 network_result_try!(pnc.send(data).await.wrap_err("send failure")?);
                 self.network_manager()
-                    .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+                    .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
                 let out = network_result_try!(network_result_try!(timeout(timeout_ms, pnc.recv())
                     .await
@@ -501,7 +489,7 @@ impl Network {
                 .wrap_err("recv failure")?);
 
                 self.network_manager()
-                    .stats_packet_rcvd(dial_info.to_ip_addr(), out.len() as u64);
+                    .stats_packet_rcvd(dial_info.to_ip_addr(), ByteCount::new(out.len() as u64));
 
                 Ok(NetworkResult::Value(out))
             }
@@ -524,14 +512,14 @@ impl Network {
                 &peer_socket_addr,
                 &descriptor.local().map(|sa| sa.to_socket_addr()),
             ) {
-                network_result_value_or_log!(debug ph.clone()
+                network_result_value_or_log!(ph.clone()
                     .send_message(data.clone(), peer_socket_addr)
                     .await
                     .wrap_err("sending data to existing conection")? => { return Ok(Some(data)); } );
 
                 // Network accounting
                 self.network_manager()
-                    .stats_packet_sent(peer_socket_addr.ip(), data_len as u64);
+                    .stats_packet_sent(peer_socket_addr.ip(), ByteCount::new(data_len as u64));
 
                 // Data was consumed
                 return Ok(None);
@@ -548,7 +536,7 @@ impl Network {
                     // Network accounting
                     self.network_manager().stats_packet_sent(
                         descriptor.remote().to_socket_addr().ip(),
-                        data_len as u64,
+                        ByteCount::new(data_len as u64),
                     );
 
                     // Data was consumed
@@ -607,7 +595,7 @@ impl Network {
 
         // Network accounting
         self.network_manager()
-            .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+            .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
         Ok(NetworkResult::value(connection_descriptor))
     }
@@ -722,8 +710,8 @@ impl Network {
                 }
 
                 ProtocolConfig {
-                    inbound,
                     outbound,
+                    inbound,
                     family_global,
                     family_local,
                 }
@@ -770,13 +758,13 @@ impl Network {
         // if we have static public dialinfo, upgrade our network class
 
         editor_public_internet.setup_network(
-            protocol_config.inbound,
             protocol_config.outbound,
+            protocol_config.inbound,
             protocol_config.family_global,
         );
         editor_local_network.setup_network(
-            protocol_config.inbound,
             protocol_config.outbound,
+            protocol_config.inbound,
             protocol_config.family_local,
         );
         let detect_address_changes = {
@@ -843,13 +831,13 @@ impl Network {
         debug!("clearing dial info");
 
         let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet);
-        editor.disable_node_info_updates();
         editor.clear_dial_info_details();
+        editor.set_network_class(None);
         editor.commit().await;
 
         let mut editor = routing_table.edit_routing_domain(RoutingDomain::LocalNetwork);
-        editor.disable_node_info_updates();
         editor.clear_dial_info_details();
+        editor.set_network_class(None);
         editor.commit().await;
 
         // Reset state including network class
@@ -904,31 +892,11 @@ impl Network {
         Ok(())
     }
 
-    #[instrument(level = "trace", skip(self), err)]
-    pub async fn natpmp_task_routine(
-        self,
-        stop_token: StopToken,
-        _l: u64,
-        _t: u64,
-    ) -> EyreResult<()> {
-        if !self.unlocked_inner.natpmp_manager.tick().await? {
-            info!("natpmp failed, restarting local network");
-            let mut inner = self.inner.lock();
-            inner.network_needs_restart = true;
-        }
-
-        Ok(())
-    }
-
     pub async fn tick(&self) -> EyreResult<()> {
-        let (detect_address_changes, upnp, natpmp) = {
+        let (detect_address_changes, upnp) = {
             let config = self.network_manager().config();
             let c = config.get();
-            (
+            (c.network.detect_address_changes, c.network.upnp)
-                c.network.detect_address_changes,
-                c.network.upnp,
-                c.network.natpmp,
-            )
         };
 
         // If we need to figure out our network class, tick the task for it
@@ -962,11 +930,6 @@ impl Network {
             self.unlocked_inner.upnp_task.tick().await?;
         }
 
-        // If we need to tick natpmp, do it
-        if natpmp && !self.needs_restart() {
-            self.unlocked_inner.natpmp_task.tick().await?;
-        }
-
         Ok(())
     }
 }

veilid-core/src/network_manager/native/natpmp_manager.rs

@@ -1,18 +0,0 @@
-use super::*;
-
-pub struct NATPMPManager {
-    config: VeilidConfig,
-}
-
-impl NATPMPManager {
-    //
-
-    pub fn new(config: VeilidConfig) -> Self {
-        Self { config }
-    }
-
-    pub async fn tick(&self) -> EyreResult<bool> {
-        // xxx
-        Ok(true)
-    }
-}

veilid-core/src/network_manager/native/network_class_discovery.rs

@@ -83,7 +83,7 @@ impl DiscoveryContext {
     async fn request_public_address(&self, node_ref: NodeRef) -> Option<SocketAddress> {
         let rpc = self.routing_table.rpc_processor();
 
-        let res = network_result_value_or_log!(debug match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await {
+        let res = network_result_value_or_log!(match rpc.rpc_call_status(Destination::direct(node_ref.clone())).await {
                 Ok(v) => v,
                 Err(e) => {
                     log_net!(error
@@ -275,7 +275,7 @@ impl DiscoveryContext {
                         LowLevelProtocolType::UDP => "udp",
                         LowLevelProtocolType::TCP => "tcp",
                     });
-                    intf::sleep(PORT_MAP_VALIDATE_DELAY_MS).await
+                    sleep(PORT_MAP_VALIDATE_DELAY_MS).await
                 } else {
                     break;
                 }
@@ -304,9 +304,9 @@ impl DiscoveryContext {
 
     #[instrument(level = "trace", skip(self), ret)]
     async fn try_port_mapping(&self) -> Option<DialInfo> {
-        let (enable_upnp, _enable_natpmp) = {
+        let enable_upnp = {
             let c = self.net.config.get();
-            (c.network.upnp, c.network.natpmp)
+            c.network.upnp
         };
 
         if enable_upnp {
@@ -434,15 +434,6 @@ impl DiscoveryContext {
             return Ok(true);
         }
 
-        // XXX: is this necessary?
-        // Redo our external_1 dial info detection because a failed port mapping attempt
-        // may cause it to become invalid
-        // Get our external address from some fast node, call it node 1
-        // if !self.protocol_get_external_address_1().await {
-        //     // If we couldn't get an external address, then we should just try the whole network class detection again later
-        //     return Ok(false);
-        // }
-
         // Get the external dial info for our use here
         let (node_1, external_1_dial_info, external_1_address, protocol_type, address_type) = {
             let inner = self.inner.lock();

veilid-core/src/network_manager/native/network_tcp.rs

@@ -58,7 +58,7 @@ impl Network {
         // Don't waste more than N seconds getting it though, in case someone
         // is trying to DoS us with a bunch of connections or something
         // read a chunk of the stream
-        intf::timeout(
+        timeout(
             tls_connection_initial_timeout_ms,
             ps.peek_exact(&mut first_packet),
         )

veilid-core/src/network_manager/native/network_udp.rs

@@ -10,7 +10,7 @@ impl Network {
             c.network.protocol.udp.socket_pool_size
         };
         if task_count == 0 {
-            task_count = intf::get_concurrency() / 2;
+            task_count = get_concurrency() / 2;
             if task_count == 0 {
                 task_count = 1;
             }
@@ -65,7 +65,7 @@ impl Network {
                                     // Network accounting
                                     network_manager.stats_packet_rcvd(
                                         descriptor.remote_address().to_ip_addr(),
-                                        size as u64,
+                                        ByteCount::new(size as u64),
                                     );
 
                                     // Pass it up for processing

veilid-core/src/network_manager/native/protocol/mod.rs

@@ -5,7 +5,6 @@ pub mod wrtc;
 pub mod ws;
 
 use super::*;
-use crate::xx::*;
 use std::io;
 
 #[derive(Debug)]

veilid-core/src/network_manager/native/protocol/sockets.rs

@@ -1,4 +1,3 @@
-use crate::xx::*;
 use crate::*;
 use async_io::Async;
 use std::io;
@@ -196,7 +195,7 @@ pub async fn nonblocking_connect(
     let async_stream = Async::new(std::net::TcpStream::from(socket))?;
 
     // The stream becomes writable when connected
-    timeout_or_try!(intf::timeout(timeout_ms, async_stream.writable())
+    timeout_or_try!(timeout(timeout_ms, async_stream.writable())
         .await
         .into_timeout_or()
         .into_result()?);

veilid-core/src/network_manager/native/protocol/udp.rs

@@ -14,7 +14,7 @@ impl RawUdpProtocolHandler {
     // #[instrument(level = "trace", err, skip(self, data), fields(data.len = data.len(), ret.len, ret.descriptor))]
     pub async fn recv_message(&self, data: &mut [u8]) -> io::Result<(usize, ConnectionDescriptor)> {
         let (size, descriptor) = loop {
-            let (size, remote_addr) = network_result_value_or_log!(debug self.socket.recv_from(data).await.into_network_result()? => continue);
+            let (size, remote_addr) = network_result_value_or_log!(self.socket.recv_from(data).await.into_network_result()? => continue);
             if size > MAX_MESSAGE_SIZE {
                 log_net!(debug "{}({}) at {}@{}:{}", "Invalid message".green(), "received too large UDP message", file!(), line!(), column!());
                 continue;

veilid-core/src/network_manager/network_connection.rs

@@ -78,19 +78,19 @@ enum RecvLoopAction {
 
 #[derive(Debug, Clone)]
 pub struct NetworkConnectionStats {
-    last_message_sent_time: Option<u64>,
+    last_message_sent_time: Option<Timestamp>,
-    last_message_recv_time: Option<u64>,
+    last_message_recv_time: Option<Timestamp>,
 }
 
 
-pub type NetworkConnectionId = u64;
+pub type NetworkConnectionId = AlignedU64;
 
 #[derive(Debug)]
 pub struct NetworkConnection {
     connection_id: NetworkConnectionId,
     descriptor: ConnectionDescriptor,
     processor: Option<MustJoinHandle<()>>,
-    established_time: u64,
+    established_time: Timestamp,
     stats: Arc<Mutex<NetworkConnectionStats>>,
     sender: flume::Sender<(Option<Id>, Vec<u8>)>,
     stop_source: Option<StopSource>,
@@ -99,13 +99,13 @@ pub struct NetworkConnection {
 impl NetworkConnection {
     pub(super) fn dummy(id: NetworkConnectionId, descriptor: ConnectionDescriptor) -> Self {
         // Create handle for sending (dummy is immediately disconnected)
-        let (sender, _receiver) = flume::bounded(intf::get_concurrency() as usize);
+        let (sender, _receiver) = flume::bounded(get_concurrency() as usize);
 
         Self {
             connection_id: id,
             descriptor,
             processor: None,
-            established_time: intf::get_timestamp(),
+            established_time: get_aligned_timestamp(),
             stats: Arc::new(Mutex::new(NetworkConnectionStats {
                 last_message_sent_time: None,
                 last_message_recv_time: None,
@@ -125,7 +125,7 @@ impl NetworkConnection {
         let descriptor = protocol_connection.descriptor();
 
         // Create handle for sending
-        let (sender, receiver) = flume::bounded(intf::get_concurrency() as usize);
+        let (sender, receiver) = flume::bounded(get_concurrency() as usize);
 
         // Create stats
         let stats = Arc::new(Mutex::new(NetworkConnectionStats {
@@ -137,7 +137,7 @@ impl NetworkConnection {
         let local_stop_token = stop_source.token();
 
         // Spawn connection processor and pass in protocol connection
-        let processor = intf::spawn(Self::process_connection(
+        let processor = spawn(Self::process_connection(
             connection_manager,
             local_stop_token,
             manager_stop_token,
@@ -153,7 +153,7 @@ impl NetworkConnection {
             connection_id,
             descriptor,
             processor: Some(processor),
-            established_time: intf::get_timestamp(),
+            established_time: get_aligned_timestamp(),
             stats,
             sender,
             stop_source: Some(stop_source),
@@ -185,7 +185,7 @@ impl NetworkConnection {
         stats: Arc<Mutex<NetworkConnectionStats>>,
         message: Vec<u8>,
     ) -> io::Result<NetworkResult<()>> {
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
         let out = network_result_try!(protocol_connection.send(message).await?);
 
         let mut stats = stats.lock();
@@ -199,7 +199,7 @@ impl NetworkConnection {
         protocol_connection: &ProtocolNetworkConnection,
         stats: Arc<Mutex<NetworkConnectionStats>>,
     ) -> io::Result<NetworkResult<Vec<u8>>> {
-        let ts = intf::get_timestamp();
+        let ts = get_aligned_timestamp();
         let out = network_result_try!(protocol_connection.recv().await?);
 
         let mut stats = stats.lock();
@@ -217,7 +217,7 @@ impl NetworkConnection {
     }
 
     #[allow(dead_code)]
-    pub fn established_time(&self) -> u64 {
+    pub fn established_time(&self) -> Timestamp {
         self.established_time
     }
 
@@ -246,7 +246,7 @@ impl NetworkConnection {
             // Push mutable timer so we can reset it
             // Normally we would use an io::timeout here, but WASM won't support that, so we use a mutable sleep future
             let new_timer = || {
-                intf::sleep(connection_manager.connection_inactivity_timeout_ms()).then(|_| async {
+                sleep(connection_manager.connection_inactivity_timeout_ms()).then(|_| async {
                     // timeout
                     log_net!("== Connection timeout on {:?}", descriptor.green());
                     RecvLoopAction::Timeout
@@ -301,7 +301,7 @@ impl NetworkConnection {
                             match res {
                                 Ok(v) => {
                                     
-                                    let message = network_result_value_or_log!(debug v => {
+                                    let message = network_result_value_or_log!(v => {
                                         return RecvLoopAction::Finish;
                                     });
 

veilid-core/src/network_manager/tasks.rs

@@ -1,699 +0,0 @@
-use super::*;
-
-use crate::crypto::*;
-use crate::xx::*;
-use futures_util::FutureExt;
-use stop_token::future::FutureExt as StopFutureExt;
-
-impl NetworkManager {
-    // Bootstrap lookup process
-    #[instrument(level = "trace", skip(self), ret, err)]
-    pub(super) async fn resolve_bootstrap(
-        &self,
-        bootstrap: Vec<String>,
-    ) -> EyreResult<BootstrapRecordMap> {
-        // Resolve from bootstrap root to bootstrap hostnames
-        let mut bsnames = Vec::<String>::new();
-        for bh in bootstrap {
-            // Get TXT record for bootstrap (bootstrap.veilid.net, or similar)
-            let records = intf::txt_lookup(&bh).await?;
-            for record in records {
-                // Split the bootstrap name record by commas
-                for rec in record.split(',') {
-                    let rec = rec.trim();
-                    // If the name specified is fully qualified, go with it
-                    let bsname = if rec.ends_with('.') {
-                        rec.to_string()
-                    }
-                    // If the name is not fully qualified, prepend it to the bootstrap name
-                    else {
-                        format!("{}.{}", rec, bh)
-                    };
-
-                    // Add to the list of bootstrap name to look up
-                    bsnames.push(bsname);
-                }
-            }
-        }
-
-        // Get bootstrap nodes from hostnames concurrently
-        let mut unord = FuturesUnordered::new();
-        for bsname in bsnames {
-            unord.push(
-                async move {
-                    // look up boostrap node txt records
-                    let bsnirecords = match intf::txt_lookup(&bsname).await {
-                        Err(e) => {
-                            warn!("bootstrap node txt lookup failed for {}: {}", bsname, e);
-                            return None;
-                        }
-                        Ok(v) => v,
-                    };
-                    // for each record resolve into key/bootstraprecord pairs
-                    let mut bootstrap_records: Vec<(DHTKey, BootstrapRecord)> = Vec::new();
-                    for bsnirecord in bsnirecords {
-                        // Bootstrap TXT Record Format Version 0:
-                        // txt_version,min_version,max_version,nodeid,hostname,dialinfoshort*
-                        //
-                        // Split bootstrap node record by commas. Example:
-                        // 0,0,0,7lxDEabK_qgjbe38RtBa3IZLrud84P6NhGP-pRTZzdQ,bootstrap-1.dev.veilid.net,T5150,U5150,W5150/ws
-                        let records: Vec<String> = bsnirecord
-                            .trim()
-                            .split(',')
-                            .map(|x| x.trim().to_owned())
-                            .collect();
-                        if records.len() < 6 {
-                            warn!("invalid number of fields in bootstrap txt record");
-                            continue;
-                        }
-
-                        // Bootstrap TXT record version
-                        let txt_version: u8 = match records[0].parse::<u8>() {
-                            Ok(v) => v,
-                            Err(e) => {
-                                warn!(
-                                "invalid txt_version specified in bootstrap node txt record: {}",
-                                e
-                            );
-                                continue;
-                            }
-                        };
-                        if txt_version != BOOTSTRAP_TXT_VERSION {
-                            warn!("unsupported bootstrap txt record version");
-                            continue;
-                        }
-
-                        // Min/Max wire protocol version
-                        let min_version: u8 = match records[1].parse::<u8>() {
-                            Ok(v) => v,
-                            Err(e) => {
-                                warn!(
-                                "invalid min_version specified in bootstrap node txt record: {}",
-                                e
-                            );
-                                continue;
-                            }
-                        };
-                        let max_version: u8 = match records[2].parse::<u8>() {
-                            Ok(v) => v,
-                            Err(e) => {
-                                warn!(
-                                "invalid max_version specified in bootstrap node txt record: {}",
-                                e
-                            );
-                                continue;
-                            }
-                        };
-
-                        // Node Id
-                        let node_id_str = &records[3];
-                        let node_id_key = match DHTKey::try_decode(node_id_str) {
-                            Ok(v) => v,
-                            Err(e) => {
-                                warn!(
-                                    "Invalid node id in bootstrap node record {}: {}",
-                                    node_id_str, e
-                                );
-                                continue;
-                            }
-                        };
-
-                        // Hostname
-                        let hostname_str = &records[4];
-
-                        // If this is our own node id, then we skip it for bootstrap, in case we are a bootstrap node
-                        if self.routing_table().node_id() == node_id_key {
-                            continue;
-                        }
-
-                        // Resolve each record and store in node dial infos list
-                        let mut bootstrap_record = BootstrapRecord {
-                            min_version,
-                            max_version,
-                            dial_info_details: Vec::new(),
-                        };
-                        for rec in &records[5..] {
-                            let rec = rec.trim();
-                            let dial_infos = match DialInfo::try_vec_from_short(rec, hostname_str) {
-                                Ok(dis) => dis,
-                                Err(e) => {
-                                    warn!(
-                                        "Couldn't resolve bootstrap node dial info {}: {}",
-                                        rec, e
-                                    );
-                                    continue;
-                                }
-                            };
-
-                            for di in dial_infos {
-                                bootstrap_record.dial_info_details.push(DialInfoDetail {
-                                    dial_info: di,
-                                    class: DialInfoClass::Direct,
-                                });
-                            }
-                        }
-                        bootstrap_records.push((node_id_key, bootstrap_record));
-                    }
-                    Some(bootstrap_records)
-                }
-                .instrument(Span::current()),
-            );
-        }
-
-        let mut bsmap = BootstrapRecordMap::new();
-        while let Some(bootstrap_records) = unord.next().await {
-            if let Some(bootstrap_records) = bootstrap_records {
-                for (bskey, mut bsrec) in bootstrap_records {
-                    let rec = bsmap.entry(bskey).or_insert_with(|| BootstrapRecord {
-                        min_version: bsrec.min_version,
-                        max_version: bsrec.max_version,
-                        dial_info_details: Vec::new(),
-                    });
-                    rec.dial_info_details.append(&mut bsrec.dial_info_details);
-                }
-            }
-        }
-
-        Ok(bsmap)
-    }
-
-    // 'direct' bootstrap task routine for systems incapable of resolving TXT records, such as browser WASM
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn direct_bootstrap_task_routine(
-        self,
-        stop_token: StopToken,
-        bootstrap_dialinfos: Vec<DialInfo>,
-    ) -> EyreResult<()> {
-        let mut unord = FuturesUnordered::new();
-        let routing_table = self.routing_table();
-
-        for bootstrap_di in bootstrap_dialinfos {
-            log_net!(debug "direct bootstrap with: {}", bootstrap_di);
-
-            let peer_info = self.boot_request(bootstrap_di).await?;
-
-            log_net!(debug "  direct bootstrap peerinfo: {:?}", peer_info);
-
-            // Got peer info, let's add it to the routing table
-            for pi in peer_info {
-                let k = pi.node_id.key;
-                // Register the node
-                if let Some(nr) = routing_table.register_node_with_signed_node_info(
-                    RoutingDomain::PublicInternet,
-                    k,
-                    pi.signed_node_info,
-                    false,
-                ) {
-                    // Add this our futures to process in parallel
-                    let routing_table = routing_table.clone();
-                    unord.push(
-                        // lets ask bootstrap to find ourselves now
-                        async move { routing_table.reverse_find_node(nr, true).await }
-                            .instrument(Span::current()),
-                    );
-                }
-            }
-        }
-
-        // Wait for all bootstrap operations to complete before we complete the singlefuture
-        while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
-
-        Ok(())
-    }
-
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn bootstrap_task_routine(self, stop_token: StopToken) -> EyreResult<()> {
-        let (bootstrap, bootstrap_nodes) = {
-            let c = self.unlocked_inner.config.get();
-            (
-                c.network.bootstrap.clone(),
-                c.network.bootstrap_nodes.clone(),
-            )
-        };
-        let routing_table = self.routing_table();
-
-        log_net!(debug "--- bootstrap_task");
-
-        // See if we are specifying a direct dialinfo for bootstrap, if so use the direct mechanism
-        if !bootstrap.is_empty() && bootstrap_nodes.is_empty() {
-            let mut bootstrap_dialinfos = Vec::<DialInfo>::new();
-            for b in &bootstrap {
-                if let Ok(bootstrap_di_vec) = DialInfo::try_vec_from_url(&b) {
-                    for bootstrap_di in bootstrap_di_vec {
-                        bootstrap_dialinfos.push(bootstrap_di);
-                    }
-                }
-            }
-            if bootstrap_dialinfos.len() > 0 {
-                return self
-                    .direct_bootstrap_task_routine(stop_token, bootstrap_dialinfos)
-                    .await;
-            }
-        }
-
-        // If we aren't specifying a bootstrap node list explicitly, then pull from the bootstrap server(s)
-        let bsmap: BootstrapRecordMap = if !bootstrap_nodes.is_empty() {
-            let mut bsmap = BootstrapRecordMap::new();
-            let mut bootstrap_node_dial_infos = Vec::new();
-            for b in bootstrap_nodes {
-                let (id_str, di_str) = b
-                    .split_once('@')
-                    .ok_or_else(|| eyre!("Invalid node dial info in bootstrap entry"))?;
-                let node_id =
-                    NodeId::from_str(id_str).wrap_err("Invalid node id in bootstrap entry")?;
-                let dial_info =
-                    DialInfo::from_str(di_str).wrap_err("Invalid dial info in bootstrap entry")?;
-                bootstrap_node_dial_infos.push((node_id, dial_info));
-            }
-            for (node_id, dial_info) in bootstrap_node_dial_infos {
-                bsmap
-                    .entry(node_id.key)
-                    .or_insert_with(|| BootstrapRecord {
-                        min_version: MIN_CRYPTO_VERSION,
-                        max_version: MAX_CRYPTO_VERSION,
-                        dial_info_details: Vec::new(),
-                    })
-                    .dial_info_details
-                    .push(DialInfoDetail {
-                        dial_info,
-                        class: DialInfoClass::Direct, // Bootstraps are always directly reachable
-                    });
-            }
-            bsmap
-        } else {
-            // Resolve bootstrap servers and recurse their TXT entries
-            self.resolve_bootstrap(bootstrap).await?
-        };
-
-        // Map all bootstrap entries to a single key with multiple dialinfo
-
-        // Run all bootstrap operations concurrently
-        let mut unord = FuturesUnordered::new();
-        for (k, mut v) in bsmap {
-            // Sort dial info so we get the preferred order correct
-            v.dial_info_details.sort();
-
-            log_net!("--- bootstrapping {} with {:?}", k.encode(), &v);
-
-            // Make invalid signed node info (no signature)
-            if let Some(nr) = routing_table.register_node_with_signed_node_info(
-                RoutingDomain::PublicInternet,
-                k,
-                SignedNodeInfo::Direct(SignedDirectNodeInfo::with_no_signature(NodeInfo {
-                    network_class: NetworkClass::InboundCapable, // Bootstraps are always inbound capable
-                    outbound_protocols: ProtocolTypeSet::only(ProtocolType::UDP), // Bootstraps do not participate in relaying and will not make outbound requests, but will have UDP enabled
-                    address_types: AddressTypeSet::all(), // Bootstraps are always IPV4 and IPV6 capable
-                    min_version: v.min_version, // Minimum crypto version specified in txt record
-                    max_version: v.max_version, // Maximum crypto version specified in txt record
-                    dial_info_detail_list: v.dial_info_details, // Dial info is as specified in the bootstrap list
-                })),
-                true,
-            ) {
-                // Add this our futures to process in parallel
-                let routing_table = routing_table.clone();
-                unord.push(
-                    async move {
-                        // Need VALID signed peer info, so ask bootstrap to find_node of itself
-                        // which will ensure it has the bootstrap's signed peer info as part of the response
-                        let _ = routing_table.find_target(nr.clone()).await;
-
-                        // Ensure we got the signed peer info
-                        if !nr.signed_node_info_has_valid_signature(RoutingDomain::PublicInternet) {
-                            log_net!(warn
-                                "bootstrap at {:?} did not return valid signed node info",
-                                nr
-                            );
-                            // If this node info is invalid, it will time out after being unpingable
-                        } else {
-                            // otherwise this bootstrap is valid, lets ask it to find ourselves now
-                            routing_table.reverse_find_node(nr, true).await
-                        }
-                    }
-                    .instrument(Span::current()),
-                );
-            }
-        }
-
-        // Wait for all bootstrap operations to complete before we complete the singlefuture
-        while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
-        Ok(())
-    }
-
-    // Ping each node in the routing table if they need to be pinged
-    // to determine their reliability
-    #[instrument(level = "trace", skip(self), err)]
-    fn ping_validator_public_internet(
-        &self,
-        cur_ts: u64,
-        unord: &mut FuturesUnordered<
-            SendPinBoxFuture<Result<NetworkResult<Answer<Option<SenderInfo>>>, RPCError>>,
-        >,
-    ) -> EyreResult<()> {
-        let rpc = self.rpc_processor();
-        let routing_table = self.routing_table();
-
-        // Get all nodes needing pings in the PublicInternet routing domain
-        let node_refs = routing_table.get_nodes_needing_ping(RoutingDomain::PublicInternet, cur_ts);
-
-        // Look up any NAT mappings we may need to try to preserve with keepalives
-        let mut mapped_port_info = routing_table.get_low_level_port_info();
-
-        // Get the PublicInternet relay if we are using one
-        let opt_relay_nr = routing_table.relay_node(RoutingDomain::PublicInternet);
-        let opt_relay_id = opt_relay_nr.map(|nr| nr.node_id());
-
-        // Get our publicinternet dial info
-        let dids = routing_table.all_filtered_dial_info_details(
-            RoutingDomain::PublicInternet.into(),
-            &DialInfoFilter::all(),
-        );
-
-        // For all nodes needing pings, figure out how many and over what protocols
-        for nr in node_refs {
-            // If this is a relay, let's check for NAT keepalives
-            let mut did_pings = false;
-            if Some(nr.node_id()) == opt_relay_id {
-                // Relay nodes get pinged over all protocols we have inbound dialinfo for
-                // This is so we can preserve the inbound NAT mappings at our router
-                for did in &dids {
-                    // Do we need to do this ping?
-                    // Check if we have already pinged over this low-level-protocol/address-type/port combo
-                    // We want to ensure we do the bare minimum required here
-                    let pt = did.dial_info.protocol_type();
-                    let at = did.dial_info.address_type();
-                    let needs_ping = if let Some((llpt, port)) =
-                        mapped_port_info.protocol_to_port.get(&(pt, at))
-                    {
-                        mapped_port_info
-                            .low_level_protocol_ports
-                            .remove(&(*llpt, at, *port))
-                    } else {
-                        false
-                    };
-                    if needs_ping {
-                        let rpc = rpc.clone();
-                        let dif = did.dial_info.make_filter();
-                        let nr_filtered =
-                            nr.filtered_clone(NodeRefFilter::new().with_dial_info_filter(dif));
-                        log_net!("--> Keepalive ping to {:?}", nr_filtered);
-                        unord.push(
-                            async move { rpc.rpc_call_status(Destination::direct(nr_filtered)).await }
-                                .instrument(Span::current())
-                                .boxed(),
-                        );
-                        did_pings = true;
-                    }
-                }
-            }
-            // Just do a single ping with the best protocol for all the other nodes,
-            // ensuring that we at least ping a relay with -something- even if we didnt have
-            // any mapped ports to preserve
-            if !did_pings {
-                let rpc = rpc.clone();
-                unord.push(
-                    async move { rpc.rpc_call_status(Destination::direct(nr)).await }
-                        .instrument(Span::current())
-                        .boxed(),
-                );
-            }
-        }
-
-        Ok(())
-    }
-
-    // Ping each node in the LocalNetwork routing domain if they
-    // need to be pinged to determine their reliability
-    #[instrument(level = "trace", skip(self), err)]
-    fn ping_validator_local_network(
-        &self,
-        cur_ts: u64,
-        unord: &mut FuturesUnordered<
-            SendPinBoxFuture<Result<NetworkResult<Answer<Option<SenderInfo>>>, RPCError>>,
-        >,
-    ) -> EyreResult<()> {
-        let rpc = self.rpc_processor();
-        let routing_table = self.routing_table();
-
-        // Get all nodes needing pings in the LocalNetwork routing domain
-        let node_refs = routing_table.get_nodes_needing_ping(RoutingDomain::LocalNetwork, cur_ts);
-
-        // For all nodes needing pings, figure out how many and over what protocols
-        for nr in node_refs {
-            let rpc = rpc.clone();
-
-            // Just do a single ping with the best protocol for all the nodes
-            unord.push(
-                async move { rpc.rpc_call_status(Destination::direct(nr)).await }
-                    .instrument(Span::current())
-                    .boxed(),
-            );
-        }
-
-        Ok(())
-    }
-
-    // Ping each node in the routing table if they need to be pinged
-    // to determine their reliability
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn ping_validator_task_routine(
-        self,
-        stop_token: StopToken,
-        _last_ts: u64,
-        cur_ts: u64,
-    ) -> EyreResult<()> {
-        let mut unord = FuturesUnordered::new();
-
-        // PublicInternet
-        self.ping_validator_public_internet(cur_ts, &mut unord)?;
-
-        // LocalNetwork
-        self.ping_validator_local_network(cur_ts, &mut unord)?;
-
-        // Wait for ping futures to complete in parallel
-        while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {}
-
-        Ok(())
-    }
-
-    // Ask our remaining peers to give us more peers before we go
-    // back to the bootstrap servers to keep us from bothering them too much
-    // This only adds PublicInternet routing domain peers. The discovery
-    // mechanism for LocalNetwork suffices for locating all the local network
-    // peers that are available. This, however, may query other LocalNetwork
-    // nodes for their PublicInternet peers, which is a very fast way to get
-    // a new node online.
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn peer_minimum_refresh_task_routine(
-        self,
-        stop_token: StopToken,
-    ) -> EyreResult<()> {
-        let routing_table = self.routing_table();
-        let mut ord = FuturesOrdered::new();
-        let min_peer_count = {
-            let c = self.unlocked_inner.config.get();
-            c.network.dht.min_peer_count as usize
-        };
-
-        // For the PublicInternet routing domain, get list of all peers we know about
-        // even the unreliable ones, and ask them to find nodes close to our node too
-        let noderefs = routing_table.find_fastest_nodes(
-            min_peer_count,
-            VecDeque::new(),
-            |_rti, k: DHTKey, v: Option<Arc<BucketEntry>>| {
-                NodeRef::new(routing_table.clone(), k, v.unwrap().clone(), None)
-            },
-        );
-        for nr in noderefs {
-            let routing_table = routing_table.clone();
-            ord.push_back(
-                async move { routing_table.reverse_find_node(nr, false).await }
-                    .instrument(Span::current()),
-            );
-        }
-
-        // do peer minimum search in order from fastest to slowest
-        while let Ok(Some(_)) = ord.next().timeout_at(stop_token.clone()).await {}
-
-        Ok(())
-    }
-
-    // Keep relays assigned and accessible
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn relay_management_task_routine(
-        self,
-        _stop_token: StopToken,
-        _last_ts: u64,
-        cur_ts: u64,
-    ) -> EyreResult<()> {
-        // Get our node's current node info and network class and do the right thing
-        let routing_table = self.routing_table();
-        let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet);
-        let own_node_info = own_peer_info.signed_node_info.node_info();
-        let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet);
-
-        // Get routing domain editor
-        let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet);
-
-        // Do we know our network class yet?
-        if let Some(network_class) = network_class {
-            // If we already have a relay, see if it is dead, or if we don't need it any more
-            let has_relay = {
-                if let Some(relay_node) = routing_table.relay_node(RoutingDomain::PublicInternet) {
-                    let state = relay_node.state(cur_ts);
-                    // Relay node is dead or no longer needed
-                    if matches!(state, BucketEntryState::Dead) {
-                        info!("Relay node died, dropping relay {}", relay_node);
-                        editor.clear_relay_node();
-                        false
-                    } else if !own_node_info.requires_relay() {
-                        info!(
-                            "Relay node no longer required, dropping relay {}",
-                            relay_node
-                        );
-                        editor.clear_relay_node();
-                        false
-                    } else {
-                        true
-                    }
-                } else {
-                    false
-                }
-            };
-
-            // Do we need a relay?
-            if !has_relay && own_node_info.requires_relay() {
-                // Do we want an outbound relay?
-                let mut got_outbound_relay = false;
-                if network_class.outbound_wants_relay() {
-                    // The outbound relay is the host of the PWA
-                    if let Some(outbound_relay_peerinfo) = intf::get_outbound_relay_peer().await {
-                        // Register new outbound relay
-                        if let Some(nr) = routing_table.register_node_with_signed_node_info(
-                            RoutingDomain::PublicInternet,
-                            outbound_relay_peerinfo.node_id.key,
-                            outbound_relay_peerinfo.signed_node_info,
-                            false,
-                        ) {
-                            info!("Outbound relay node selected: {}", nr);
-                            editor.set_relay_node(nr);
-                            got_outbound_relay = true;
-                        }
-                    }
-                }
-                if !got_outbound_relay {
-                    // Find a node in our routing table that is an acceptable inbound relay
-                    if let Some(nr) =
-                        routing_table.find_inbound_relay(RoutingDomain::PublicInternet, cur_ts)
-                    {
-                        info!("Inbound relay node selected: {}", nr);
-                        editor.set_relay_node(nr);
-                    }
-                }
-            }
-        }
-
-        // Commit the changes
-        editor.commit().await;
-
-        Ok(())
-    }
-
-    // Keep private routes assigned and accessible
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn private_route_management_task_routine(
-        self,
-        _stop_token: StopToken,
-        _last_ts: u64,
-        cur_ts: u64,
-    ) -> EyreResult<()> {
-        // Get our node's current node info and network class and do the right thing
-        let routing_table = self.routing_table();
-        let own_peer_info = routing_table.get_own_peer_info(RoutingDomain::PublicInternet);
-        let network_class = routing_table.get_network_class(RoutingDomain::PublicInternet);
-
-        // Get routing domain editor
-        let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet);
-
-        // Do we know our network class yet?
-        if let Some(network_class) = network_class {
-
-            // see if we have any routes that need extending
-        }
-
-        // Commit the changes
-        editor.commit().await;
-
-        Ok(())
-    }
-
-    // Compute transfer statistics for the low level network
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn rolling_transfers_task_routine(
-        self,
-        _stop_token: StopToken,
-        last_ts: u64,
-        cur_ts: u64,
-    ) -> EyreResult<()> {
-        // log_net!("--- network manager rolling_transfers task");
-        {
-            let inner = &mut *self.inner.lock();
-
-            // Roll the low level network transfer stats for our address
-            inner
-                .stats
-                .self_stats
-                .transfer_stats_accounting
-                .roll_transfers(last_ts, cur_ts, &mut inner.stats.self_stats.transfer_stats);
-
-            // Roll all per-address transfers
-            let mut dead_addrs: HashSet<PerAddressStatsKey> = HashSet::new();
-            for (addr, stats) in &mut inner.stats.per_address_stats {
-                stats.transfer_stats_accounting.roll_transfers(
-                    last_ts,
-                    cur_ts,
-                    &mut stats.transfer_stats,
-                );
-
-                // While we're here, lets see if this address has timed out
-                if cur_ts - stats.last_seen_ts >= IPADDR_MAX_INACTIVE_DURATION_US {
-                    // it's dead, put it in the dead list
-                    dead_addrs.insert(*addr);
-                }
-            }
-
-            // Remove the dead addresses from our tables
-            for da in &dead_addrs {
-                inner.stats.per_address_stats.remove(da);
-            }
-        }
-
-        // Send update
-        self.send_network_update();
-
-        Ok(())
-    }
-
-    // Clean up the public address check tables, removing entries that have timed out
-    #[instrument(level = "trace", skip(self), err)]
-    pub(super) async fn public_address_check_task_routine(
-        self,
-        stop_token: StopToken,
-        _last_ts: u64,
-        cur_ts: u64,
-    ) -> EyreResult<()> {
-        // go through public_address_inconsistencies_table and time out things that have expired
-        let mut inner = self.inner.lock();
-        for (_, pait_v) in &mut inner.public_address_inconsistencies_table {
-            let mut expired = Vec::new();
-            for (addr, exp_ts) in pait_v.iter() {
-                if *exp_ts <= cur_ts {
-                    expired.push(*addr);
-                }
-            }
-            for exp in expired {
-                pait_v.remove(&exp);
-            }
-        }
-        Ok(())
-    }
-}

veilid-core/src/network_manager/tasks/mod.rs

@@ -0,0 +1,76 @@
+pub mod public_address_check;
+pub mod rolling_transfers;
+
+use super::*;
+
+impl NetworkManager {
+    pub(crate) fn start_tasks(&self) {
+        // Set rolling transfers tick task
+        {
+            let this = self.clone();
+            self.unlocked_inner
+                .rolling_transfers_task
+                .set_routine(move |s, l, t| {
+                    Box::pin(
+                        this.clone()
+                            .rolling_transfers_task_routine(s, Timestamp::new(l), Timestamp::new(t))
+                            .instrument(trace_span!(
+                                parent: None,
+                                "NetworkManager rolling transfers task routine"
+                            )),
+                    )
+                });
+        }
+
+        // Set public address check task
+        {
+            let this = self.clone();
+            self.unlocked_inner
+                .public_address_check_task
+                .set_routine(move |s, l, t| {
+                    Box::pin(
+                        this.clone()
+                            .public_address_check_task_routine(
+                                s,
+                                Timestamp::new(l),
+                                Timestamp::new(t),
+                            )
+                            .instrument(trace_span!(
+                                parent: None,
+                                "public address check task routine"
+                            )),
+                    )
+                });
+        }
+    }
+
+    pub async fn tick(&self) -> EyreResult<()> {
+        let routing_table = self.routing_table();
+        let net = self.net();
+        let receipt_manager = self.receipt_manager();
+
+        // Run the rolling transfers task
+        self.unlocked_inner.rolling_transfers_task.tick().await?;
+
+        // Run the routing table tick
+        routing_table.tick().await?;
+
+        // Run the low level network tick
+        net.tick().await?;
+
+        // Run the receipt manager tick
+        receipt_manager.tick().await?;
+
+        // Purge the client whitelist
+        self.purge_client_whitelist();
+
+        Ok(())
+    }
+
+    pub(crate) async fn stop_tasks(&self) {
+        debug!("stopping rolling transfers task");
+        if let Err(e) = self.unlocked_inner.rolling_transfers_task.stop().await {
+            warn!("rolling_transfers_task not stopped: {}", e);
+        }
+    }
+}

veilid-core/src/network_manager/tasks/public_address_check.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+impl NetworkManager {
+    // Clean up the public address check tables, removing entries that have timed out
+    #[instrument(level = "trace", skip(self), err)]
+    pub(crate) async fn public_address_check_task_routine(
+        self,
+        stop_token: StopToken,
+        _last_ts: Timestamp,
+        cur_ts: Timestamp,
+    ) -> EyreResult<()> {
+        // go through public_address_inconsistencies_table and time out things that have expired
+        let mut inner = self.inner.lock();
+        for (_, pait_v) in &mut inner.public_address_inconsistencies_table {
+            let mut expired = Vec::new();
+            for (addr, exp_ts) in pait_v.iter() {
+                if *exp_ts <= cur_ts {
+                    expired.push(*addr);
+                }
+            }
+            for exp in expired {
+                pait_v.remove(&exp);
+            }
+        }
+        Ok(())
+    }
+}

veilid-core/src/network_manager/tasks/rolling_transfers.rs

@@ -0,0 +1,50 @@
+use super::*;
+
+impl NetworkManager {
+    // Compute transfer statistics for the low level network
+    #[instrument(level = "trace", skip(self), err)]
+    pub(crate) async fn rolling_transfers_task_routine(
+        self,
+        _stop_token: StopToken,
+        last_ts: Timestamp,
+        cur_ts: Timestamp,
+    ) -> EyreResult<()> {
+        // log_net!("--- network manager rolling_transfers task");
+        {
+            let inner = &mut *self.inner.lock();
+
+            // Roll the low level network transfer stats for our address
+            inner
+                .stats
+                .self_stats
+                .transfer_stats_accounting
+                .roll_transfers(last_ts, cur_ts, &mut inner.stats.self_stats.transfer_stats);
+
+            // Roll all per-address transfers
+            let mut dead_addrs: HashSet<PerAddressStatsKey> = HashSet::new();
+            for (addr, stats) in &mut inner.stats.per_address_stats {
+                stats.transfer_stats_accounting.roll_transfers(
+                    last_ts,
+                    cur_ts,
+                    &mut stats.transfer_stats,
+                );
+
+                // While we're here, lets see if this address has timed out
+                if cur_ts.saturating_sub(stats.last_seen_ts) >= IPADDR_MAX_INACTIVE_DURATION_US {
+                    // it's dead, put it in the dead list
+                    dead_addrs.insert(*addr);
+                }
+            }
+
+            // Remove the dead addresses from our tables
+            for da in &dead_addrs {
+                inner.stats.per_address_stats.remove(da);
+            }
+        }
+
+        // Send update
+        self.send_network_update();
+
+        Ok(())
+    }
+}

veilid-core/src/network_manager/tests/test_connection_table.rs

@@ -1,8 +1,7 @@
+use super::*;
+
 use super::connection_table::*;
-use super::network_connection::*;
 use crate::tests::common::test_veilid_config::*;
-use crate::xx::*;
-use crate::*;
 
 pub async fn test_add_get_remove() {
     let config = get_config();
@@ -51,13 +50,13 @@ pub async fn test_add_get_remove() {
         ))),
     );
 
-    let c1 = NetworkConnection::dummy(1, a1);
+    let c1 = NetworkConnection::dummy(1.into(), a1);
-    let c1b = NetworkConnection::dummy(10, a1);
+    let c1b = NetworkConnection::dummy(10.into(), a1);
     let c1h = c1.get_handle();
-    let c2 = NetworkConnection::dummy(2, a2);
+    let c2 = NetworkConnection::dummy(2.into(), a2);
-    let c3 = NetworkConnection::dummy(3, a3);
+    let c3 = NetworkConnection::dummy(3.into(), a3);
-    let c4 = NetworkConnection::dummy(4, a4);
+    let c4 = NetworkConnection::dummy(4.into(), a4);
-    let c5 = NetworkConnection::dummy(5, a5);
+    let c5 = NetworkConnection::dummy(5.into(), a5);
 
     assert_eq!(a1, c2.connection_descriptor());
     assert_ne!(a3, c4.connection_descriptor());
@@ -69,8 +68,8 @@ pub async fn test_add_get_remove() {
     assert!(table.add_connection(c1b).is_err());
 
     assert_eq!(table.connection_count(), 1);
-    assert!(table.remove_connection_by_id(4).is_none());
+    assert!(table.remove_connection_by_id(4.into()).is_none());
-    assert!(table.remove_connection_by_id(5).is_none());
+    assert!(table.remove_connection_by_id(5.into()).is_none());
     assert_eq!(table.connection_count(), 1);
     assert_eq!(table.get_connection_by_descriptor(a1), Some(c1h.clone()));
     assert_eq!(table.get_connection_by_descriptor(a1), Some(c1h.clone()));
@@ -82,41 +81,41 @@ pub async fn test_add_get_remove() {
     assert_eq!(table.connection_count(), 1);
     assert_eq!(
         table
-            .remove_connection_by_id(1)
+            .remove_connection_by_id(1.into())
             .map(|c| c.connection_descriptor())
             .unwrap(),
         a1
     );
     assert_eq!(table.connection_count(), 0);
-    assert!(table.remove_connection_by_id(2).is_none());
+    assert!(table.remove_connection_by_id(2.into()).is_none());
     assert_eq!(table.connection_count(), 0);
     assert_eq!(table.get_connection_by_descriptor(a2), None);
     assert_eq!(table.get_connection_by_descriptor(a1), None);
     assert_eq!(table.connection_count(), 0);
-    let c1 = NetworkConnection::dummy(6, a1);
+    let c1 = NetworkConnection::dummy(6.into(), a1);
     table.add_connection(c1).unwrap();
-    let c2 = NetworkConnection::dummy(7, a2);
+    let c2 = NetworkConnection::dummy(7.into(), a2);
     assert_err!(table.add_connection(c2));
     table.add_connection(c3).unwrap();
     table.add_connection(c4).unwrap();
     assert_eq!(table.connection_count(), 3);
     assert_eq!(
         table
-            .remove_connection_by_id(6)
+            .remove_connection_by_id(6.into())
             .map(|c| c.connection_descriptor())
             .unwrap(),
         a2
     );
     assert_eq!(
         table
-            .remove_connection_by_id(3)
+            .remove_connection_by_id(3.into())
             .map(|c| c.connection_descriptor())
             .unwrap(),
         a3
     );
     assert_eq!(
         table
-            .remove_connection_by_id(4)
+            .remove_connection_by_id(4.into())
             .map(|c| c.connection_descriptor())
             .unwrap(),
         a4

veilid-core/src/network_manager/wasm/mod.rs

@@ -1,6 +1,7 @@
 mod protocol;
 
 use super::*;
+
 use crate::routing_table::*;
 use connection_manager::*;
 use protocol::ws::WebsocketProtocolHandler;
@@ -108,7 +109,7 @@ impl Network {
 
         // Network accounting
         self.network_manager()
-            .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+            .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
         Ok(NetworkResult::Value(()))
     }
@@ -151,7 +152,7 @@ impl Network {
 
                 network_result_try!(pnc.send(data).await.wrap_err("send failure")?);
                 self.network_manager()
-                    .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+                    .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
                 let out = network_result_try!(network_result_try!(timeout(timeout_ms, pnc.recv())
                     .await
@@ -159,7 +160,7 @@ impl Network {
                 .wrap_err("recv failure")?);
 
                 self.network_manager()
-                    .stats_packet_rcvd(dial_info.to_ip_addr(), out.len() as u64);
+                    .stats_packet_rcvd(dial_info.to_ip_addr(), ByteCount::new(out.len() as u64));
 
                 Ok(NetworkResult::Value(out))
             }
@@ -193,7 +194,7 @@ impl Network {
                     // Network accounting
                     self.network_manager().stats_packet_sent(
                         descriptor.remote().to_socket_addr().ip(),
-                        data_len as u64,
+                        ByteCount::new(data_len as u64),
                     );
 
                     // Data was consumed
@@ -242,7 +243,7 @@ impl Network {
 
         // Network accounting
         self.network_manager()
-            .stats_packet_sent(dial_info.to_ip_addr(), data_len as u64);
+            .stats_packet_sent(dial_info.to_ip_addr(), ByteCount::new(data_len as u64));
 
         Ok(NetworkResult::value(connection_descriptor))
     }
@@ -251,7 +252,7 @@ impl Network {
 
     pub async fn startup(&self) -> EyreResult<()> {
         // get protocol config
-        self.inner.lock().protocol_config = {
+        let protocol_config = {
             let c = self.config.get();
             let inbound = ProtocolTypeSet::new();
             let mut outbound = ProtocolTypeSet::new();
@@ -268,12 +269,31 @@ impl Network {
             let family_local = AddressTypeSet::all();
 
             ProtocolConfig {
-                inbound,
                 outbound,
+                inbound,
                 family_global,
                 family_local,
             }
         };
+        self.inner.lock().protocol_config = protocol_config;
+
+        // Start editing routing table
+        let mut editor_public_internet = self
+            .unlocked_inner
+            .routing_table
+            .edit_routing_domain(RoutingDomain::PublicInternet);
+
+        // set up the routing table's network config
+        // if we have static public dialinfo, upgrade our network class
+        editor_public_internet.setup_network(
+            protocol_config.outbound,
+            protocol_config.inbound,
+            protocol_config.family_global,
+        );
+        editor_public_internet.set_network_class(Some(NetworkClass::WebApp));
+
+        // commit routing table edits
+        editor_public_internet.commit().await;
 
         self.inner.lock().network_started = true;
         Ok(())
@@ -299,13 +319,8 @@ impl Network {
 
         // Drop all dial info
         let mut editor = routing_table.edit_routing_domain(RoutingDomain::PublicInternet);
-        editor.disable_node_info_updates();
-        editor.clear_dial_info_details();
-        editor.commit().await;
-
-        let mut editor = routing_table.edit_routing_domain(RoutingDomain::LocalNetwork);
-        editor.disable_node_info_updates();
         editor.clear_dial_info_details();
+        editor.set_network_class(None);
         editor.commit().await;
 
         // Cancels all async background tasks by dropping join handles
@@ -335,15 +350,6 @@ impl Network {
         false
     }
 
-    pub fn get_network_class(&self, _routing_domain: RoutingDomain) -> Option<NetworkClass> {
-        // xxx eventually detect tor browser?
-        return if self.inner.lock().network_started {
-            Some(NetworkClass::WebApp)
-        } else {
-            None
-        };
-    }
-
     pub fn get_protocol_config(&self) -> ProtocolConfig {
         self.inner.lock().protocol_config.clone()
     }

veilid-core/src/network_manager/wasm/protocol/mod.rs

@@ -2,7 +2,6 @@ pub mod wrtc;
 pub mod ws;
 
 use super::*;
-use crate::xx::*;
 use std::io;
 
 #[derive(Debug)]

veilid-core/src/network_manager/wasm/protocol/ws.rs

@@ -11,12 +11,20 @@ struct WebsocketNetworkConnectionInner {
 
 fn to_io(err: WsErr) -> io::Error {
     match err {
-        WsErr::InvalidWsState { supplied: _ } => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
+        WsErr::InvalidWsState { supplied: _ } => {
+            io::Error::new(io::ErrorKind::InvalidInput, err.to_string())
+        }
         WsErr::ConnectionNotOpen => io::Error::new(io::ErrorKind::NotConnected, err.to_string()),
-        WsErr::InvalidUrl { supplied: _ } => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
+        WsErr::InvalidUrl { supplied: _ } => {
-        WsErr::InvalidCloseCode { supplied: _ } => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
+            io::Error::new(io::ErrorKind::InvalidInput, err.to_string())
+        }
+        WsErr::InvalidCloseCode { supplied: _ } => {
+            io::Error::new(io::ErrorKind::InvalidInput, err.to_string())
+        }
         WsErr::ReasonStringToLong => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
-        WsErr::ConnectionFailed { event: _ }  => io::Error::new(io::ErrorKind::ConnectionRefused, err.to_string()),
+        WsErr::ConnectionFailed { event: _ } => {
+            io::Error::new(io::ErrorKind::ConnectionRefused, err.to_string())
+        }
         WsErr::InvalidEncoding => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
         WsErr::CantDecodeBlob => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
         WsErr::UnknownDataType => io::Error::new(io::ErrorKind::InvalidInput, err.to_string()),
@@ -80,19 +88,19 @@ impl WebsocketNetworkConnection {
         let out = match SendWrapper::new(self.inner.ws_stream.clone().next()).await {
             Some(WsMessage::Binary(v)) => {
                 if v.len() > MAX_MESSAGE_SIZE {
-                    return Err(io::Error::new(
+                    return Ok(NetworkResult::invalid_message("too large ws message"));
-                        io::ErrorKind::InvalidData,
-                        "too large ws message",
-                    ));
                 }
                 NetworkResult::Value(v)
             }
-            Some(_) => NetworkResult::NoConnection(io::Error::new(
+            Some(_) => NetworkResult::no_connection_other(io::Error::new(
                 io::ErrorKind::ConnectionReset,
                 "Unexpected WS message type",
             )),
             None => {
-                bail_io_error_other!("WS stream closed");
+                return Ok(NetworkResult::no_connection(io::Error::new(
+                    io::ErrorKind::ConnectionReset,
+                    "WS stream closed",
+                )));
             }
         };
         // tracing::Span::current().record("network_result", &tracing::field::display(&out));
@@ -126,7 +134,7 @@ impl WebsocketProtocolHandler {
         let fut = SendWrapper::new(timeout(timeout_ms, async move {
             WsMeta::connect(request, None).await.map_err(to_io)
         }));
-        
+
         let (wsmeta, wsio) = network_result_try!(network_result_try!(fut
             .await
             .into_network_result())

veilid-core/src/receipt_manager.rs

@@ -5,7 +5,6 @@ use futures_util::stream::{FuturesUnordered, StreamExt};
 use network_manager::*;
 use routing_table::*;
 use stop_token::future::FutureExt;
-use xx::*;
 
 #[derive(Clone, Debug)]
 pub enum ReceiptEvent {
@@ -71,7 +70,7 @@ impl fmt::Debug for ReceiptRecordCallbackType {
 }
 
 pub struct ReceiptRecord {
-    expiration_ts: u64,
+    expiration_ts: Timestamp,
     receipt: Receipt,
     expected_returns: u32,
     returns_so_far: u32,
@@ -93,7 +92,7 @@ impl fmt::Debug for ReceiptRecord {
 impl ReceiptRecord {
     pub fn new(
         receipt: Receipt,
-        expiration_ts: u64,
+        expiration_ts: Timestamp,
         expected_returns: u32,
         receipt_callback: impl ReceiptCallback,
     ) -> Self {
@@ -108,7 +107,7 @@ impl ReceiptRecord {
 
     pub fn new_single_shot(
         receipt: Receipt,
-        expiration_ts: u64,
+        expiration_ts: Timestamp,
         eventual: ReceiptSingleShotType,
     ) -> Self {
         Self {
@@ -124,7 +123,7 @@ impl ReceiptRecord {
 /* XXX: may be useful for O(1) timestamp expiration
 #[derive(Clone, Debug)]
 struct ReceiptRecordTimestampSort {
-    expiration_ts: u64,
+    expiration_ts: Timestamp,
     record: Arc<Mutex<ReceiptRecord>>,
 }
 
@@ -151,7 +150,7 @@ impl PartialOrd for ReceiptRecordTimestampSort {
 pub struct ReceiptManagerInner {
     network_manager: NetworkManager,
     records_by_nonce: BTreeMap<ReceiptNonce, Arc<Mutex<ReceiptRecord>>>,
-    next_oldest_ts: Option<u64>,
+    next_oldest_ts: Option<Timestamp>,
     stop_source: Option<StopSource>,
     timeout_task: MustJoinSingleFuture<()>,
 }
@@ -220,9 +219,9 @@ impl ReceiptManager {
     }
 
     #[instrument(level = "trace", skip(self))]
-    pub async fn timeout_task_routine(self, now: u64, stop_token: StopToken) {
+    pub async fn timeout_task_routine(self, now: Timestamp, stop_token: StopToken) {
         // Go through all receipts and build a list of expired nonces
-        let mut new_next_oldest_ts: Option<u64> = None;
+        let mut new_next_oldest_ts: Option<Timestamp> = None;
         let mut expired_records = Vec::new();
         {
             let mut inner = self.inner.lock();
@@ -281,7 +280,7 @@ impl ReceiptManager {
             };
             (inner.next_oldest_ts, inner.timeout_task.clone(), stop_token)
         };
-        let now = intf::get_timestamp();
+        let now = get_aligned_timestamp();
         // If we have at least one timestamp to expire, lets do it
         if let Some(next_oldest_ts) = next_oldest_ts {
             if now >= next_oldest_ts {
@@ -319,7 +318,7 @@ impl ReceiptManager {
     pub fn record_receipt(
         &self,
         receipt: Receipt,
-        expiration: u64,
+        expiration: Timestamp,
         expected_returns: u32,
         callback: impl ReceiptCallback,
     ) {
@@ -340,7 +339,7 @@ impl ReceiptManager {
     pub fn record_single_shot_receipt(
         &self,
         receipt: Receipt,
-        expiration: u64,
+        expiration: Timestamp,
         eventual: ReceiptSingleShotType,
     ) {
         let receipt_nonce = receipt.get_nonce();
@@ -357,7 +356,7 @@ impl ReceiptManager {
 
     fn update_next_oldest_timestamp(inner: &mut ReceiptManagerInner) {
         // Update the next oldest timestamp
-        let mut new_next_oldest_ts: Option<u64> = None;
+        let mut new_next_oldest_ts: Option<Timestamp> = None;
         for v in inner.records_by_nonce.values() {
             let receipt_inner = v.lock();
             if new_next_oldest_ts.is_none()

veilid-core/src/routing_table/bucket.rs

@@ -120,7 +120,7 @@ impl Bucket {
             .iter()
             .map(|(k, v)| (k.clone(), v.clone()))
             .collect();
-        let cur_ts = intf::get_timestamp();
+        let cur_ts = get_aligned_timestamp();
         sorted_entries.sort_by(|a, b| -> core::cmp::Ordering {
             if a.0 == b.0 {
                 return core::cmp::Ordering::Equal;

veilid-core/src/routing_table/bucket_entry.rs

@@ -50,8 +50,8 @@ pub struct LastConnectionKey(ProtocolType, AddressType);
 pub struct BucketEntryPublicInternet {
     /// The PublicInternet node info
     signed_node_info: Option<Box<SignedNodeInfo>>,
-    /// If this node has seen our publicinternet node info
+    /// The last node info timestamp of ours that this entry has seen
-    seen_our_node_info: bool,
+    last_seen_our_node_info_ts: Timestamp,
     /// Last known node status
     node_status: Option<PublicInternetNodeStatus>,
 }
@@ -62,8 +62,8 @@ pub struct BucketEntryPublicInternet {
 pub struct BucketEntryLocalNetwork {
     /// The LocalNetwork node info
     signed_node_info: Option<Box<SignedNodeInfo>>,
-    /// If this node has seen our localnetwork node info
+    /// The last node info timestamp of ours that this entry has seen
-    seen_our_node_info: bool,
+    last_seen_our_node_info_ts: Timestamp,
     /// Last known node status
     node_status: Option<LocalNetworkNodeStatus>,
 }
@@ -93,7 +93,7 @@ pub struct BucketEntryInner {
     updated_since_last_network_change: bool,
     /// The last connection descriptors used to contact this node, per protocol type
     #[with(Skip)]
-    last_connections: BTreeMap<LastConnectionKey, (ConnectionDescriptor, u64)>,
+    last_connections: BTreeMap<LastConnectionKey, (ConnectionDescriptor, Timestamp)>,
     /// The node info for this entry on the publicinternet routing domain
     public_internet: BucketEntryPublicInternet,
     /// The node info for this entry on the localnetwork routing domain
@@ -148,7 +148,7 @@ impl BucketEntryInner {
     }
 
     // Less is more reliable then faster
-    pub fn cmp_fastest_reliable(cur_ts: u64, e1: &Self, e2: &Self) -> std::cmp::Ordering {
+    pub fn cmp_fastest_reliable(cur_ts: Timestamp, e1: &Self, e2: &Self) -> std::cmp::Ordering {
         // Reverse compare so most reliable is at front
         let ret = e2.state(cur_ts).cmp(&e1.state(cur_ts));
         if ret != std::cmp::Ordering::Equal {
@@ -170,7 +170,7 @@ impl BucketEntryInner {
     }
 
     // Less is more reliable then older
-    pub fn cmp_oldest_reliable(cur_ts: u64, e1: &Self, e2: &Self) -> std::cmp::Ordering {
+    pub fn cmp_oldest_reliable(cur_ts: Timestamp, e1: &Self, e2: &Self) -> std::cmp::Ordering {
         // Reverse compare so most reliable is at front
         let ret = e2.state(cur_ts).cmp(&e1.state(cur_ts));
         if ret != std::cmp::Ordering::Equal {
@@ -191,7 +191,7 @@ impl BucketEntryInner {
         }
     }
 
-    pub fn sort_fastest_reliable_fn(cur_ts: u64) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+    pub fn sort_fastest_reliable_fn(cur_ts: Timestamp) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
         move |e1, e2| Self::cmp_fastest_reliable(cur_ts, e1, e2)
     }
 
@@ -231,7 +231,7 @@ impl BucketEntryInner {
                         // No need to update the signednodeinfo though since the timestamp is the same
                         // Touch the node and let it try to live again
                         self.updated_since_last_network_change = true;
-                        self.touch_last_seen(intf::get_timestamp());
+                        self.touch_last_seen(get_aligned_timestamp());
                     }
                     return;
                 }
@@ -258,7 +258,7 @@ impl BucketEntryInner {
         // Update the signed node info
         *opt_current_sni = Some(Box::new(signed_node_info));
         self.updated_since_last_network_change = true;
-        self.touch_last_seen(intf::get_timestamp());
+        self.touch_last_seen(get_aligned_timestamp());
     }
 
     pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool {
@@ -275,6 +275,25 @@ impl BucketEntryInner {
         false
     }
 
+    pub fn exists_in_routing_domain(
+        &self,
+        rti: &RoutingTableInner,
+        routing_domain: RoutingDomain,
+    ) -> bool {
+        // Check node info
+        if self.has_node_info(routing_domain.into()) {
+            return true;
+        }
+
+        // Check connections
+        let last_connections = self.last_connections(
+            rti,
+            true,
+            Some(NodeRefFilter::new().with_routing_domain(routing_domain)),
+        );
+        !last_connections.is_empty()
+    }
+
     pub fn node_info(&self, routing_domain: RoutingDomain) -> Option<&NodeInfo> {
         let opt_current_sni = match routing_domain {
             RoutingDomain::LocalNetwork => &self.local_network.signed_node_info,
@@ -304,8 +323,10 @@ impl BucketEntryInner {
 
     pub fn best_routing_domain(
         &self,
+        rti: &RoutingTableInner,
         routing_domain_set: RoutingDomainSet,
     ) -> Option<RoutingDomain> {
+        // Check node info
         for routing_domain in routing_domain_set {
             let opt_current_sni = match routing_domain {
                 RoutingDomain::LocalNetwork => &self.local_network.signed_node_info,
@@ -315,7 +336,27 @@ impl BucketEntryInner {
                 return Some(routing_domain);
             }
         }
-        None
+        // Check connections
+        let mut best_routing_domain: Option<RoutingDomain> = None;
+        let last_connections = self.last_connections(
+            rti,
+            true,
+            Some(NodeRefFilter::new().with_routing_domain_set(routing_domain_set)),
+        );
+        for lc in last_connections {
+            if let Some(rd) =
+                rti.routing_domain_for_address(lc.0.remote_address().address())
+            {
+                if let Some(brd) = best_routing_domain {
+                    if rd < brd {
+                        best_routing_domain = Some(rd);
+                    }
+                } else {
+                    best_routing_domain = Some(rd);
+                }
+            }
+        }
+        best_routing_domain
     }
 
     fn descriptor_to_key(&self, last_connection: ConnectionDescriptor) -> LastConnectionKey {
@@ -326,7 +367,7 @@ impl BucketEntryInner {
     }
 
     // Stores a connection descriptor in this entry's table of last connections
-    pub fn set_last_connection(&mut self, last_connection: ConnectionDescriptor, timestamp: u64) {
+    pub fn set_last_connection(&mut self, last_connection: ConnectionDescriptor, timestamp: Timestamp) {
         let key = self.descriptor_to_key(last_connection);
         self.last_connections
             .insert(key, (last_connection, timestamp));
@@ -337,13 +378,17 @@ impl BucketEntryInner {
         self.last_connections.clear();
     }
 
-    // Gets all the 'last connections' that match a particular filter
+    // Gets all the 'last connections' that match a particular filter, and their accompanying timestamps of last use
     pub(super) fn last_connections(
         &self,
         rti: &RoutingTableInner,
+        only_live: bool,
         filter: Option<NodeRefFilter>,
-    ) -> Vec<(ConnectionDescriptor, u64)> {
+    ) -> Vec<(ConnectionDescriptor, Timestamp)> {
-        let mut out: Vec<(ConnectionDescriptor, u64)> = self
+        let connection_manager =
+            rti.unlocked_inner.network_manager.connection_manager();
+
+        let mut out: Vec<(ConnectionDescriptor, Timestamp)> = self
             .last_connections
             .iter()
             .filter_map(|(k, v)| {
@@ -368,7 +413,29 @@ impl BucketEntryInner {
                     // no filter
                     true
                 };
-                if include {
+
+                if !include {
+                    return None;
+                }
+
+                if !only_live {
+                    return Some(v.clone());
+                }
+
+                // Check if the connection is still considered live
+                let alive = 
+                    // Should we check the connection table?
+                    if v.0.protocol_type().is_connection_oriented() {
+                        // Look the connection up in the connection manager and see if it's still there
+                        connection_manager.get_connection(v.0).is_some()
+                    } else {
+                        // If this is not connection oriented, then we check our last seen time
+                        // to see if this mapping has expired (beyond our timeout)
+                        let cur_ts = get_aligned_timestamp();
+                        (v.1 + TimestampDuration::new(CONNECTIONLESS_TIMEOUT_SECS as u64 * 1_000_000u64)) >= cur_ts
+                    };
+
+                if alive {
                     Some(v.clone())
                 } else {
                     None
@@ -388,7 +455,7 @@ impl BucketEntryInner {
         self.min_max_version
     }
 
-    pub fn state(&self, cur_ts: u64) -> BucketEntryState {
+    pub fn state(&self, cur_ts: Timestamp) -> BucketEntryState {
         if self.check_reliable(cur_ts) {
             BucketEntryState::Reliable
         } else if self.check_dead(cur_ts) {
@@ -427,21 +494,29 @@ impl BucketEntryInner {
         }
     }
 
-    pub fn set_seen_our_node_info(&mut self, routing_domain: RoutingDomain, seen: bool) {
+    pub fn set_our_node_info_ts(&mut self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
         match routing_domain {
             RoutingDomain::LocalNetwork => {
-                self.local_network.seen_our_node_info = seen;
+                self.local_network.last_seen_our_node_info_ts = seen_ts;
             }
             RoutingDomain::PublicInternet => {
-                self.public_internet.seen_our_node_info = seen;
+                self.public_internet.last_seen_our_node_info_ts = seen_ts;
             }
         }
     }
 
-    pub fn has_seen_our_node_info(&self, routing_domain: RoutingDomain) -> bool {
+    pub fn has_seen_our_node_info_ts(
+        &self,
+        routing_domain: RoutingDomain,
+        our_node_info_ts: Timestamp,
+    ) -> bool {
         match routing_domain {
-            RoutingDomain::LocalNetwork => self.local_network.seen_our_node_info,
+            RoutingDomain::LocalNetwork => {
-            RoutingDomain::PublicInternet => self.public_internet.seen_our_node_info,
+                our_node_info_ts == self.local_network.last_seen_our_node_info_ts
+            }
+            RoutingDomain::PublicInternet => {
+                our_node_info_ts == self.public_internet.last_seen_our_node_info_ts
+            }
         }
     }
 
@@ -455,7 +530,7 @@ impl BucketEntryInner {
 
     ///// stats methods
     // called every ROLLING_TRANSFERS_INTERVAL_SECS seconds
-    pub(super) fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) {
+    pub(super) fn roll_transfers(&mut self, last_ts: Timestamp, cur_ts: Timestamp) {
         self.transfer_stats_accounting.roll_transfers(
             last_ts,
             cur_ts,
@@ -464,12 +539,12 @@ impl BucketEntryInner {
     }
 
     // Called for every round trip packet we receive
-    fn record_latency(&mut self, latency: u64) {
+    fn record_latency(&mut self, latency: TimestampDuration) {
         self.peer_stats.latency = Some(self.latency_stats_accounting.record_latency(latency));
     }
 
     ///// state machine handling
-    pub(super) fn check_reliable(&self, cur_ts: u64) -> bool {
+    pub(super) fn check_reliable(&self, cur_ts: Timestamp) -> bool {
         // If we have had any failures to send, this is not reliable
         if self.peer_stats.rpc_stats.failed_to_send > 0 {
             return false;
@@ -479,11 +554,11 @@ impl BucketEntryInner {
         match self.peer_stats.rpc_stats.first_consecutive_seen_ts {
             None => false,
             Some(ts) => {
-                cur_ts.saturating_sub(ts) >= (UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64)
+                cur_ts.saturating_sub(ts) >= TimestampDuration::new(UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64)
             }
         }
     }
-    pub(super) fn check_dead(&self, cur_ts: u64) -> bool {
+    pub(super) fn check_dead(&self, cur_ts: Timestamp) -> bool {
         // If we have failured to send NEVER_REACHED_PING_COUNT times in a row, the node is dead
         if self.peer_stats.rpc_stats.failed_to_send >= NEVER_REACHED_PING_COUNT {
             return true;
@@ -494,20 +569,20 @@ impl BucketEntryInner {
         match self.peer_stats.rpc_stats.last_seen_ts {
             None => self.peer_stats.rpc_stats.recent_lost_answers < NEVER_REACHED_PING_COUNT,
             Some(ts) => {
-                cur_ts.saturating_sub(ts) >= (UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64)
+                cur_ts.saturating_sub(ts) >= TimestampDuration::new(UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64)
             }
         }
     }
 
     /// Return the last time we either saw a node, or asked it a question
-    fn latest_contact_time(&self) -> Option<u64> {
+    fn latest_contact_time(&self) -> Option<Timestamp> {
         self.peer_stats
             .rpc_stats
             .last_seen_ts
-            .max(self.peer_stats.rpc_stats.last_question)
+            .max(self.peer_stats.rpc_stats.last_question_ts)
     }
 
-    fn needs_constant_ping(&self, cur_ts: u64, interval: u64) -> bool {
+    fn needs_constant_ping(&self, cur_ts: Timestamp, interval_us: TimestampDuration) -> bool {
         // If we have not either seen the node in the last 'interval' then we should ping it
         let latest_contact_time = self.latest_contact_time();
 
@@ -515,20 +590,20 @@ impl BucketEntryInner {
             None => true,
             Some(latest_contact_time) => {
                 // If we haven't done anything with this node in 'interval' seconds
-                cur_ts.saturating_sub(latest_contact_time) >= (interval * 1000000u64)
+                cur_ts.saturating_sub(latest_contact_time) >= interval_us
             }
         }
     }
 
     // Check if this node needs a ping right now to validate it is still reachable
-    pub(super) fn needs_ping(&self, cur_ts: u64, needs_keepalive: bool) -> bool {
+    pub(super) fn needs_ping(&self, cur_ts: Timestamp, needs_keepalive: bool) -> bool {
         // See which ping pattern we are to use
         let state = self.state(cur_ts);
 
         // If this entry needs a keepalive (like a relay node),
         // then we should ping it regularly to keep our association alive
         if needs_keepalive {
-            return self.needs_constant_ping(cur_ts, KEEPALIVE_PING_INTERVAL_SECS as u64);
+            return self.needs_constant_ping(cur_ts, TimestampDuration::new(KEEPALIVE_PING_INTERVAL_SECS as u64 * 1000000u64));
         }
 
         // If we don't have node status for this node, then we should ping it to get some node status
@@ -561,8 +636,8 @@ impl BucketEntryInner {
                             latest_contact_time.saturating_sub(start_of_reliable_time);
 
                         retry_falloff_log(
-                            reliable_last,
+                            reliable_last.as_u64(),
-                            reliable_cur,
+                            reliable_cur.as_u64(),
                             RELIABLE_PING_INTERVAL_START_SECS as u64 * 1_000_000u64,
                             RELIABLE_PING_INTERVAL_MAX_SECS as u64 * 1_000_000u64,
                             RELIABLE_PING_INTERVAL_MULTIPLIER,
@@ -572,13 +647,13 @@ impl BucketEntryInner {
             }
             BucketEntryState::Unreliable => {
                 // If we are in an unreliable state, we need a ping every UNRELIABLE_PING_INTERVAL_SECS seconds
-                self.needs_constant_ping(cur_ts, UNRELIABLE_PING_INTERVAL_SECS as u64)
+                self.needs_constant_ping(cur_ts, TimestampDuration::new(UNRELIABLE_PING_INTERVAL_SECS as u64 * 1000000u64))
             }
             BucketEntryState::Dead => false,
         }
     }
 
-    pub(super) fn touch_last_seen(&mut self, ts: u64) {
+    pub(super) fn touch_last_seen(&mut self, ts: Timestamp) {
         // Mark the node as seen
         if self
             .peer_stats
@@ -592,13 +667,13 @@ impl BucketEntryInner {
         self.peer_stats.rpc_stats.last_seen_ts = Some(ts);
     }
 
-    pub(super) fn _state_debug_info(&self, cur_ts: u64) -> String {
+    pub(super) fn _state_debug_info(&self, cur_ts: Timestamp) -> String {
         let first_consecutive_seen_ts = if let Some(first_consecutive_seen_ts) =
             self.peer_stats.rpc_stats.first_consecutive_seen_ts
         {
             format!(
                 "{}s ago",
-                timestamp_to_secs(cur_ts.saturating_sub(first_consecutive_seen_ts))
+                timestamp_to_secs(cur_ts.saturating_sub(first_consecutive_seen_ts).as_u64())
             )
         } else {
             "never".to_owned()
@@ -606,7 +681,7 @@ impl BucketEntryInner {
         let last_seen_ts_str = if let Some(last_seen_ts) = self.peer_stats.rpc_stats.last_seen_ts {
             format!(
                 "{}s ago",
-                timestamp_to_secs(cur_ts.saturating_sub(last_seen_ts))
+                timestamp_to_secs(cur_ts.saturating_sub(last_seen_ts).as_u64())
             )
         } else {
             "never".to_owned()
@@ -623,30 +698,30 @@ impl BucketEntryInner {
     ////////////////////////////////////////////////////////////////
     /// Called when rpc processor things happen
 
-    pub(super) fn question_sent(&mut self, ts: u64, bytes: u64, expects_answer: bool) {
+    pub(super) fn question_sent(&mut self, ts: Timestamp, bytes: ByteCount, expects_answer: bool) {
         self.transfer_stats_accounting.add_up(bytes);
         self.peer_stats.rpc_stats.messages_sent += 1;
         self.peer_stats.rpc_stats.failed_to_send = 0;
         if expects_answer {
             self.peer_stats.rpc_stats.questions_in_flight += 1;
-            self.peer_stats.rpc_stats.last_question = Some(ts);
+            self.peer_stats.rpc_stats.last_question_ts = Some(ts);
         }
     }
-    pub(super) fn question_rcvd(&mut self, ts: u64, bytes: u64) {
+    pub(super) fn question_rcvd(&mut self, ts: Timestamp, bytes: ByteCount) {
         self.transfer_stats_accounting.add_down(bytes);
         self.peer_stats.rpc_stats.messages_rcvd += 1;
         self.touch_last_seen(ts);
     }
-    pub(super) fn answer_sent(&mut self, bytes: u64) {
+    pub(super) fn answer_sent(&mut self, bytes: ByteCount) {
         self.transfer_stats_accounting.add_up(bytes);
         self.peer_stats.rpc_stats.messages_sent += 1;
         self.peer_stats.rpc_stats.failed_to_send = 0;
     }
-    pub(super) fn answer_rcvd(&mut self, send_ts: u64, recv_ts: u64, bytes: u64) {
+    pub(super) fn answer_rcvd(&mut self, send_ts: Timestamp, recv_ts: Timestamp, bytes: ByteCount) {
         self.transfer_stats_accounting.add_down(bytes);
         self.peer_stats.rpc_stats.messages_rcvd += 1;
         self.peer_stats.rpc_stats.questions_in_flight -= 1;
-        self.record_latency(recv_ts - send_ts);
+        self.record_latency(recv_ts.saturating_sub(send_ts));
         self.touch_last_seen(recv_ts);
         self.peer_stats.rpc_stats.recent_lost_answers = 0;
     }
@@ -655,9 +730,9 @@ impl BucketEntryInner {
         self.peer_stats.rpc_stats.questions_in_flight -= 1;
         self.peer_stats.rpc_stats.recent_lost_answers += 1;
     }
-    pub(super) fn failed_to_send(&mut self, ts: u64, expects_answer: bool) {
+    pub(super) fn failed_to_send(&mut self, ts: Timestamp, expects_answer: bool) {
         if expects_answer {
-            self.peer_stats.rpc_stats.last_question = Some(ts);
+            self.peer_stats.rpc_stats.last_question_ts = Some(ts);
         }
         self.peer_stats.rpc_stats.failed_to_send += 1;
         self.peer_stats.rpc_stats.first_consecutive_seen_ts = None;
@@ -672,7 +747,7 @@ pub struct BucketEntry {
 
 impl BucketEntry {
     pub(super) fn new() -> Self {
-        let now = intf::get_timestamp();
+        let now = get_aligned_timestamp();
         Self {
             ref_count: AtomicU32::new(0),
             inner: RwLock::new(BucketEntryInner {
@@ -680,12 +755,12 @@ impl BucketEntry {
                 updated_since_last_network_change: false,
                 last_connections: BTreeMap::new(),
                 local_network: BucketEntryLocalNetwork {
-                    seen_our_node_info: false,
+                    last_seen_our_node_info_ts: Timestamp::new(0u64),
                     signed_node_info: None,
                     node_status: None,
                 },
                 public_internet: BucketEntryPublicInternet {
-                    seen_our_node_info: false,
+                    last_seen_our_node_info_ts: Timestamp::new(0u64),
                     signed_node_info: None,
                     node_status: None,
                 },

veilid-core/src/routing_table/debug.rs

@@ -1,4 +1,5 @@
 use super::*;
+use routing_table::tasks::bootstrap::BOOTSTRAP_TXT_VERSION;
 
 impl RoutingTable {
     pub(crate) fn debug_info_nodeinfo(&self) -> String {
@@ -103,7 +104,7 @@ impl RoutingTable {
     pub(crate) fn debug_info_entries(&self, limit: usize, min_state: BucketEntryState) -> String {
         let inner = self.inner.read();
         let inner = &*inner;
-        let cur_ts = intf::get_timestamp();
+        let cur_ts = get_aligned_timestamp();
 
         let mut out = String::new();
 
@@ -163,7 +164,7 @@ impl RoutingTable {
     pub(crate) fn debug_info_buckets(&self, min_state: BucketEntryState) -> String {
         let inner = self.inner.read();
         let inner = &*inner;
-        let cur_ts = intf::get_timestamp();
+        let cur_ts = get_aligned_timestamp();
 
         let mut out = String::new();
         const COLS: usize = 16;