initial import of main veilid core

veilid-core/src/intf/native/block_store.rs

@@ -0,0 +1,2 @@
+//use crate::intf::*;
+//use crate::xx::*;

veilid-core/src/intf/native/mod.rs

@@ -0,0 +1,12 @@
+mod block_store;
+mod network;
+mod protected_store;
+mod system;
+pub mod table_store;
+pub mod utils;
+
+pub use block_store::*;
+pub use network::*;
+pub use protected_store::*;
+pub use system::*;
+pub use table_store::*;

veilid-core/src/intf/native/network/listener_state.rs

@@ -0,0 +1,54 @@
+use crate::intf::*;
+use crate::network_manager::*;
+use utils::async_peek_stream::*;
+
+use async_std::net::*;
+use async_tls::TlsAcceptor;
+
+pub trait TcpProtocolHandler: TcpProtocolHandlerClone + Send + Sync {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SendPinBoxFuture<Result<bool, ()>>;
+}
+
+pub trait TcpProtocolHandlerClone {
+    fn clone_box(&self) -> Box<dyn TcpProtocolHandler>;
+}
+
+impl<T> TcpProtocolHandlerClone for T
+where
+    T: 'static + TcpProtocolHandler + Clone,
+{
+    fn clone_box(&self) -> Box<dyn TcpProtocolHandler> {
+        Box::new(self.clone())
+    }
+}
+impl Clone for Box<dyn TcpProtocolHandler> {
+    fn clone(&self) -> Box<dyn TcpProtocolHandler> {
+        self.clone_box()
+    }
+}
+
+pub type NewTcpProtocolHandler =
+    dyn Fn(NetworkManager, bool, SocketAddr) -> Box<dyn TcpProtocolHandler> + Send;
+
+/////////////////////////////////////////////////////////////////
+
+#[derive(Clone)]
+pub struct ListenerState {
+    pub protocol_handlers: Vec<Box<dyn TcpProtocolHandler + 'static>>,
+    pub tls_protocol_handlers: Vec<Box<dyn TcpProtocolHandler + 'static>>,
+    pub tls_acceptor: Option<TlsAcceptor>,
+}
+
+impl ListenerState {
+    pub fn new() -> Self {
+        Self {
+            protocol_handlers: Vec::new(),
+            tls_protocol_handlers: Vec::new(),
+            tls_acceptor: None,
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/mod.rs

@@ -0,0 +1,63 @@
+pub mod tcp;
+pub mod udp;
+pub mod wrtc;
+pub mod ws;
+
+use super::listener_state::*;
+use crate::veilid_api::ProtocolType;
+use crate::xx::*;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DummyNetworkConnection {}
+
+impl DummyNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        ProtocolType::UDP
+    }
+    pub fn send(&self, _message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        Box::pin(async { Ok(()) })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        Box::pin(async { Ok(Vec::new()) })
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum NetworkConnection {
+    Dummy(DummyNetworkConnection),
+    RawTcp(tcp::RawTcpNetworkConnection),
+    WSAccepted(ws::WebSocketNetworkConnectionAccepted),
+    WS(ws::WebsocketNetworkConnectionWS),
+    WSS(ws::WebsocketNetworkConnectionWSS),
+    //WebRTC(wrtc::WebRTCNetworkConnection),
+}
+
+impl NetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        match self {
+            Self::Dummy(d) => d.protocol_type(),
+            Self::RawTcp(t) => t.protocol_type(),
+            Self::WSAccepted(w) => w.protocol_type(),
+            Self::WS(w) => w.protocol_type(),
+            Self::WSS(w) => w.protocol_type(),
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        match self {
+            Self::Dummy(d) => d.send(message),
+            Self::RawTcp(t) => t.send(message),
+            Self::WSAccepted(w) => w.send(message),
+            Self::WS(w) => w.send(message),
+            Self::WSS(w) => w.send(message),
+        }
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        match self {
+            Self::Dummy(d) => d.recv(),
+            Self::RawTcp(t) => t.recv(),
+            Self::WSAccepted(w) => w.recv(),
+            Self::WS(w) => w.recv(),
+            Self::WSS(w) => w.recv(),
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/tcp.rs

@@ -0,0 +1,234 @@
+use super::*;
+use crate::intf::native::utils::async_peek_stream::*;
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::net::*;
+use async_std::prelude::*;
+use async_std::sync::Mutex as AsyncMutex;
+use socket2::{Domain, Protocol, Socket, Type};
+use std::fmt;
+
+struct RawTcpNetworkConnectionInner {
+    stream: AsyncPeekStream,
+}
+
+#[derive(Clone)]
+pub struct RawTcpNetworkConnection {
+    inner: Arc<AsyncMutex<RawTcpNetworkConnectionInner>>,
+}
+
+impl fmt::Debug for RawTcpNetworkConnection {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", std::any::type_name::<Self>())
+    }
+}
+
+impl PartialEq for RawTcpNetworkConnection {
+    fn eq(&self, other: &Self) -> bool {
+        Arc::as_ptr(&self.inner) == Arc::as_ptr(&other.inner)
+    }
+}
+
+impl Eq for RawTcpNetworkConnection {}
+
+impl RawTcpNetworkConnection {
+    fn new_inner(stream: AsyncPeekStream) -> RawTcpNetworkConnectionInner {
+        RawTcpNetworkConnectionInner { stream: stream }
+    }
+
+    pub fn new(stream: AsyncPeekStream) -> Self {
+        Self {
+            inner: Arc::new(AsyncMutex::new(Self::new_inner(stream))),
+        }
+    }
+}
+
+impl RawTcpNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        ProtocolType::TCP
+    }
+
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            if message.len() > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+            let len = message.len() as u16;
+            let header = [b'V', b'L', len as u8, (len >> 8) as u8];
+
+            let mut inner = inner.lock().await;
+            inner.stream.write_all(&header).await.map_err(drop)?;
+            inner.stream.write_all(&message).await.map_err(drop)
+        })
+    }
+
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            let mut header = [0u8; 4];
+            let mut inner = inner.lock().await;
+
+            inner.stream.read_exact(&mut header).await.map_err(drop)?;
+            if header[0] != b'V' || header[1] != b'L' {
+                return Err(());
+            }
+            let len = ((header[3] as usize) << 8) | (header[2] as usize);
+            if len > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+
+            let mut out: Vec<u8> = Vec::with_capacity(len);
+            out.resize(len, 0u8);
+            inner.stream.read_exact(&mut out).await.map_err(drop)?;
+            Ok(out)
+        })
+    }
+}
+
+///////////////////////////////////////////////////////////
+///
+
+struct RawTcpProtocolHandlerInner {
+    network_manager: NetworkManager,
+    local_address: SocketAddr,
+}
+
+#[derive(Clone)]
+pub struct RawTcpProtocolHandler
+where
+    Self: TcpProtocolHandler,
+{
+    inner: Arc<Mutex<RawTcpProtocolHandlerInner>>,
+}
+
+impl RawTcpProtocolHandler {
+    fn new_inner(
+        network_manager: NetworkManager,
+        local_address: SocketAddr,
+    ) -> RawTcpProtocolHandlerInner {
+        RawTcpProtocolHandlerInner {
+            network_manager: network_manager,
+            local_address: local_address,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager, local_address: SocketAddr) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager, local_address))),
+        }
+    }
+
+    pub async fn on_accept_async(
+        self,
+        stream: AsyncPeekStream,
+        socket_addr: SocketAddr,
+    ) -> Result<bool, ()> {
+        let mut peekbuf: [u8; PEEK_DETECT_LEN] = [0u8; PEEK_DETECT_LEN];
+        let peeklen = stream.peek(&mut peekbuf).await.map_err(drop)?;
+        assert_eq!(peeklen, PEEK_DETECT_LEN);
+
+        let conn = NetworkConnection::RawTcp(RawTcpNetworkConnection::new(stream));
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(socket_addr),
+            socket_addr.port(),
+            ProtocolType::TCP,
+        );
+        let (network_manager, local_address) = {
+            let inner = self.inner.lock();
+            (inner.network_manager.clone(), inner.local_address.clone())
+        };
+        network_manager
+            .on_new_connection(ConnectionDescriptor::new(peer_addr, local_address), conn)
+            .await?;
+        Ok(true)
+    }
+
+    pub async fn connect(
+        network_manager: NetworkManager,
+        preferred_local_address: Option<SocketAddr>,
+        remote_socket_addr: SocketAddr,
+    ) -> Result<NetworkConnection, ()> {
+        // Make a low level socket that can connect to the remote socket address
+        // and attempt to reuse the local address that our listening socket uses
+        // for hole-punch compatibility
+        let domain = Domain::for_address(remote_socket_addr);
+        let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).map_err(drop)?;
+        if let Err(e) = socket.set_linger(None) {
+            warn!("Couldn't set TCP linger: {}", e);
+        }
+        if let Err(e) = socket.set_nodelay(true) {
+            warn!("Couldn't set TCP nodelay: {}", e);
+        }
+        if let Err(e) = socket.set_reuse_address(true) {
+            warn!("Couldn't set reuse address: {}", e);
+        }
+        cfg_if! {
+            if #[cfg(unix)] {
+                if let Err(e) = socket.set_reuse_port(true) {
+                    warn!("Couldn't set reuse port: {}", e);
+                }
+            }
+        }
+
+        // Try to bind it to the preferred local address
+        if let Some(some_local_addr) = preferred_local_address {
+            let socket2_addr = socket2::SockAddr::from(some_local_addr);
+            if let Err(e) = socket.bind(&socket2_addr) {
+                warn!("failed to bind TCP socket: {}", e);
+            }
+        }
+
+        // Connect to the remote address
+        let remote_socket2_addr = socket2::SockAddr::from(remote_socket_addr);
+        socket.connect(&remote_socket2_addr).map_err(drop)?;
+        let std_stream: std::net::TcpStream = socket.into();
+        let ts = TcpStream::from(std_stream);
+
+        // See what local address we ended up with and turn this into a stream
+        let local_address = ts.local_addr().map_err(drop)?.clone();
+        let ps = AsyncPeekStream::new(ts);
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(remote_socket_addr),
+            remote_socket_addr.port(),
+            ProtocolType::TCP,
+        );
+
+        // Wrap the stream in a network connection and register it
+        let conn = NetworkConnection::RawTcp(RawTcpNetworkConnection::new(ps));
+        network_manager
+            .on_new_connection(
+                ConnectionDescriptor::new(peer_addr, local_address),
+                conn.clone(),
+            )
+            .await?;
+        Ok(conn)
+    }
+
+    pub async fn send_unbound_message(data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+        trace!(
+            "sending unbound message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        let mut stream = TcpStream::connect(socket_addr).await.map_err(drop)?;
+        stream.write_all(&data).await.map_err(drop)
+    }
+}
+
+impl TcpProtocolHandler for RawTcpProtocolHandler {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SendPinBoxFuture<Result<bool, ()>> {
+        Box::pin(self.clone().on_accept_async(stream, peer_addr))
+    }
+}

veilid-core/src/intf/native/network/protocol/udp.rs

@@ -0,0 +1,109 @@
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::net::*;
+
+struct RawUdpProtocolHandlerInner {
+    network_manager: NetworkManager,
+    socket: Arc<UdpSocket>,
+}
+
+#[derive(Clone)]
+pub struct RawUdpProtocolHandler {
+    inner: Arc<Mutex<RawUdpProtocolHandlerInner>>,
+}
+
+impl RawUdpProtocolHandler {
+    fn new_inner(
+        network_manager: NetworkManager,
+        socket: Arc<UdpSocket>,
+    ) -> RawUdpProtocolHandlerInner {
+        RawUdpProtocolHandlerInner {
+            network_manager: network_manager,
+            socket: socket,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager, socket: Arc<UdpSocket>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager, socket))),
+        }
+    }
+
+    pub async fn on_message(&self, data: &[u8], remote_addr: SocketAddr) -> Result<bool, ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+
+        trace!(
+            "receiving message of length {} from {}",
+            data.len(),
+            remote_addr
+        );
+
+        // Process envelope
+        let (network_manager, socket) = {
+            let inner = self.inner.lock();
+            (inner.network_manager.clone(), inner.socket.clone())
+        };
+
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(remote_addr),
+            remote_addr.port(),
+            ProtocolType::UDP,
+        );
+        let local_socket_addr = socket.local_addr().map_err(drop)?;
+        network_manager
+            .on_recv_envelope(
+                data,
+                &ConnectionDescriptor::new(peer_addr, local_socket_addr),
+            )
+            .await
+    }
+
+    pub async fn send_message(&self, data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+
+        trace!(
+            "sending message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        let socket = self.inner.lock().socket.clone();
+        let len = socket.send_to(&data, socket_addr).await.map_err(drop)?;
+        if len != data.len() {
+            Err(())
+        } else {
+            Ok(())
+        }
+    }
+
+    pub async fn send_unbound_message(data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+        trace!(
+            "sending unbound message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        // get local wildcard address for bind
+        let local_socket_addr = match socket_addr {
+            SocketAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
+            SocketAddr::V6(_) => {
+                SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), 0)
+            }
+        };
+        let socket = UdpSocket::bind(local_socket_addr).await.map_err(drop)?;
+        let len = socket.send_to(&data, socket_addr).await.map_err(drop)?;
+        if len != data.len() {
+            Err(())
+        } else {
+            Ok(())
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/ws.rs

@@ -0,0 +1,302 @@
+use super::*;
+use crate::intf::native::utils::async_peek_stream::*;
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::io;
+use async_std::net::*;
+use async_std::sync::Mutex as AsyncMutex;
+use async_tls::TlsConnector;
+use async_tungstenite::tungstenite::protocol::Message;
+use async_tungstenite::{accept_async, client_async, WebSocketStream};
+use futures_util::sink::SinkExt;
+use futures_util::stream::StreamExt;
+use std::fmt;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type WebSocketNetworkConnectionAccepted = WebsocketNetworkConnection<AsyncPeekStream>;
+pub type WebsocketNetworkConnectionWSS =
+    WebsocketNetworkConnection<async_tls::client::TlsStream<async_std::net::TcpStream>>;
+pub type WebsocketNetworkConnectionWS = WebsocketNetworkConnection<async_std::net::TcpStream>;
+
+struct WebSocketNetworkConnectionInner<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    ws_stream: WebSocketStream<T>,
+}
+
+pub struct WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    tls: bool,
+    inner: Arc<AsyncMutex<WebSocketNetworkConnectionInner<T>>>,
+}
+
+impl<T> Clone for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn clone(&self) -> Self {
+        Self {
+            tls: self.tls,
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<T> fmt::Debug for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", std::any::type_name::<Self>())
+    }
+}
+
+impl<T> PartialEq for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.tls == other.tls && Arc::as_ptr(&self.inner) == Arc::as_ptr(&other.inner)
+    }
+}
+
+impl<T> Eq for WebsocketNetworkConnection<T> where T: io::Read + io::Write + Send + Unpin + 'static {}
+
+impl<T> WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    pub fn new(tls: bool, ws_stream: WebSocketStream<T>) -> Self {
+        Self {
+            tls: tls,
+            inner: Arc::new(AsyncMutex::new(WebSocketNetworkConnectionInner {
+                ws_stream: ws_stream,
+            })),
+        }
+    }
+
+    pub fn protocol_type(&self) -> ProtocolType {
+        if self.tls {
+            ProtocolType::WSS
+        } else {
+            ProtocolType::WS
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            if message.len() > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+            let mut inner = inner.lock().await;
+            inner
+                .ws_stream
+                .send(Message::binary(message))
+                .await
+                .map_err(drop)
+        })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            let mut inner = inner.lock().await;
+
+            let out = match inner.ws_stream.next().await {
+                Some(Ok(Message::Binary(v))) => v,
+                _ => {
+                    trace!("websocket recv failed");
+                    return Err(());
+                }
+            };
+            if out.len() > MAX_MESSAGE_SIZE {
+                Err(())
+            } else {
+                Ok(out)
+            }
+        })
+    }
+}
+
+///////////////////////////////////////////////////////////
+///
+struct WebsocketProtocolHandlerInner {
+    tls: bool,
+    network_manager: NetworkManager,
+    local_address: SocketAddr,
+    request_path: Vec<u8>,
+    connection_initial_timeout: u64,
+}
+
+#[derive(Clone)]
+pub struct WebsocketProtocolHandler
+where
+    Self: TcpProtocolHandler,
+{
+    inner: Arc<WebsocketProtocolHandlerInner>,
+}
+impl WebsocketProtocolHandler {
+    pub fn new(network_manager: NetworkManager, tls: bool, local_address: SocketAddr) -> Self {
+        let config = network_manager.config();
+        let c = config.get();
+        let path = format!("GET {}", c.network.protocol.ws.path.trim_end_matches('/'));
+        let connection_initial_timeout = if tls {
+            c.network.tls.connection_initial_timeout
+        } else {
+            c.network.connection_initial_timeout
+        };
+
+        let inner = WebsocketProtocolHandlerInner {
+            tls: tls,
+            network_manager: network_manager,
+            local_address: local_address,
+            request_path: path.as_bytes().to_vec(),
+            connection_initial_timeout: connection_initial_timeout,
+        };
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    pub async fn on_accept_async(
+        self,
+        ps: AsyncPeekStream,
+        socket_addr: SocketAddr,
+    ) -> Result<bool, ()> {
+        let request_path_len = self.inner.request_path.len() + 2;
+        let mut peekbuf: Vec<u8> = Vec::with_capacity(request_path_len);
+        peekbuf.resize(request_path_len, 0u8);
+        match io::timeout(
+            Duration::from_micros(self.inner.connection_initial_timeout),
+            ps.peek_exact(&mut peekbuf),
+        )
+        .await
+        {
+            Ok(_) => (),
+            Err(e) => {
+                trace!("failed to peek stream: {:?}", e);
+                return Err(());
+            }
+        }
+        // Check for websocket path
+        let matches_path = &peekbuf[0..request_path_len - 2] == self.inner.request_path.as_slice()
+            && (peekbuf[request_path_len - 2] == b' '
+                || (peekbuf[request_path_len - 2] == b'/'
+                    && peekbuf[request_path_len - 1] == b' '));
+
+        if !matches_path {
+            trace!("not websocket");
+            return Ok(false);
+        }
+        trace!("found websocket");
+
+        let ws_stream = match accept_async(ps).await {
+            Ok(s) => s,
+            Err(e) => {
+                trace!("failed websockets handshake: {:?}", e);
+                return Err(());
+            }
+        };
+
+        // Wrap the websocket in a NetworkConnection and register it
+        let protocol_type = if self.inner.tls {
+            ProtocolType::WSS
+        } else {
+            ProtocolType::WS
+        };
+
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(socket_addr),
+            socket_addr.port(),
+            protocol_type,
+        );
+
+        let conn = NetworkConnection::WSAccepted(WebsocketNetworkConnection::new(
+            self.inner.tls,
+            ws_stream,
+        ));
+        self.inner
+            .network_manager
+            .clone()
+            .on_new_connection(
+                ConnectionDescriptor::new(peer_addr, self.inner.local_address.clone()),
+                conn,
+            )
+            .await?;
+        Ok(true)
+    }
+
+    pub async fn connect(
+        network_manager: NetworkManager,
+        dial_info: &DialInfo,
+    ) -> Result<NetworkConnection, ()> {
+        let (tls, request, domain, port, protocol_type) = match &dial_info {
+            DialInfo::WS(di) => (
+                false,
+                di.path.clone(),
+                di.fqdn.clone(),
+                di.port,
+                ProtocolType::WS,
+            ),
+            DialInfo::WSS(di) => (
+                true,
+                di.path.clone(),
+                di.fqdn.clone(),
+                di.port,
+                ProtocolType::WSS,
+            ),
+            _ => panic!("invalid dialinfo for WS/WSS protocol"),
+        };
+
+        let tcp_stream = TcpStream::connect(format!("{}:{}", &domain, &port))
+            .await
+            .map_err(drop)?;
+        let local_addr = tcp_stream.local_addr().map_err(drop)?;
+        let peer_socket_addr = tcp_stream.peer_addr().map_err(drop)?;
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(peer_socket_addr),
+            peer_socket_addr.port(),
+            protocol_type,
+        );
+
+        if tls {
+            let connector = TlsConnector::default();
+            let tls_stream = connector.connect(domain, tcp_stream).await.map_err(drop)?;
+            let (ws_stream, _response) = client_async(request, tls_stream).await.map_err(drop)?;
+            let conn = NetworkConnection::WSS(WebsocketNetworkConnection::new(tls, ws_stream));
+            network_manager
+                .on_new_connection(
+                    ConnectionDescriptor::new(peer_addr, local_addr),
+                    conn.clone(),
+                )
+                .await?;
+            Ok(conn)
+        } else {
+            let (ws_stream, _response) = client_async(request, tcp_stream).await.map_err(drop)?;
+            let conn = NetworkConnection::WS(WebsocketNetworkConnection::new(tls, ws_stream));
+            network_manager
+                .on_new_connection(
+                    ConnectionDescriptor::new(peer_addr, local_addr),
+                    conn.clone(),
+                )
+                .await?;
+            Ok(conn)
+        }
+    }
+}
+
+impl TcpProtocolHandler for WebsocketProtocolHandler {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SystemPinBoxFuture<Result<bool, ()>> {
+        Box::pin(self.clone().on_accept_async(stream, peer_addr))
+    }
+}

veilid-core/src/intf/native/network/public_dialinfo_discovery.rs

@@ -0,0 +1,222 @@
+use super::*;
+
+use crate::intf::*;
+use crate::network_manager::*;
+use crate::routing_table::*;
+use crate::*;
+
+use async_std::net::*;
+
+impl Network {
+    // Ask for a public address check from a particular noderef
+    async fn request_public_address(&self, node_ref: NodeRef) -> Option<SocketAddr> {
+        let routing_table = self.routing_table();
+        let rpc = routing_table.rpc_processor();
+        let info_answer = match rpc.rpc_call_info(node_ref.clone()).await {
+            Err(e) => {
+                trace!("failed to get info answer from {:?}: {:?}", node_ref, e);
+                return None;
+            }
+            Ok(ia) => ia,
+        };
+        info_answer.sender_info.socket_address
+    }
+
+    // find fast peers with a particular address type, and ask them to tell us what our external address is
+    async fn discover_external_address(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+        ignore_node: Option<DHTKey>,
+    ) -> Result<(SocketAddr, NodeRef), String> {
+        let routing_table = self.routing_table();
+        let peers = routing_table.get_fast_nodes_of_type(protocol_address_type);
+        if peers.len() == 0 {
+            return Err(format!("no peers of type '{:?}'", protocol_address_type));
+        }
+        for peer in peers {
+            if let Some(ignore_node) = ignore_node {
+                if peer.node_id() == ignore_node {
+                    continue;
+                }
+            }
+            if let Some(sa) = self.request_public_address(peer.clone()).await {
+                return Ok((sa, peer));
+            }
+        }
+        Err("no peers responded with an external address".to_owned())
+    }
+
+    fn discover_local_address(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Result<SocketAddr, String> {
+        let routing_table = self.routing_table();
+
+        match routing_table
+            .get_own_peer_info(PeerScope::Public)
+            .dial_infos
+            .iter()
+            .find_map(|di| {
+                if di.protocol_address_type() == protocol_address_type {
+                    if let Ok(addr) = di.to_socket_addr() {
+                        return Some(addr);
+                    }
+                }
+                None
+            }) {
+            None => Err(format!(
+                "no local address for protocol address type: {:?}",
+                protocol_address_type
+            )),
+            Some(addr) => Ok(addr),
+        }
+    }
+
+    async fn validate_dial_info(
+        &self,
+        node_ref: NodeRef,
+        dial_info: DialInfo,
+        redirect: bool,
+        alternate_port: bool,
+    ) -> bool {
+        let routing_table = self.routing_table();
+        let rpc = routing_table.rpc_processor();
+        match rpc
+            .rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect, alternate_port)
+            .await
+        {
+            Err(e) => {
+                error!(
+                    "failed to send validate_dial_info to {:?}: {:?}",
+                    node_ref, e
+                );
+                false
+            }
+            Ok(val) => val,
+        }
+    }
+
+    async fn try_port_mapping(
+        &self,
+        local_addr: SocketAddr,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Option<SocketAddr> {
+        //xxx
+        None
+    }
+
+    pub async fn update_udpv4_dialinfo_task_routine(self, l: u64, t: u64) -> Result<(), String> {
+        trace!("looking for udpv4 public dial info");
+        let routing_table = self.routing_table();
+
+        // Get our local address
+        let local1 = self.discover_local_address(ProtocolAddressType::UDPv4)?;
+        // Get our external address from some fast node, call it node B
+        let (external1, node_b) = self
+            .discover_external_address(ProtocolAddressType::UDPv4, None)
+            .await?;
+        let external1_dial_info = DialInfo::udp_from_socketaddr(external1);
+
+        // If local1 == external1 then there is no NAT in place
+        if local1 == external1 {
+            // No NAT
+            // Do a validate_dial_info on the external address from a routed node
+            if self
+                .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false)
+                .await
+            {
+                // Add public dial info with Server network class
+                routing_table.register_public_dial_info(
+                    external1_dial_info,
+                    Some(NetworkClass::Server),
+                    DialInfoOrigin::Discovered,
+                );
+            } else {
+                // UDP firewall?
+                warn!("UDP static public dial info not reachable. UDP firewall may be blocking inbound to {:?} for {:?}",external1_dial_info, node_b);
+            }
+        } else {
+            // There is -some NAT-
+            // Attempt a UDP port mapping via all available and enabled mechanisms
+            if let Some(external_mapped) = self
+                .try_port_mapping(local1.clone(), ProtocolAddressType::UDPv4)
+                .await
+            {
+                // Got a port mapping, let's use it
+                let external_mapped_dial_info = DialInfo::udp_from_socketaddr(external_mapped);
+                routing_table.register_public_dial_info(
+                    external_mapped_dial_info,
+                    Some(NetworkClass::Mapped),
+                    DialInfoOrigin::Mapped,
+                );
+            } else {
+                // Port mapping was not possible, let's see what kind of NAT we have
+
+                // Does a redirected dial info validation find us?
+                if self
+                    .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false)
+                    .await
+                {
+                    // Yes, another machine can use the dial info directly, so Full Cone
+                    // Add public dial info with full cone NAT network class
+                    routing_table.register_public_dial_info(
+                        external1_dial_info,
+                        Some(NetworkClass::FullNAT),
+                        DialInfoOrigin::Discovered,
+                    );
+                } else {
+                    // No, we are restricted, determine what kind of restriction
+
+                    // Get our external address from some fast node, that is not node B, call it node D
+                    let (external2, node_d) = self
+                        .discover_external_address(
+                            ProtocolAddressType::UDPv4,
+                            Some(node_b.node_id()),
+                        )
+                        .await?;
+                    // If we have two different external addresses, then this is a symmetric NAT
+                    if external2 != external1 {
+                        // Symmetric NAT is outbound only, no public dial info will work
+                        self.inner.lock().network_class = Some(NetworkClass::OutboundOnly);
+                    } else {
+                        // Address is the same, so it's address or port restricted
+                        let external2_dial_info = DialInfo::udp_from_socketaddr(external2);
+                        // Do a validate_dial_info on the external address from a routed node
+                        if self
+                            .validate_dial_info(
+                                node_d.clone(),
+                                external2_dial_info.clone(),
+                                false,
+                                true,
+                            )
+                            .await
+                        {
+                            // Got a reply from a non-default port, which means we're only address restricted
+                            routing_table.register_public_dial_info(
+                                external1_dial_info,
+                                Some(NetworkClass::AddressRestrictedNAT),
+                                DialInfoOrigin::Discovered,
+                            );
+                        } else {
+                            // Didn't get a reply from a non-default port, which means we are also port restricted
+                            routing_table.register_public_dial_info(
+                                external1_dial_info,
+                                Some(NetworkClass::PortRestrictedNAT),
+                                DialInfoOrigin::Discovered,
+                            );
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn update_tcpv4_dialinfo_task_routine(self, l: u64, t: u64) -> Result<(), String> {
+        trace!("looking for tcpv4 public dial info");
+        // xxx
+        //Err("unimplemented".to_owned())
+        Ok(())
+    }
+}

veilid-core/src/intf/native/protected_store.rs

@@ -0,0 +1,58 @@
+use cfg_if::*;
+use keyring::{Keyring, KeyringError};
+
+fn keyring_name(namespace: &str) -> String {
+    if namespace.len() == 0 {
+        "veilid".to_owned()
+    } else {
+        format!("veilid_{}", namespace)
+    }
+}
+
+fn get_keyring<'a>(krname: &'a str, key: &'a str) -> Keyring<'a> {
+    cfg_if! {
+        if #[cfg(target_os = "android")] {
+            let agopt = super::utils::android::ANDROID_GLOBALS.lock();
+            let ag = agopt.as_ref().unwrap();
+            let vm = ag.vm.attach_current_thread().unwrap().get_java_vm().unwrap(); // cmon jni, no clone for javavm
+            let ctx = ag.ctx.clone();
+            Keyring::new("veilid", krname, key, (vm, ctx))
+        } else {
+            Keyring::new("veilid", krname, key)
+        }
+    }
+}
+
+pub async fn save_user_secret_string(
+    namespace: &str,
+    key: &str,
+    value: &str,
+) -> Result<bool, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    let existed = kr.get_password().is_ok();
+    let _ = kr
+        .set_password(value)
+        .map_err(|e| format!("Failed to save user secret: {}", e).to_owned())?;
+    Ok(existed)
+}
+
+pub async fn load_user_secret_string(namespace: &str, key: &str) -> Result<Option<String>, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    match kr.get_password() {
+        Ok(v) => Ok(Some(v)),
+        Err(KeyringError::NoPasswordFound) => Ok(None),
+        Err(e) => Err(format!("Failed to load user secret: {}", e).to_owned()),
+    }
+}
+
+pub async fn remove_user_secret_string(namespace: &str, key: &str) -> Result<bool, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    match kr.delete_password() {
+        Ok(_) => Ok(true),
+        Err(KeyringError::NoPasswordFound) => Ok(false),
+        Err(e) => Err(format!("Failed to remove user secret: {}", e).to_owned()),
+    }
+}

veilid-core/src/intf/native/system.rs

@@ -0,0 +1,102 @@
+use crate::xx::*;
+pub use async_executors::JoinHandle;
+use async_executors::{AsyncStd, LocalSpawnHandleExt, SpawnHandleExt};
+use rand::prelude::*;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+pub fn get_timestamp() -> u64 {
+    match SystemTime::now().duration_since(UNIX_EPOCH) {
+        Ok(n) => n.as_micros() as u64,
+        Err(_) => panic!("SystemTime before UNIX_EPOCH!"),
+    }
+}
+
+pub fn random_bytes(dest: &mut [u8]) -> Result<(), String> {
+    let mut rng = rand::thread_rng();
+    rng.try_fill_bytes(dest).map_err(|err| format!("{:?}", err))
+}
+
+pub fn get_random_u32() -> u32 {
+    let mut rng = rand::thread_rng();
+    rng.next_u32()
+}
+
+pub fn get_random_u64() -> u64 {
+    let mut rng = rand::thread_rng();
+    rng.next_u64()
+}
+
+pub async fn sleep(millis: u32) {
+    if millis == 0 {
+        async_std::task::yield_now().await;
+    } else {
+        async_std::task::sleep(Duration::from_millis(u64::from(millis))).await;
+    }
+}
+
+pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> JoinHandle<Out>
+where
+    Out: Send + 'static,
+{
+    AsyncStd
+        .spawn_handle(future)
+        .expect("async-std spawn should never error out")
+}
+
+pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> JoinHandle<Out>
+where
+    Out: 'static,
+{
+    AsyncStd
+        .spawn_handle_local(future)
+        .expect("async-std spawn_local should never error out")
+}
+
+pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SystemPinBoxFuture<()>
+where
+    F: Fn() -> FUT + Send + Sync + 'static,
+    FUT: Future<Output = ()> + Send,
+{
+    let e = Eventual::new();
+
+    let ie = e.clone();
+    let jh = spawn(async move {
+        while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
+            callback().await;
+        }
+    });
+
+    Box::pin(async move {
+        e.resolve().await;
+        jh.await;
+    })
+}
+
+pub use async_std::future::TimeoutError;
+
+pub async fn timeout<F, T>(dur_ms: u32, f: F) -> Result<T, TimeoutError>
+where
+    F: Future<Output = T>,
+{
+    async_std::future::timeout(Duration::from_millis(dur_ms as u64), f).await
+}
+
+pub fn get_concurrency() -> u32 {
+    num_cpus::get() as u32
+}
+
+/*
+pub fn async_callback<F, OF, EF, T, E>(fut: F, ok_fn: OF, err_fn: EF)
+where
+    F: Future<Output = Result<T, E>> + Send + 'static,
+    OF: FnOnce(T) + Send + 'static,
+    EF: FnOnce(E) + Send + 'static,
+{
+    spawn(Box::pin(async move {
+        match fut.await {
+            Ok(v) => ok_fn(v),
+            Err(e) => err_fn(e),
+        };
+    }));
+}
+*/

veilid-core/src/intf/native/table_store.rs

@@ -0,0 +1,121 @@
+use crate::intf::table_db::*;
+use crate::intf::*;
+use crate::*;
+use keyvaluedb_sqlite::*;
+use std::path::PathBuf;
+
+struct TableStoreInner {
+    config: VeilidConfig,
+    opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
+}
+
+#[derive(Clone)]
+pub struct TableStore {
+    inner: Arc<Mutex<TableStoreInner>>,
+}
+
+impl TableStore {
+    fn new_inner(config: VeilidConfig) -> TableStoreInner {
+        TableStoreInner {
+            config: config,
+            opened: BTreeMap::new(),
+        }
+    }
+    pub fn new(config: VeilidConfig) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(config))),
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        Ok(())
+    }
+
+    pub async fn terminate(&self) {
+        assert!(
+            self.inner.lock().opened.len() == 0,
+            "all open databases should have been closed"
+        );
+    }
+
+    pub fn on_table_db_drop(&self, table: String) {
+        let mut inner = self.inner.lock();
+        match inner.opened.remove(&table) {
+            Some(_) => (),
+            None => {
+                assert!(false, "should have removed an item");
+            }
+        }
+    }
+
+    fn get_dbpath(inner: &TableStoreInner, table: &str) -> Result<PathBuf, String> {
+        if !table
+            .chars()
+            .all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
+        {
+            return Err(format!("table name '{}' is invalid", table));
+        }
+        let c = inner.config.get();
+        let tablestoredir = c.tablestore.directory.clone();
+        std::fs::create_dir_all(&tablestoredir)
+            .map_err(|e| format!("failed to create tablestore path: {}", e))?;
+
+        let dbpath: PathBuf = [tablestoredir, String::from(table)].iter().collect();
+        Ok(dbpath)
+    }
+
+    fn get_table_name(inner: &TableStoreInner, table: &str) -> Result<String, String> {
+        if !table
+            .chars()
+            .all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
+        {
+            return Err(format!("table name '{}' is invalid", table));
+        }
+        let c = inner.config.get();
+        let namespace = c.namespace.clone();
+        Ok(if namespace.len() == 0 {
+            format!("{}", table)
+        } else {
+            format!("_ns_{}_{}", namespace, table)
+        })
+    }
+
+    pub async fn open(&self, name: &str, column_count: u32) -> Result<TableDB, String> {
+        let mut inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
+            match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) {
+                Some(tdb) => {
+                    return Ok(tdb);
+                }
+                None => {
+                    inner.opened.remove(&table_name);
+                }
+            };
+        }
+
+        let dbpath = Self::get_dbpath(&inner, &table_name)?;
+        let cfg = DatabaseConfig::with_columns(column_count);
+        let db =
+            Database::open(dbpath, cfg).map_err(|e| format!("failed to open tabledb: {}", e))?;
+
+        let table_db = TableDB::new(table_name.clone(), self.clone(), db);
+
+        inner.opened.insert(table_name, table_db.weak_inner());
+
+        Ok(table_db)
+    }
+
+    pub async fn delete(&self, name: &str) -> Result<bool, String> {
+        let inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if inner.opened.contains_key(&table_name) {
+            return Err("Not deleting table that is still opened".to_owned());
+        }
+        let dbpath = Self::get_dbpath(&inner, &table_name)?;
+        let ret = std::fs::remove_file(dbpath).is_ok();
+        Ok(ret)
+    }
+}

veilid-core/src/intf/native/utils/android/android_get_if_addrs.rs

@@ -0,0 +1,192 @@
+use super::*;
+use crate::xx::*;
+pub use if_addrs::{IfAddr, Ifv4Addr, Ifv6Addr, Interface};
+use jni::objects::JValue;
+use std::io;
+
+fn get_netmask_from_prefix_length_v4(out: &mut [u8; 4], mut plen: i16) {
+    for n in 0..4 {
+        out[n] = if plen >= 8 {
+            plen -= 8;
+            255u8
+        } else if plen <= 0 {
+            0u8
+        } else {
+            let v = 255u8 << (8 - plen);
+            plen = 0;
+            v
+        }
+    }
+}
+fn get_netmask_from_prefix_length_v6(out: &mut [u8; 16], mut plen: i16) {
+    for n in 0..16 {
+        out[n] = if plen >= 8 {
+            plen -= 8;
+            255u8
+        } else if plen == 0 {
+            0u8
+        } else {
+            let v = 255u8 << (8 - plen);
+            plen = 0;
+            v
+        }
+    }
+}
+
+fn convert_to_unsigned_4(x: [i8; 4]) -> [u8; 4] {
+    let mut out: [u8; 4] = [0u8; 4];
+    for i in 0..4 {
+        out[i] = x[i] as u8;
+    }
+    out
+}
+
+fn convert_to_unsigned_16(x: [i8; 16]) -> [u8; 16] {
+    let mut out: [u8; 16] = [0u8; 16];
+    for i in 0..16 {
+        out[i] = x[i] as u8;
+    }
+    out
+}
+
+macro_rules! call_method_checked {
+    ($env:expr, $obj:expr, $name:expr, $sig:expr, $args:expr, $kind:ident) => {
+        $env.call_method($obj, $name, $sig, $args)
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("call_method {} {} failed: {}", $name, $sig, e),
+                )
+            })?
+            .$kind()
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?
+    };
+}
+
+pub fn get_if_addrs() -> io::Result<Vec<Interface>> {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+
+    let niclass = env.find_class("java/net/NetworkInterface").unwrap();
+    let intfenum = env
+        .call_static_method(
+            niclass,
+            "getNetworkInterfaces",
+            "()Ljava/util/Enumeration;",
+            &[],
+        )
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let mut out: Vec<Interface> = Vec::new();
+    while call_method_checked!(env, intfenum, "hasMoreElements", "()Z", &[], z) {
+        let intf =
+            call_method_checked!(env, intfenum, "nextElement", "()Ljava/lang/Object;", &[], l);
+
+        let nameobj = call_method_checked!(env, intf, "getName", "()Ljava/lang/String;", &[], l);
+        let name_jstrval = env.get_string(JString::from(nameobj)).unwrap();
+        let name = String::from(name_jstrval.to_string_lossy());
+
+        let intfaddrs = call_method_checked!(
+            env,
+            intf,
+            "getInterfaceAddresses",
+            "()Ljava/util/List;",
+            &[],
+            l
+        );
+        let size = call_method_checked!(env, intfaddrs, "size", "()I", &[], i);
+        for i in 0..size {
+            let intfaddr = call_method_checked!(
+                env,
+                intfaddrs,
+                "get",
+                "(I)Ljava/lang/Object;",
+                &[JValue::Int(i)],
+                l
+            );
+
+            let ia_addr = call_method_checked!(
+                env,
+                intfaddr,
+                "getAddress",
+                "()Ljava/net/InetAddress;",
+                &[],
+                l
+            );
+            let ia_bcst = call_method_checked!(
+                env,
+                intfaddr,
+                "getBroadcast",
+                "()Ljava/net/InetAddress;",
+                &[],
+                l
+            );
+            let ia_plen =
+                call_method_checked!(env, intfaddr, "getNetworkPrefixLength", "()S", &[], s);
+
+            let ia_addr_bytearray =
+                call_method_checked!(env, ia_addr, "getAddress", "()[B", &[], l);
+            let ia_addr_bytearray_len = env.get_array_length(*ia_addr_bytearray).unwrap();
+            let addr: IfAddr;
+            if ia_addr_bytearray_len == 4 {
+                let mut ia_addr_bytes_v4 = [0i8; 4];
+                env.get_byte_array_region(*ia_addr_bytearray, 0, &mut ia_addr_bytes_v4)
+                    .unwrap();
+
+                let broadcast = if !env.is_same_object(ia_bcst, JObject::null()).unwrap() {
+                    let ia_bcst_bytearray =
+                        call_method_checked!(env, ia_bcst, "getAddress", "()[B", &[], l);
+                    let ia_bcst_bytearray_len = env.get_array_length(*ia_bcst_bytearray).unwrap();
+                    if ia_bcst_bytearray_len != 4 {
+                        warn!(
+                            "mismatched inet4 broadcast address length: {}",
+                            ia_bcst_bytearray_len
+                        );
+                        continue;
+                    }
+
+                    let mut ia_bsct_bytes_v4 = [0i8; 4];
+                    env.get_byte_array_region(*ia_bcst_bytearray, 0, &mut ia_bsct_bytes_v4)
+                        .unwrap();
+
+                    Some(Ipv4Addr::from(convert_to_unsigned_4(ia_bsct_bytes_v4)))
+                } else {
+                    None
+                };
+
+                let mut ia_netmask_bytes_v4 = [0u8; 4];
+                get_netmask_from_prefix_length_v4(&mut ia_netmask_bytes_v4, ia_plen);
+                addr = IfAddr::V4(Ifv4Addr {
+                    ip: Ipv4Addr::from(convert_to_unsigned_4(ia_addr_bytes_v4)),
+                    netmask: Ipv4Addr::from(ia_netmask_bytes_v4),
+                    broadcast: broadcast,
+                });
+            } else if ia_addr_bytearray_len == 16 {
+                let mut ia_addr_bytes_v6 = [0i8; 16];
+                env.get_byte_array_region(*ia_addr_bytearray, 0, &mut ia_addr_bytes_v6)
+                    .unwrap();
+
+                let mut ia_netmask_bytes_v6 = [0u8; 16];
+                get_netmask_from_prefix_length_v6(&mut ia_netmask_bytes_v6, ia_plen);
+                addr = IfAddr::V6(Ifv6Addr {
+                    ip: Ipv6Addr::from(convert_to_unsigned_16(ia_addr_bytes_v6)),
+                    netmask: Ipv6Addr::from(ia_netmask_bytes_v6),
+                    broadcast: None,
+                });
+            } else {
+                warn!("weird inet address length: {}", ia_addr_bytearray_len);
+                continue;
+            }
+            let elem = Interface {
+                name: name.clone(),
+                addr: addr,
+            };
+
+            out.push(elem);
+        }
+    }
+    Ok(out)
+}

veilid-core/src/intf/native/utils/android/get_directories.rs

@@ -0,0 +1,42 @@
+use super::*;
+use crate::xx::*;
+
+pub fn get_files_dir() -> String {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+    // context.getFilesDir().getAbsolutePath()
+    let file = env
+        .call_method(ag.ctx.as_obj(), "getFilesDir", "()Ljava/io/File;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+    let path = env
+        .call_method(file, "getAbsolutePath", "()Ljava/lang/String;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let jstrval = env.get_string(JString::from(path)).unwrap();
+    String::from(jstrval.to_string_lossy())
+}
+
+pub fn get_cache_dir() -> String {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+    // context.getCacheDir().getAbsolutePath()
+    let file = env
+        .call_method(ag.ctx.as_obj(), "getCacheDir", "()Ljava/io/File;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+    let path = env
+        .call_method(file, "getAbsolutePath", "()Ljava/lang/String;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let jstrval = env.get_string(JString::from(path)).unwrap();
+    String::from(jstrval.to_string_lossy())
+}

veilid-core/src/intf/native/utils/android/mod.rs

@@ -0,0 +1,66 @@
+mod android_get_if_addrs;
+mod get_directories;
+pub use android_get_if_addrs::*;
+pub use get_directories::*;
+
+use crate::xx::*;
+use android_logger::{Config, FilterBuilder};
+use backtrace::Backtrace;
+use jni::{objects::GlobalRef, objects::JObject, objects::JString, JNIEnv, JavaVM};
+use lazy_static::*;
+use log::*;
+use std::panic;
+
+pub struct AndroidGlobals {
+    pub vm: JavaVM,
+    pub ctx: GlobalRef,
+}
+
+lazy_static! {
+    pub static ref ANDROID_GLOBALS: Arc<Mutex<Option<AndroidGlobals>>> = Arc::new(Mutex::new(None));
+}
+
+pub fn veilid_core_setup_android<'a>(
+    env: JNIEnv<'a>,
+    ctx: JObject<'a>,
+    log_tag: &'a str,
+    log_level: Level,
+) {
+    android_logger::init_once(
+        Config::default()
+            .with_min_level(log_level)
+            .with_tag(log_tag)
+            .with_filter(
+                FilterBuilder::new()
+                    .filter(Some(log_tag), log_level.to_level_filter())
+                    .build(),
+            ),
+    );
+    panic::set_hook(Box::new(|panic_info| {
+        let bt = Backtrace::new();
+        if let Some(location) = panic_info.location() {
+            error!(
+                "panic occurred in file '{}' at line {}",
+                location.file(),
+                location.line(),
+            );
+        } else {
+            error!("panic occurred but can't get location information...");
+        }
+        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
+            error!("panic payload: {:?}", a);
+        } else {
+            error!("no panic payload");
+        }
+        error!("Backtrace:\n{:?}", bt);
+    }));
+
+    *ANDROID_GLOBALS.lock() = Some(AndroidGlobals {
+        vm: env.get_java_vm().unwrap(),
+        ctx: env.new_global_ref(ctx).unwrap(),
+    });
+}

veilid-core/src/intf/native/utils/async_peek_stream.rs

@@ -0,0 +1,171 @@
+use crate::xx::*;
+use async_std::io::{Read, ReadExt, Result, Write};
+use core::task::{Context, Poll};
+use std::pin::Pin;
+
+////////
+///
+trait SendStream: Read + Write + Send + Unpin {
+    fn clone_stream(&self) -> Box<dyn SendStream>;
+}
+impl<S> SendStream for S
+where
+    S: Read + Write + Send + Clone + Unpin + 'static,
+{
+    fn clone_stream(&self) -> Box<dyn SendStream> {
+        Box::new(self.clone())
+    }
+}
+
+/////////
+///
+struct AsyncPeekStreamInner {
+    stream: Box<dyn SendStream>,
+    peekbuf: Vec<u8>,
+    peekbuf_len: usize,
+}
+
+#[derive(Clone)]
+pub struct AsyncPeekStream
+where
+    Self: Read + Write + Send + Unpin,
+{
+    inner: Arc<Mutex<AsyncPeekStreamInner>>,
+}
+
+impl AsyncPeekStream {
+    pub fn new<S>(stream: S) -> Self
+    where
+        S: Read + Write + Send + Clone + Unpin + 'static,
+    {
+        Self {
+            inner: Arc::new(Mutex::new(AsyncPeekStreamInner {
+                stream: Box::new(stream),
+                peekbuf: Vec::new(),
+                peekbuf_len: 0,
+            })),
+        }
+    }
+
+    pub async fn peek(&'_ self, buf: &'_ mut [u8]) -> Result<usize> {
+        let (mut stream, mut peekbuf, mut peekbuf_len) = {
+            let inner = self.inner.lock();
+            (
+                inner.stream.clone_stream(),
+                inner.peekbuf.clone(),
+                inner.peekbuf_len,
+            )
+        };
+        //
+        let buf_len = buf.len();
+        let mut copy_len = buf_len;
+        if buf_len > peekbuf_len {
+            //
+            peekbuf.resize(buf_len, 0u8);
+            let read_len = stream
+                .read(&mut peekbuf.as_mut_slice()[peekbuf_len..buf_len])
+                .await?;
+            peekbuf_len += read_len;
+            copy_len = peekbuf_len;
+        }
+        buf[..copy_len].copy_from_slice(&peekbuf[..copy_len]);
+
+        let mut inner = self.inner.lock();
+        inner.peekbuf = peekbuf;
+        inner.peekbuf_len = peekbuf_len;
+        Ok(copy_len)
+    }
+
+    pub async fn peek_exact(&'_ self, buf: &'_ mut [u8]) -> Result<()> {
+        let (mut stream, mut peekbuf, mut peekbuf_len) = {
+            let inner = self.inner.lock();
+            (
+                inner.stream.clone_stream(),
+                inner.peekbuf.clone(),
+                inner.peekbuf_len,
+            )
+        };
+        //
+        let buf_len = buf.len();
+        if buf_len > peekbuf_len {
+            //
+            peekbuf.resize(buf_len, 0u8);
+            stream
+                .read_exact(&mut peekbuf.as_mut_slice()[peekbuf_len..buf_len])
+                .await?;
+            peekbuf_len = buf_len;
+        }
+        buf.copy_from_slice(&peekbuf[..buf_len]);
+
+        let mut inner = self.inner.lock();
+        inner.peekbuf = peekbuf;
+        inner.peekbuf_len = peekbuf_len;
+        Ok(())
+    }
+}
+
+impl Read for AsyncPeekStream {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        //
+        let buflen = buf.len();
+        let bufcopylen = cmp::min(buflen, inner.peekbuf_len);
+        let bufreadlen = if buflen > inner.peekbuf_len {
+            buflen - inner.peekbuf_len
+        } else {
+            0
+        };
+
+        if bufreadlen > 0 {
+            match Pin::new(&mut inner.stream).poll_read(cx, &mut buf[bufcopylen..buflen]) {
+                Poll::Ready(res) => {
+                    let readlen = res?;
+                    buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+                    inner.peekbuf_len = 0;
+                    Poll::Ready(Ok(bufcopylen + readlen))
+                }
+                Poll::Pending => {
+                    if bufcopylen > 0 {
+                        buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+                        inner.peekbuf_len = 0;
+                        Poll::Ready(Ok(bufcopylen))
+                    } else {
+                        Poll::Pending
+                    }
+                }
+            }
+        } else {
+            buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+            if bufcopylen == inner.peekbuf_len {
+                inner.peekbuf_len = 0;
+            } else if bufcopylen != 0 {
+                // slide buffer over by bufcopylen
+                let tail = inner.peekbuf.split_off(bufcopylen);
+                inner.peekbuf = tail;
+                inner.peekbuf_len -= bufcopylen;
+            }
+            Poll::Ready(Ok(bufcopylen))
+        }
+    }
+}
+
+impl Write for AsyncPeekStream {
+    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_write(cx, buf)
+    }
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_flush(cx)
+    }
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_close(cx)
+    }
+}
+
+impl std::marker::Unpin for AsyncPeekStream {}

veilid-core/src/intf/native/utils/channel.rs

@@ -0,0 +1,83 @@
+pub use async_std::channel;
+
+#[derive(Debug)]
+pub struct Sender<T> {
+    imp: channel::Sender<T>,
+}
+
+impl<T> Clone for Sender<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Receiver<T> {
+    imp: channel::Receiver<T>,
+}
+
+impl<T> Clone for Receiver<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+pub fn channel<T>(cap: usize) -> (Sender<T>, Receiver<T>) {
+    let imp = channel::bounded(cap);
+    (Sender { imp: imp.0 }, Receiver { imp: imp.1 })
+}
+
+pub use channel::SendError;
+pub use channel::TrySendError;
+
+#[allow(dead_code)]
+impl<T> Sender<T> {
+    // NOTE: This needs a timeout or you could block a very long time
+    // pub async fn send(&self, msg: T) -> Result<(), SendError<T>> {
+    //     self.imp.send(msg).await
+    // }
+    pub async fn try_send(&self, msg: T) -> Result<(), TrySendError<T>> {
+        self.imp.try_send(msg)
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.capacity().unwrap()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        self.imp.is_full()
+    }
+    pub fn len(&self) -> usize {
+        self.imp.len()
+    }
+}
+
+pub use channel::RecvError;
+pub use channel::TryRecvError;
+
+#[allow(dead_code)]
+impl<T> Receiver<T> {
+    pub async fn recv(&self) -> Result<T, RecvError> {
+        self.imp.recv().await
+    }
+    pub async fn try_recv(&self) -> Result<T, TryRecvError> {
+        self.imp.try_recv()
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.capacity().unwrap()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        self.imp.is_full()
+    }
+    pub fn len(&self) -> usize {
+        self.imp.len()
+    }
+}

veilid-core/src/intf/native/utils/clone_stream.rs

@@ -0,0 +1,66 @@
+use crate::xx::*;
+use async_std::io::{Read, Result, Write};
+use core::task::{Context, Poll};
+use std::pin::Pin;
+
+pub struct CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    inner: Arc<Mutex<T>>,
+}
+
+impl<T> Clone for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<T> CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    pub fn new(t: T) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(t)),
+        }
+    }
+}
+impl<T> Read for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_read(cx, buf)
+    }
+}
+
+impl<T> Write for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_write(cx, buf)
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_flush(cx)
+    }
+
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_close(cx)
+    }
+}

veilid-core/src/intf/native/utils/ios/mod.rs

@@ -0,0 +1,99 @@
+use crate::xx::*;
+use backtrace::Backtrace;
+use lazy_static::*;
+use log::*;
+use simplelog::*;
+use std::fs::OpenOptions;
+use std::panic;
+use std::path::{Path, PathBuf};
+
+pub struct IOSGlobals {}
+
+lazy_static! {
+    pub static ref IOS_GLOBALS: Arc<Mutex<Option<IOSGlobals>>> = Arc::new(Mutex::new(None));
+}
+
+pub fn veilid_core_setup_ios<'a>(
+    log_tag: &'a str,
+    terminal_log: Option<Level>,
+    file_log: Option<(Level, &Path)>,
+) {
+    if let Err(e) = veilid_core_setup_ios_internal(log_tag, terminal_log, file_log) {
+        panic!("failed to set up veilid-core: {}", e);
+    }
+}
+
+fn veilid_core_setup_ios_internal<'a>(
+    _log_tag: &'a str,
+    terminal_log: Option<Level>,
+    file_log: Option<(Level, &Path)>,
+) -> Result<(), String> {
+    let mut logs: Vec<Box<dyn SharedLogger>> = Vec::new();
+
+    let mut cb = ConfigBuilder::new();
+    cb.add_filter_ignore_str("async_std");
+    cb.add_filter_ignore_str("async_io");
+    cb.add_filter_ignore_str("polling");
+    cb.add_filter_ignore_str("rustls");
+    cb.add_filter_ignore_str("async_tungstenite");
+    cb.add_filter_ignore_str("tungstenite");
+
+    if let Some(level) = terminal_log {
+        logs.push(TermLogger::new(
+            level.to_level_filter(),
+            cb.build(),
+            TerminalMode::Mixed,
+            ColorChoice::Auto,
+        ))
+    }
+    if let Some((level, log_path)) = file_log {
+        let logfile = OpenOptions::new()
+            .truncate(true)
+            .create(true)
+            .write(true)
+            .open(log_path)
+            .map_err(|e| {
+                format!(
+                    "log open error: {} path={:?} all_dirs={:?}",
+                    e,
+                    log_path,
+                    std::fs::read_dir(std::env::var("HOME").unwrap())
+                        .unwrap()
+                        .map(|d| d.unwrap().path())
+                        .collect::<Vec<PathBuf>>()
+                )
+            })?;
+        logs.push(WriteLogger::new(
+            level.to_level_filter(),
+            cb.build(),
+            logfile,
+        ))
+    }
+    CombinedLogger::init(logs).map_err(|e| format!("logger init error: {}", e))?;
+
+    panic::set_hook(Box::new(|panic_info| {
+        let bt = Backtrace::new();
+        if let Some(location) = panic_info.location() {
+            error!(
+                "panic occurred in file '{}' at line {}",
+                location.file(),
+                location.line(),
+            );
+        } else {
+            error!("panic occurred but can't get location information...");
+        }
+        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
+            error!("panic payload: {:?}", a);
+        } else {
+            error!("no panic payload");
+        }
+        error!("Backtrace:\n{:?}", bt);
+    }));
+
+    *IOS_GLOBALS.lock() = Some(IOSGlobals {});
+    Ok(())
+}

veilid-core/src/intf/native/utils/mod.rs

@@ -0,0 +1,8 @@
+#[cfg(target_os = "android")]
+pub mod android;
+pub mod async_peek_stream;
+pub mod channel;
+pub mod clone_stream;
+#[cfg(target_os = "ios")]
+pub mod ios;
+pub mod network_interfaces;

veilid-core/src/intf/native/utils/network_interfaces.rs

@@ -0,0 +1,110 @@
+#[cfg(target_os = "android")]
+pub use super::android::*;
+use crate::xx::*;
+#[cfg(not(target_os = "android"))]
+pub use if_addrs::*;
+
+#[derive(PartialEq, Eq, Clone)]
+pub struct NetworkInterface {
+    name: String,
+    is_loopback: bool,
+    addrs: Vec<IfAddr>,
+}
+
+#[allow(dead_code)]
+impl NetworkInterface {
+    pub fn new(name: String, is_loopback: bool) -> Self {
+        Self {
+            name: name,
+            is_loopback: is_loopback,
+            addrs: Vec::new(),
+        }
+    }
+    pub fn name(&self) -> String {
+        self.name.clone()
+    }
+    pub fn is_loopback(&self) -> bool {
+        self.is_loopback
+    }
+    pub fn primary_ipv4(&self) -> Option<Ipv4Addr> {
+        for x in self.addrs.iter() {
+            match x {
+                IfAddr::V4(a) => return Some(a.ip.clone()),
+                _ => (),
+            };
+        }
+        None
+    }
+    pub fn primary_ipv6(&self) -> Option<Ipv6Addr> {
+        for x in self.addrs.iter() {
+            match x {
+                IfAddr::V6(a) => return Some(a.ip.clone()),
+                _ => (),
+            };
+        }
+        None
+    }
+}
+
+pub struct NetworkInterfaces {
+    valid: bool,
+    interfaces: BTreeMap<String, NetworkInterface>,
+}
+
+#[allow(dead_code)]
+impl NetworkInterfaces {
+    pub fn new() -> Self {
+        Self {
+            valid: false,
+            interfaces: BTreeMap::new(),
+        }
+    }
+    pub fn is_valid(&self) -> bool {
+        self.valid
+    }
+    pub fn clear(&mut self) {
+        self.interfaces.clear();
+        self.valid = false;
+    }
+    // returns Ok(false) if refresh had no changes, Ok(true) if changes were present
+    pub fn refresh(&mut self) -> Result<bool, String> {
+        self.valid = false;
+
+        let last_interfaces = core::mem::take(&mut self.interfaces);
+
+        let mut intfs = match get_if_addrs() {
+            Err(e) => {
+                return Err(format!("failed to refresh network interfaces: {}", e));
+            }
+            Ok(v) => v,
+        };
+        intfs.sort();
+
+        // debug!("{} interfaces found", intfs.len());
+        for intf in intfs {
+            // trace!("interface {} at {}", &intf.name, &intf.addr.ip());
+            let ni = match self.interfaces.get_mut(&intf.name) {
+                None => {
+                    self.interfaces.insert(
+                        intf.name.clone(),
+                        NetworkInterface::new(intf.name.clone(), intf.is_loopback()),
+                    );
+                    self.interfaces.get_mut(&intf.name).unwrap()
+                }
+                Some(v) => v,
+            };
+
+            ni.addrs.push(intf.addr.clone());
+        }
+
+        self.valid = true;
+
+        Ok(last_interfaces != self.interfaces)
+    }
+    pub fn len(&self) -> usize {
+        self.interfaces.len()
+    }
+    pub fn iter(&self) -> std::collections::btree_map::Iter<String, NetworkInterface> {
+        self.interfaces.iter()
+    }
+}