initial import of main veilid core

.gitignore

@@ -0,0 +1,59 @@
+##############################################################################
+### MacOS
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+##############################################################################
+### Windows
+
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+

.vscode/launch.json

@@ -0,0 +1,111 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "lldb",
+      "request": "attach",
+      "name": "Attach to veilid-server",
+      "program": "${workspaceFolder}/veilid-server/target/debug/veilid-server",
+      "pid": "${command:pickMyProcess}"
+    },
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "Launch veilid-cli",
+      "args": ["--debug"],
+      "program": "${workspaceFolder}/veilid-cli/target/debug/veilid-cli",
+      "windows": {
+        "program": "${workspaceFolder}/veilid-cli/target/debug/veilid-cli.exe"
+      },
+      "cwd": "${workspaceFolder}",
+      "sourceLanguages": ["rust"],
+      "terminal": "console"
+    },
+    // {
+    //   "type": "lldb",
+    //   "request": "launch",
+    //   "name": "Debug veilid-server",
+    //   "cargo": {
+    //     "args": ["run", "--manifest-path", "veilid-server/Cargo.toml"]
+    //   },
+    //   "args": ["--trace"],
+    //   "cwd": "${workspaceFolder}/veilid-server"
+    // }
+
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "Debug veilid-server",
+      "program": "${workspaceFolder}/veilid-server/target/debug/veilid-server",
+      "args": ["--trace", "--attach=true"],
+      "cwd": "${workspaceFolder}/veilid-server/target/debug/",
+      "env": {
+        "RUST_BACKTRACE": "1"
+      },
+      "terminal": "console"
+    },
+
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "Debug veilid-core unit test",
+      "cargo": {
+        "args": [
+          "test",
+          "--no-run",
+          "--manifest-path",
+          "veilid-core/Cargo.toml"
+        ],
+        "filter": {
+          "kind": "cdylib",
+          "name": "veilid-core"
+        }
+      },
+      "args": ["${selectedText}"],
+      "cwd": "${workspaceFolder}/veilid-core"
+    },
+
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "Debug veilid-server unit test",
+      "cargo": {
+        "args": [
+          "test",
+          "--no-run",
+          "--manifest-path",
+          "veilid-server/Cargo.toml"
+        ],
+        "filter": {
+          "kind": "bin",
+          "name": "veilid-server"
+        }
+      },
+      "args": ["${selectedText}"],
+      "cwd": "${workspaceFolder}/veilid-server"
+    },
+
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "Debug keyvaluedb-sqlite unit test",
+      "cargo": {
+        "args": [
+          "test",
+          "--no-run",
+          "--manifest-path",
+          "external/keyvaluedb/keyvaluedb-sqlite/Cargo.toml"
+        ],
+        "filter": {
+          "kind": "lib",
+          "name": "keyvaluedb-sqlite"
+        }
+      },
+      "args": ["${selectedText}"],
+      "cwd": "${workspaceFolder}/external/keyvaluedb/keyvaluedb-sqlite"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

external/keyring-rs

@@ -1 +1 @@
-Subproject commit 972802f4f57275a00c29158353adcf7d8ffc7fdc
+Subproject commit 6562019f0b86f622ab5dcadede80f7c8c8ea1eea

setup_android.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [ ! "$(grep -Ei 'debian|buntu|mint' /etc/*release)" ]; then
+    echo Not a supported Linux
+    exit 1
+fi
+
+# ensure ANDROID_SDK_ROOT is defined and exists
+if [ -d "$ANDROID_SDK_ROOT" ]; then
+    echo '[X] $ANDROID_SDK_ROOT is defined and exists' 
+else
+    echo '$ANDROID_SDK_ROOT is not defined or does not exist'
+    exit 1
+fi
+
+
+# ensure ANDROID_NDK_HOME is defined and exists
+if [ -d "$ANDROID_NDK_HOME" ]; then
+    echo '[X] $ANDROID_NDK_HOME is defined and exists' 
+else
+    echo '$ANDROID_NDK_HOME is not defined or does not exist'
+    exit 1
+fi
+
+# ensure ndk is installed
+if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then
+    echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME' 
+else
+    echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME'
+    exit 1
+fi
+
+# ensure cmake is installed
+if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then
+    echo '[X] Android SDK CMake is installed' 
+else
+    echo 'Android SDK CMake is not installed'
+    exit 1
+fi
+
+# ensure emulator is installed
+if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then
+    echo '[X] Android SDK emulator is installed' 
+else
+    echo 'Android SDK emulator is not installed'
+    exit 1
+fi
+
+# ensure adb is installed
+if command -v adb &> /dev/null; then 
+    echo '[X] adb is available in the path'
+else
+    echo 'adb is not available in the path'
+    exit 1
+fi
+
+# install android targets
+rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android
+
+# install cargo ndk
+cargo install cargo-ndk
+cargo install cargo-apk
+
+# Ensure packages are installed
+sudo apt-get install libc6-dev-i386 libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 libbz2-1.0:i386 openjdk-11-jdk
+
+

setup_ios.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [ ! "$(uname)" == "Darwin" ]; then
+    echo Not running on MacOS
+    exit 1
+fi
+
+# install android targets
+rustup target add aarch64-apple-darwin aarch64-apple-ios x86_64-apple-darwin x86_64-apple-ios
+
+# xxx: install +ios-arm64-nightly-2021-06-12 toolchain for bitcode from https://github.com/getditto/rust-bitcode
+
+# Ensure brew is installed
+if command -v brew &> /dev/null; then 
+    echo '[X] brew is available in the path'
+else
+    echo 'brew is not available in the path'
+    exit 1
+fi
+
+# Ensure xcode is installed
+if command -v xcode-select &> /dev/null; then 
+    echo '[X] XCode is available in the path'
+else
+    echo 'XCode is not available in the path'
+    exit 1
+fi
+
+# Ensure we have command line tools
+xcode-select --install
+
+# Ensure packages are installed
+brew install capnp
+

test/scripts/debug_main_node.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec ./run_local_test.py 20 -w 0 --config-file ./no-timeout.cfg $1

test/scripts/debug_subnode_1.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec ./run_local_test.py 2 -w 1 --log_trace --config-file ./no-timeout.cfg

test/scripts/no-timeout.yml

@@ -0,0 +1,12 @@
+---
+core:
+  network:
+    rpc:
+      max_timestamp_behind:
+      max_timestamp_ahead:
+      timeout: 86400000000
+    dht:
+      resolve_node_timeout:
+      get_value_timeout:
+      set_value_timeout:
+    address_filter: false

test/scripts/run_20_no_timeout.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+exec ./run_local_test.py 20 --config-file ./no-timeout.cfg $1
+
+

test/scripts/run_2_no_timeout.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+exec ./run_local_test.py 2 --config-file ./no-timeout.cfg $1
+
+

test/scripts/run_4_no_timeout.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+exec ./run_local_test.py 4 --config-file ./no-timeout.cfg $1
+
+

test/scripts/run_local_test.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import io
+import argparse
+import subprocess
+import signal
+import time
+from threading import Thread
+
+if sys.version_info < (3, 0, 0):
+    print(__file__ + ' requires Python 3, while Python ' +
+          str(sys.version[0] + ' was detected. Terminating. '))
+    sys.exit(1)
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+veilid_server_exe_debug = os.path.join(script_dir, '..', 'veilid-server',
+                                       'target', 'debug', 'veilid-server')
+veilid_server_exe_release = os.path.join(
+    script_dir, '..', 'veilid-server', 'target', 'release', 'veilid-server')
+main_process = None
+subindex_processes = []
+
+try:
+    # Python 3, open as binary, then wrap in a TextIOWrapper with write-through.
+    sys.stdout = io.TextIOWrapper(
+        open(sys.stdout.fileno(), 'wb', 0), write_through=True)
+    sys.stderr = io.TextIOWrapper(
+        open(sys.stderr.fileno(), 'wb', 0), write_through=True)
+except TypeError:
+    # Python 2
+    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+    sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
+
+
+def tee(prefix, infile, *files):
+    """Print `infile` to `files` in a separate thread."""
+
+    def fanout(prefix, infile, *files):
+        with infile:
+            for line in iter(infile.readline, b""):
+                for f in files:
+                    f.write(prefix + line)
+                    f.flush()
+
+    t = Thread(target=fanout, args=(prefix, infile,) + files)
+    t.daemon = True
+    t.start()
+    return t
+
+
+def read_until_local_dial_info(proc, proto):
+
+    local_dial_info_str = b"Local Dial Info: "
+    for ln in iter(proc.stdout.readline, ""):
+        sys.stdout.buffer.write(ln)
+        sys.stdout.flush()
+
+        idx = ln.find(local_dial_info_str)
+        if idx != -1:
+            idx += len(local_dial_info_str)
+            di = ln[idx:]
+            if di.startswith(proto):
+                return di.decode("utf-8").strip()
+
+    return None
+
+
+class CleanChildProcesses:
+    def __enter__(self):
+        os.setpgrp()  # create new process group, become its leader
+
+    def __exit__(self, type, value, traceback):
+        try:
+            os.killpg(0, signal.SIGKILL)  # kill all processes in my group
+        except KeyboardInterrupt:
+            pass
+
+
+def main():
+    threads = []
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Run veilid servers locally')
+    parser.add_argument("count", type=int,
+                        help='number of instances to run')
+    parser.add_argument("--release", action='store_true',
+                        help='use release mode build')
+    parser.add_argument("--log_trace", action='store_true',
+                        help='use trace logging')
+    parser.add_argument("--log_info", action='store_true',
+                        help='use info logging')
+    parser.add_argument("-w", "--wait-for-debug", action='append',
+                        help='specify subnode index to wait for the debugger')
+    parser.add_argument("--config-file", type=str,
+                        help='configuration file to specify for the bootstrap node')
+    parser.add_argument("--protocol", type=bytes, default=b"udp",
+                        help='default protocol to choose for dial info')
+    args = parser.parse_args()
+
+    if args.count < 1:
+        print("Must specify more than one instance")
+        sys.exit(1)
+
+    veilid_server_exe = None
+    if args.release:
+        veilid_server_exe = veilid_server_exe_release
+    else:
+        veilid_server_exe = veilid_server_exe_debug
+
+    base_args = [veilid_server_exe]
+    base_args.append("--attach=true")
+    if args.log_info:
+        pass
+    elif args.log_trace:
+        base_args.append("--trace")
+    else:
+        base_args.append("--debug")
+
+    if args.config_file:
+        base_args.append("--config-file={}".format(args.config_file))
+
+    # Run primary node and get node id
+    main_args = base_args.copy()
+    if args.wait_for_debug and ("0" in args.wait_for_debug):
+        main_args.append("--wait-for-debug")
+
+    print("Running main node: {}".format(str(main_args)))
+    main_proc = subprocess.Popen(
+        main_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    print(">>> MAIN NODE PID={}".format(main_proc.pid))
+
+    main_di = read_until_local_dial_info(main_proc, args.protocol)
+
+    threads.append(
+        tee(b"Veilid-0: ", main_proc.stdout, open("/tmp/veilid-0-out", "wb"),
+            getattr(sys.stdout, "buffer", sys.stdout))
+    )
+
+    # Run all secondaries and add primary to bootstrap
+    for n in range(1, args.count):
+
+        # time.sleep(2)
+
+        sub_args = base_args.copy()
+        sub_args.append("--subnode_index={}".format(n))
+        sub_args.append("--bootstrap={}".format(main_di))
+        if args.wait_for_debug and (str(n) in args.wait_for_debug):
+            sub_args.append("--wait-for-debug")
+
+        print("Running subnode {}: {}".format(n, str(sub_args)))
+
+        sub_proc = subprocess.Popen(
+            sub_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        print(">>> SUBNODE {} NODE PID={}".format(n, sub_proc.pid))
+
+        threads.append(
+            tee("Veilid-{}: ".format(n).encode("utf-8"), sub_proc.stdout, open("/tmp/veilid-{}-out".format(n), "wb"),
+                getattr(sys.stdout, "buffer", sys.stdout))
+        )
+
+    for t in threads:
+        t.join()  # wait for IO completion
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+
+    with CleanChildProcesses():
+        main()

veilid-cli/.gitignore

@@ -0,0 +1,2 @@
+/target
+/logs

veilid-cli/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "veilid-cli"
+version = "0.1.0"
+authors = ["John Smith <jsmith@example.com>"]
+edition = "2018"
+build = "build.rs"
+license = "LGPL-2.0-or-later OR MPL-2.0 OR (MIT AND BSD-3-Clause)"
+
+[[bin]]
+name = "veilid-cli"
+path = "src/main.rs"
+
+[patch.crates-io]
+cursive = { path = "../external/cursive/cursive", default-features = false, features = ["crossterm-backend", "toml"]}
+cursive_core = { path = "../external/cursive/cursive-core" }
+
+[dependencies]
+async-std = { version = "^1.9", features = ["unstable", "attributes"] }
+async-tungstenite = { version = "^0.8", features = ["async-std-runtime"] }
+cursive = { path = "../external/cursive/cursive", default-features = false, features = ["crossterm-backend", "toml"]}
+cursive-flexi-logger-view = { path = "../external/cursive-flexi-logger-view" }
+cursive_buffered_backend = { path = "../external/cursive_buffered_backend" }
+# cursive-multiplex = "0.4.0"
+# cursive_tree_view = "0.6.0"
+# cursive_table_view = "0.12.0"
+# cursive-tabs = "0.5.0"
+clap = "^2.33.2"
+directories = "^3"
+log = "^0.4"
+futures = "^0.3"
+serde = "^1.0.122"
+serde_derive = "^1.0.122"
+parking_lot = "^0.11"
+cfg-if = "^0.1"
+capnp = "^0.14"
+capnp-rpc = "^0.14"
+config = { version = "0.10.1", features = ["yaml"] }
+anyhow = "^1"
+bugsalot = "^0.2"
+flexi_logger = "0.17"
+thiserror = "^1.0"
+crossbeam-channel = "0.5"
+veilid-core = { path = "../veilid-core" }
+
+[dev-dependencies]
+serial_test = "^0.4"
+
+[build-dependencies]
+capnpc = "^0.14"
+
+[profile.release]
+lto = true

veilid-cli/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    ::capnpc::CompilerCommand::new()
+        .file("../veilid-server/proto/veilid-client.capnp")
+        .src_prefix("../veilid-server/")
+        .run()
+        .expect("compiling schema");
+}

veilid-cli/src/client_api_connection.rs

@@ -0,0 +1,214 @@
+use crate::command_processor::*;
+use crate::veilid_client_capnp::*;
+use anyhow::*;
+use async_std::prelude::*;
+use capnp::capability::Promise;
+use capnp_rpc::{pry, rpc_twoparty_capnp, twoparty, Disconnector, RpcSystem};
+use futures::AsyncReadExt;
+use log::*;
+use std::cell::RefCell;
+use std::net::SocketAddr;
+use std::rc::Rc;
+
+struct VeilidClientImpl {
+    comproc: CommandProcessor,
+}
+
+impl VeilidClientImpl {
+    pub fn new(comproc: CommandProcessor) -> Self {
+        Self { comproc: comproc }
+    }
+}
+
+impl veilid_client::Server for VeilidClientImpl {
+    fn state_changed(
+        &mut self,
+        params: veilid_client::StateChangedParams,
+        _results: veilid_client::StateChangedResults,
+    ) -> Promise<(), ::capnp::Error> {
+        let changed = pry!(pry!(params.get()).get_changed());
+
+        if changed.has_attachment() {
+            let attachment = pry!(changed.get_attachment());
+            let old_state = pry!(attachment.get_old_state());
+            let new_state = pry!(attachment.get_new_state());
+
+            trace!(
+                "AttachmentStateChange: old_state={} new_state={}",
+                old_state as u16,
+                new_state as u16
+            );
+            self.comproc.set_attachment_state(new_state);
+        }
+
+        Promise::ok(())
+    }
+}
+
+struct ClientApiConnectionInner {
+    comproc: CommandProcessor,
+    connect_addr: Option<SocketAddr>,
+    disconnector: Option<Disconnector<rpc_twoparty_capnp::Side>>,
+    server: Option<Rc<RefCell<veilid_server::Client>>>,
+    disconnect_requested: bool,
+}
+
+type Handle<T> = Rc<RefCell<T>>;
+
+#[derive(Clone)]
+pub struct ClientApiConnection {
+    inner: Handle<ClientApiConnectionInner>,
+}
+
+impl ClientApiConnection {
+    pub fn new(comproc: CommandProcessor) -> Self {
+        Self {
+            inner: Rc::new(RefCell::new(ClientApiConnectionInner {
+                comproc: comproc,
+                connect_addr: None,
+                disconnector: None,
+                server: None,
+                disconnect_requested: false,
+            })),
+        }
+    }
+
+    async fn handle_connection(&mut self) -> Result<()> {
+        trace!("ClientApiConnection::handle_connection");
+        let connect_addr = self.inner.borrow().connect_addr.unwrap().clone();
+        // Connect the TCP socket
+        let stream = async_std::net::TcpStream::connect(connect_addr.clone()).await?;
+        // If it succeed, disable nagle algorithm
+        stream.set_nodelay(true)?;
+
+        // Create the VAT network
+        let (reader, writer) = stream.split();
+        let rpc_network = Box::new(twoparty::VatNetwork::new(
+            reader,
+            writer,
+            rpc_twoparty_capnp::Side::Client,
+            Default::default(),
+        ));
+        // Create the rpc system
+        let mut rpc_system = RpcSystem::new(rpc_network, None);
+        let mut request;
+        {
+            let mut inner = self.inner.borrow_mut();
+
+            // Get the bootstrap server connection object
+            inner.server = Some(Rc::new(RefCell::new(
+                rpc_system.bootstrap(rpc_twoparty_capnp::Side::Server),
+            )));
+
+            // Store our disconnector future for later (must happen after bootstrap, contrary to documentation)
+            inner.disconnector = Some(rpc_system.get_disconnector());
+
+            // Get a client object to pass to the server for status update callbacks
+            let client = capnp_rpc::new_client(VeilidClientImpl::new(inner.comproc.clone()));
+
+            // Register our client and get a registration object back
+            request = inner
+                .server
+                .as_ref()
+                .unwrap()
+                .borrow_mut()
+                .register_request();
+            request.get().set_veilid_client(client);
+
+            inner
+                .comproc
+                .set_connection_state(ConnectionState::Connected(
+                    connect_addr,
+                    std::time::SystemTime::now(),
+                ));
+        }
+
+        // Don't drop the registration
+        rpc_system.try_join(request.send().promise).await?;
+
+        // Drop the server and disconnector too (if we still have it)
+        let mut inner = self.inner.borrow_mut();
+        let disconnect_requested = inner.disconnect_requested;
+        inner.server = None;
+        inner.disconnector = None;
+        inner.disconnect_requested = false;
+
+        if !disconnect_requested {
+            // Connection lost
+            Err(anyhow!("Connection lost"))
+        } else {
+            // Connection finished
+            Ok(())
+        }
+    }
+
+    pub async fn server_attach(&mut self) -> Result<bool> {
+        trace!("ClientApiConnection::server_attach");
+        let server = {
+            let inner = self.inner.borrow();
+            inner
+                .server
+                .as_ref()
+                .ok_or(anyhow!("Not connected, ignoring attach request"))?
+                .clone()
+        };
+        let request = server.borrow().attach_request();
+        let response = request.send().promise.await?;
+        Ok(response.get()?.get_result())
+    }
+
+    pub async fn server_detach(&mut self) -> Result<bool> {
+        trace!("ClientApiConnection::server_detach");
+        let server = {
+            let inner = self.inner.borrow();
+            inner
+                .server
+                .as_ref()
+                .ok_or(anyhow!("Not connected, ignoring detach request"))?
+                .clone()
+        };
+        let request = server.borrow().detach_request();
+        let response = request.send().promise.await?;
+        Ok(response.get()?.get_result())
+    }
+
+    pub async fn server_shutdown(&mut self) -> Result<bool> {
+        trace!("ClientApiConnection::server_shutdown");
+        let server = {
+            let inner = self.inner.borrow();
+            inner
+                .server
+                .as_ref()
+                .ok_or(anyhow!("Not connected, ignoring attach request"))?
+                .clone()
+        };
+        let request = server.borrow().shutdown_request();
+        let response = request.send().promise.await?;
+        Ok(response.get()?.get_result())
+    }
+
+    // Start Client API connection
+    pub async fn connect(&mut self, connect_addr: SocketAddr) -> Result<()> {
+        trace!("ClientApiConnection::connect");
+        // Save the address to connect to
+        self.inner.borrow_mut().connect_addr = Some(connect_addr);
+
+        self.handle_connection().await
+    }
+
+    // End Client API connection
+    pub async fn disconnect(&mut self) {
+        trace!("ClientApiConnection::disconnect");
+        let disconnector = self.inner.borrow_mut().disconnector.take();
+        match disconnector {
+            Some(d) => {
+                self.inner.borrow_mut().disconnect_requested = true;
+                d.await.unwrap();
+                self.inner.borrow_mut().connect_addr = None;
+            }
+            None => {
+                debug!("disconnector doesn't exist");
+            }
+        }
+    }
+}

veilid-cli/src/command_processor.rs

@@ -0,0 +1,325 @@
+use crate::client_api_connection::*;
+use crate::settings::Settings;
+use crate::ui::*;
+use crate::veilid_client_capnp::*;
+use async_std::prelude::FutureExt;
+use log::*;
+use std::cell::*;
+use std::net::SocketAddr;
+use std::rc::Rc;
+use std::time::{Duration, SystemTime};
+use veilid_core::xx::{Eventual, EventualCommon};
+
+#[derive(PartialEq, Clone)]
+pub enum ConnectionState {
+    Disconnected,
+    Connected(SocketAddr, SystemTime),
+    Retrying(SocketAddr, SystemTime),
+}
+impl ConnectionState {
+    pub fn is_disconnected(&self) -> bool {
+        match *self {
+            Self::Disconnected => true,
+            _ => false,
+        }
+    }
+    pub fn is_connected(&self) -> bool {
+        match *self {
+            Self::Connected(_, _) => true,
+            _ => false,
+        }
+    }
+    pub fn is_retrying(&self) -> bool {
+        match *self {
+            Self::Retrying(_, _) => true,
+            _ => false,
+        }
+    }
+}
+
+struct CommandProcessorInner {
+    ui: UI,
+    capi: Option<ClientApiConnection>,
+    reconnect: bool,
+    finished: bool,
+    autoconnect: bool,
+    autoreconnect: bool,
+    server_addr: Option<SocketAddr>,
+    connection_waker: Eventual,
+}
+
+type Handle<T> = Rc<RefCell<T>>;
+
+#[derive(Clone)]
+pub struct CommandProcessor {
+    inner: Handle<CommandProcessorInner>,
+}
+
+impl CommandProcessor {
+    pub fn new(ui: UI, settings: &Settings) -> Self {
+        Self {
+            inner: Rc::new(RefCell::new(CommandProcessorInner {
+                ui: ui,
+                capi: None,
+                reconnect: settings.autoreconnect,
+                finished: false,
+                autoconnect: settings.autoconnect,
+                autoreconnect: settings.autoreconnect,
+                server_addr: None,
+                connection_waker: Eventual::new(),
+            })),
+        }
+    }
+    pub fn set_client_api_connection(&mut self, capi: ClientApiConnection) {
+        self.inner.borrow_mut().capi = Some(capi);
+    }
+    fn inner(&self) -> Ref<CommandProcessorInner> {
+        self.inner.borrow()
+    }
+    fn inner_mut(&self) -> RefMut<CommandProcessorInner> {
+        self.inner.borrow_mut()
+    }
+    fn ui(&self) -> UI {
+        self.inner.borrow().ui.clone()
+    }
+    fn capi(&self) -> ClientApiConnection {
+        self.inner.borrow().capi.as_ref().unwrap().clone()
+    }
+
+    fn word_split(line: &str) -> (String, Option<String>) {
+        let trimmed = line.trim();
+        if let Some(p) = trimmed.find(char::is_whitespace) {
+            let first = trimmed[0..p].to_owned();
+            let rest = trimmed[p..].trim_start().to_owned();
+            (first, Some(rest))
+        } else {
+            (trimmed.to_owned(), None)
+        }
+    }
+
+    pub fn cmd_help(&self, _rest: Option<String>, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_help");
+        self.ui().add_node_event(
+            r#"Commands:
+exit/quit           - exit the client
+disconnect          - disconnect the client from the Veilid node 
+shutdown            - shut the server down
+attach              - attach the server to the Veilid network
+detach              - detach the server from the Veilid network
+"#,
+        );
+        let ui = self.ui();
+        callback(ui);
+        Ok(())
+    }
+
+    pub fn cmd_exit(&self, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_exit");
+        let ui = self.ui();
+        callback(ui);
+        //
+        self.ui().quit();
+        Ok(())
+    }
+
+    pub fn cmd_shutdown(&self, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_shutdown");
+        let mut capi = self.capi();
+        let ui = self.ui();
+        async_std::task::spawn_local(async move {
+            if let Err(e) = capi.server_shutdown().await {
+                error!("Server command 'shutdown' failed to execute: {}", e);
+            }
+            callback(ui);
+        });
+        Ok(())
+    }
+
+    pub fn cmd_attach(&self, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_attach");
+        let mut capi = self.capi();
+        let ui = self.ui();
+        async_std::task::spawn_local(async move {
+            if let Err(e) = capi.server_attach().await {
+                error!("Server command 'attach' failed to execute: {}", e);
+            }
+            callback(ui);
+        });
+        Ok(())
+    }
+
+    pub fn cmd_detach(&self, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_detach");
+        let mut capi = self.capi();
+        let ui = self.ui();
+        async_std::task::spawn_local(async move {
+            if let Err(e) = capi.server_detach().await {
+                error!("Server command 'detach' failed to execute: {}", e);
+            }
+            callback(ui);
+        });
+        Ok(())
+    }
+
+    pub fn cmd_disconnect(&self, callback: UICallback) -> Result<(), String> {
+        trace!("CommandProcessor::cmd_disconnect");
+        let mut capi = self.capi();
+        let ui = self.ui();
+        async_std::task::spawn_local(async move {
+            capi.disconnect().await;
+            callback(ui);
+        });
+        Ok(())
+    }
+
+    pub fn run_command(&self, command_line: &str, callback: UICallback) -> Result<(), String> {
+        //
+        let (cmd, rest) = Self::word_split(command_line);
+
+        match cmd.as_str() {
+            "help" => self.cmd_help(rest, callback),
+            "exit" => self.cmd_exit(callback),
+            "quit" => self.cmd_exit(callback),
+            "disconnect" => self.cmd_disconnect(callback),
+            "shutdown" => self.cmd_shutdown(callback),
+            "attach" => self.cmd_attach(callback),
+            "detach" => self.cmd_detach(callback),
+            _ => {
+                callback(self.ui());
+                Err(format!("Invalid command: {}", cmd))
+            }
+        }
+    }
+
+    pub async fn connection_manager(&mut self) {
+        // Connect until we're done
+        while !self.inner_mut().finished {
+            // Wait for connection request
+            if !self.inner().autoconnect {
+                let waker = self.inner_mut().connection_waker.instance_clone(());
+                waker.await;
+            } else {
+                self.inner_mut().autoconnect = false;
+            }
+            self.inner_mut().connection_waker.reset();
+            // Loop while we want to keep the connection
+            let mut first = true;
+            while self.inner().reconnect {
+                let server_addr_opt = self.inner_mut().server_addr.clone();
+                let server_addr = match server_addr_opt {
+                    None => break,
+                    Some(addr) => addr,
+                };
+                if first {
+                    info!("Connecting to server at {}", server_addr);
+                    self.set_connection_state(ConnectionState::Retrying(
+                        server_addr.clone(),
+                        SystemTime::now(),
+                    ));
+                } else {
+                    debug!("Retrying connection to {}", server_addr);
+                }
+                let mut capi = self.capi();
+                let res = capi.connect(server_addr.clone()).await;
+                if let Ok(_) = res {
+                    info!(
+                        "Connection to server at {} terminated normally",
+                        server_addr
+                    );
+                    break;
+                }
+                if !self.inner().autoreconnect {
+                    info!("Connection to server lost.");
+                    break;
+                }
+
+                self.set_connection_state(ConnectionState::Retrying(
+                    server_addr.clone(),
+                    SystemTime::now(),
+                ));
+
+                debug!("Connection lost, retrying in 2 seconds");
+                {
+                    let waker = self.inner_mut().connection_waker.instance_clone(());
+                    waker
+                        .race(async_std::task::sleep(Duration::from_millis(2000)))
+                        .await;
+                }
+                self.inner_mut().connection_waker.reset();
+                first = false;
+            }
+            info!("Disconnected.");
+            self.set_connection_state(ConnectionState::Disconnected);
+            self.inner_mut().reconnect = true;
+        }
+    }
+
+    // called by ui
+    ////////////////////////////////////////////
+    pub fn set_server_address(&mut self, server_addr: Option<SocketAddr>) {
+        self.inner_mut().server_addr = server_addr;
+    }
+    pub fn get_server_address(&self) -> Option<SocketAddr> {
+        self.inner().server_addr.clone()
+    }
+    // called by client_api_connection
+    // calls into ui
+    ////////////////////////////////////////////
+    pub fn set_attachment_state(&mut self, state: AttachmentState) {
+        self.inner_mut().ui.set_attachment_state(state);
+    }
+
+    // called by client_api_connection
+    // calls into ui
+    ////////////////////////////////////////////
+    pub fn set_connection_state(&mut self, state: ConnectionState) {
+        self.inner_mut().ui.set_connection_state(state);
+    }
+    // called by ui
+    ////////////////////////////////////////////
+    pub fn start_connection(&mut self) {
+        self.inner_mut().reconnect = true;
+        self.inner_mut().connection_waker.resolve();
+    }
+    // pub fn stop_connection(&mut self) {
+    //     self.inner_mut().reconnect = false;
+    //     let mut capi = self.capi().clone();
+    //     async_std::task::spawn_local(async move {
+    //         capi.disconnect().await;
+    //     });
+    // }
+    pub fn cancel_reconnect(&mut self) {
+        self.inner_mut().reconnect = false;
+        self.inner_mut().connection_waker.resolve();
+    }
+    pub fn quit(&mut self) {
+        self.inner_mut().finished = true;
+        self.inner_mut().reconnect = false;
+        self.inner_mut().connection_waker.resolve();
+    }
+
+    // called by ui
+    // calls into client_api_connection
+    ////////////////////////////////////////////
+    pub fn attach(&mut self) {
+        trace!("CommandProcessor::attach");
+        let mut capi = self.capi();
+
+        async_std::task::spawn_local(async move {
+            if let Err(e) = capi.server_attach().await {
+                error!("Server command 'attach' failed to execute: {}", e);
+            }
+        });
+    }
+
+    pub fn detach(&mut self) {
+        trace!("CommandProcessor::detach");
+        let mut capi = self.capi();
+
+        async_std::task::spawn_local(async move {
+            if let Err(e) = capi.server_detach().await {
+                error!("Server command 'detach' failed to execute: {}", e);
+            }
+        });
+    }
+}

veilid-cli/src/main.rs

@@ -0,0 +1,171 @@
+use anyhow::*;
+use async_std::prelude::*;
+use clap::{App, Arg};
+use flexi_logger::*;
+use log::*;
+use std::ffi::OsStr;
+use std::net::ToSocketAddrs;
+
+mod client_api_connection;
+mod command_processor;
+mod settings;
+mod ui;
+
+pub mod veilid_client_capnp {
+    include!(concat!(env!("OUT_DIR"), "/proto/veilid_client_capnp.rs"));
+}
+
+fn parse_command_line<'a>(
+    default_config_path: &'a OsStr,
+) -> Result<clap::ArgMatches<'a>, clap::Error> {
+    let matches = App::new("veilid-cli")
+        .version("0.1")
+        .about("Veilid Console Client")
+        .arg(
+            Arg::with_name("address")
+                .required(false)
+                .help("Address to connect to"),
+        )
+        .arg(
+            Arg::with_name("debug")
+                .long("debug")
+                .help("Turn on debug logging"),
+        )
+        .arg(
+            Arg::with_name("wait-for-debug")
+                .long("wait-for-debug")
+                .help("Wait for debugger to attach"),
+        )
+        .arg(
+            Arg::with_name("trace")
+                .long("trace")
+                .conflicts_with("debug")
+                .help("Turn on trace logging"),
+        )
+        .arg(
+            Arg::with_name("config-file")
+                .short("c")
+                .takes_value(true)
+                .value_name("FILE")
+                .default_value_os(default_config_path)
+                .help("Specify a configuration file to use"),
+        )
+        .get_matches();
+
+    Ok(matches)
+}
+
+#[async_std::main]
+async fn main() -> Result<()> {
+    // Get command line options
+    let default_config_path = settings::Settings::get_default_config_path();
+    let matches = parse_command_line(default_config_path.as_os_str())?;
+    if matches.occurrences_of("wait-for-debug") != 0 {
+        use bugsalot::debugger;
+        debugger::wait_until_attached(None).expect("state() not implemented on this platform");
+    }
+
+    // Attempt to load configuration
+    let mut settings = settings::Settings::new(
+        matches.occurrences_of("config-file") == 0,
+        matches.value_of_os("config-file").unwrap(),
+    )
+    .map_err(|x| Box::new(x))?;
+
+    // Set config from command line
+    if matches.occurrences_of("debug") != 0 {
+        settings.logging.level = settings::LogLevel::Debug;
+        settings.logging.terminal.enabled = true;
+    }
+    if matches.occurrences_of("trace") != 0 {
+        settings.logging.level = settings::LogLevel::Trace;
+        settings.logging.terminal.enabled = true;
+    }
+
+    // Create UI object
+    let mut sivui = ui::UI::new(settings.interface.node_log.scrollback, &settings);
+
+    // Set up loggers
+    {
+        let mut specbuilder = LogSpecBuilder::new();
+        specbuilder.default(settings::convert_loglevel(settings.logging.level));
+        specbuilder.module("cursive_core", LevelFilter::Off);
+        specbuilder.module("cursive_buffered_backend", LevelFilter::Off);
+        specbuilder.module("mio", LevelFilter::Off);
+        specbuilder.module("async_std", LevelFilter::Off);
+        specbuilder.module("async_io", LevelFilter::Off);
+        specbuilder.module("polling", LevelFilter::Off);
+
+        let logger = Logger::with(specbuilder.build());
+
+        if settings.logging.terminal.enabled {
+            let flv = sivui.cursive_flexi_logger();
+            if settings.logging.file.enabled {
+                std::fs::create_dir_all(settings.logging.file.directory.clone())?;
+                logger
+                    .log_target(LogTarget::FileAndWriter(flv))
+                    .suppress_timestamp()
+                    //    .format(flexi_logger::colored_default_format)
+                    .directory(settings.logging.file.directory.clone())
+                    .start()
+                    .expect("failed to initialize logger!");
+            } else {
+                logger
+                    .log_target(LogTarget::Writer(flv))
+                    .suppress_timestamp()
+                    .format(flexi_logger::colored_default_format)
+                    .start()
+                    .expect("failed to initialize logger!");
+            }
+        } else if settings.logging.file.enabled {
+            std::fs::create_dir_all(settings.logging.file.directory.clone())?;
+            logger
+                .log_target(LogTarget::File)
+                .suppress_timestamp()
+                .directory(settings.logging.file.directory.clone())
+                .start()
+                .expect("failed to initialize logger!");
+        }
+    }
+    // Get client address
+    let server_addrs;
+    if let Some(address_arg) = matches.value_of("address") {
+        server_addrs = address_arg
+            .to_socket_addrs()
+            .context(format!("Invalid server address '{}'", address_arg))?
+            .collect()
+    } else {
+        server_addrs = settings.address.addrs.clone();
+    }
+    let server_addr = server_addrs.first().cloned();
+
+    // Create command processor
+    debug!("Creating Command Processor ");
+    let mut comproc = command_processor::CommandProcessor::new(sivui.clone(), &settings);
+    sivui.set_command_processor(comproc.clone());
+
+    // Create client api client side
+    info!("Starting API connection");
+    let mut capi = client_api_connection::ClientApiConnection::new(comproc.clone());
+
+    // Save client api in command processor
+    comproc.set_client_api_connection(capi.clone());
+
+    // Keep a connection to the server
+    comproc.set_server_address(server_addr);
+    let mut comproc2 = comproc.clone();
+    let connection_future = comproc.connection_manager();
+    // Start UI
+    let ui_future = async_std::task::spawn_local(async move {
+        sivui.run_async().await;
+
+        // When UI quits, close connection and command processor cleanly
+        comproc2.quit();
+        capi.disconnect().await;
+    });
+
+    // Wait for ui and connection to complete
+    ui_future.join(connection_future).await;
+
+    Ok(())
+}

veilid-cli/src/settings.rs

@@ -0,0 +1,269 @@
+use config;
+use directories::*;
+use log;
+use serde;
+use serde_derive::*;
+use std::ffi::OsStr;
+use std::net::{SocketAddr, ToSocketAddrs};
+use std::path::{Path, PathBuf};
+
+pub fn load_default_config(cfg: &mut config::Config) -> Result<(), config::ConfigError> {
+    let default_config = r###"---
+address: "localhost:5959"
+autoconnect: true
+autoreconnect: true
+logging: 
+    level: "info"
+    terminal: 
+        enabled: false
+    file:
+        enabled: true
+        directory: ""
+        append: true
+interface:
+    node_log:
+        scrollback: 2048
+    command_line:
+        history_size: 2048
+    theme:
+        shadow: false
+        borders: "simple"
+        colors:
+            background         : "#333D3D"
+            shadow             : "#000000"
+            view               : "#1c2323"
+            primary            : "#a6d8d3"
+            secondary          : "#8cb4b7"
+            tertiary           : "#eeeeee"
+            title_primary      : "#f93fbd"
+            title_secondary    : "#ff0000"
+            highlight          : "#f93fbd"
+            highlight_inactive : "#a6d8d3"
+            highlight_text     : "#333333"
+        log_colors:
+            trace              : "#707070"
+            debug              : "#a0a0a0"
+            info               : "#5cd3c6"
+            warn               : "#fedc50"
+            error              : "#ff4a15"
+    "###;
+    cfg.merge(config::File::from_str(
+        default_config,
+        config::FileFormat::Yaml,
+    ))
+    .map(drop)
+}
+
+pub fn load_config(
+    cfg: &mut config::Config,
+    config_file: &Path,
+) -> Result<(), config::ConfigError> {
+    if let Some(config_file_str) = config_file.to_str() {
+        cfg.merge(config::File::new(config_file_str, config::FileFormat::Yaml))
+            .map(drop)
+    } else {
+        Err(config::ConfigError::Message(
+            "config file path is not valid UTF-8".to_owned(),
+        ))
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub enum LogLevel {
+    Error,
+    Warn,
+    Info,
+    Debug,
+    Trace,
+}
+impl<'de> serde::Deserialize<'de> for LogLevel {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        match s.to_ascii_lowercase().as_str() {
+            "error" => Ok(LogLevel::Error),
+            "warn" => Ok(LogLevel::Warn),
+            "info" => Ok(LogLevel::Info),
+            "debug" => Ok(LogLevel::Debug),
+            "trace" => Ok(LogLevel::Trace),
+            _ => Err(serde::de::Error::custom(format!(
+                "Invalid log level: {}",
+                s
+            ))),
+        }
+    }
+}
+pub fn convert_loglevel(log_level: LogLevel) -> log::LevelFilter {
+    match log_level {
+        LogLevel::Error => log::LevelFilter::Error,
+        LogLevel::Warn => log::LevelFilter::Warn,
+        LogLevel::Info => log::LevelFilter::Info,
+        LogLevel::Debug => log::LevelFilter::Debug,
+        LogLevel::Trace => log::LevelFilter::Trace,
+    }
+}
+
+#[derive(Debug)]
+pub struct NamedSocketAddrs {
+    pub name: String,
+    pub addrs: Vec<SocketAddr>,
+}
+impl<'de> serde::Deserialize<'de> for NamedSocketAddrs {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let addr_iter = s
+            .to_socket_addrs()
+            .map_err(|x| serde::de::Error::custom(x))?;
+        Ok(NamedSocketAddrs {
+            name: s,
+            addrs: addr_iter.collect(),
+        })
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Terminal {
+    pub enabled: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct File {
+    pub enabled: bool,
+    pub directory: String,
+    pub append: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Logging {
+    pub terminal: Terminal,
+    pub file: File,
+    pub level: LogLevel,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Colors {
+    pub background: String,
+    pub shadow: String,
+    pub view: String,
+    pub primary: String,
+    pub secondary: String,
+    pub tertiary: String,
+    pub title_primary: String,
+    pub title_secondary: String,
+    pub highlight: String,
+    pub highlight_inactive: String,
+    pub highlight_text: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct LogColors {
+    pub trace: String,
+    pub debug: String,
+    pub info: String,
+    pub warn: String,
+    pub error: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Theme {
+    pub shadow: bool,
+    pub borders: String,
+    pub colors: Colors,
+    pub log_colors: LogColors,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct NodeLog {
+    pub scrollback: usize,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CommandLine {
+    pub history_size: usize,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Interface {
+    pub theme: Theme,
+    pub node_log: NodeLog,
+    pub command_line: CommandLine,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Settings {
+    pub address: NamedSocketAddrs,
+    pub autoconnect: bool,
+    pub autoreconnect: bool,
+    pub logging: Logging,
+    pub interface: Interface,
+}
+
+impl Settings {
+    pub fn get_default_config_path() -> PathBuf {
+        // Get default configuration file location
+        let mut default_config_path;
+
+        if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") {
+            default_config_path = PathBuf::from(my_proj_dirs.config_dir());
+        } else {
+            default_config_path = PathBuf::from("./");
+        }
+        default_config_path.push("veilid-client.conf");
+
+        default_config_path
+    }
+
+    pub fn get_default_log_directory() -> PathBuf {
+        // Get default configuration file location
+        let mut default_log_directory;
+
+        if let Some(my_proj_dirs) = ProjectDirs::from("org", "Veilid", "Veilid") {
+            default_log_directory = PathBuf::from(my_proj_dirs.config_dir());
+        } else {
+            default_log_directory = PathBuf::from("./");
+        }
+        default_log_directory.push("logs/");
+
+        default_log_directory
+    }
+
+    pub fn new(
+        config_file_is_default: bool,
+        config_file: &OsStr,
+    ) -> Result<Self, config::ConfigError> {
+        // Create a config
+        let mut cfg = config::Config::default();
+
+        // Load the default config
+        load_default_config(&mut cfg)?;
+
+        // Use default log directory for logs
+        cfg.set(
+            "logging.file.directory",
+            Settings::get_default_log_directory().to_str(),
+        )?;
+
+        // Merge in the config file if we have one
+        let config_file_path = Path::new(config_file);
+        if !config_file_is_default || config_file_path.exists() {
+            // If the user specifies a config file on the command line then it must exist
+            load_config(&mut cfg, config_file_path)?;
+        }
+        cfg.try_into()
+    }
+}
+
+#[test]
+fn test_default_config() {
+    let mut cfg = config::Config::default();
+
+    load_default_config(&mut cfg).unwrap();
+    let settings = cfg.try_into::<Settings>().unwrap();
+
+    println!("default settings: {:?}", settings);
+}

veilid-cli/src/ui.rs

@@ -0,0 +1,785 @@
+use crate::command_processor::*;
+use crate::settings::Settings;
+use crate::veilid_client_capnp::*;
+use crossbeam_channel::Sender;
+use cursive::align::*;
+use cursive::event::*;
+use cursive::theme::*;
+use cursive::traits::*;
+use cursive::utils::markup::StyledString;
+use cursive::views::*;
+use cursive::Cursive;
+use cursive::CursiveRunnable;
+use cursive_flexi_logger_view::{CursiveLogWriter, FlexiLoggerView};
+use log::*;
+use std::cell::RefCell;
+use std::collections::{HashMap, VecDeque};
+use std::rc::Rc;
+use thiserror::Error;
+//////////////////////////////////////////////////////////////
+///
+struct Dirty<T> {
+    pub value: T,
+    dirty: bool,
+}
+
+impl<T> Dirty<T> {
+    pub fn new(value: T) -> Self {
+        Self {
+            value: value,
+            dirty: true,
+        }
+    }
+    pub fn set(&mut self, value: T) {
+        self.value = value;
+        self.dirty = true;
+    }
+    pub fn get(&self) -> &T {
+        &self.value
+    }
+    // pub fn get_mut(&mut self) -> &mut T {
+    //     &mut self.value
+    // }
+    pub fn take_dirty(&mut self) -> bool {
+        let is_dirty = self.dirty;
+        self.dirty = false;
+        is_dirty
+    }
+}
+
+pub type UICallback = Box<dyn Fn(UI) + 'static>;
+
+struct UIState {
+    attachment_state: Dirty<AttachmentState>,
+    connection_state: Dirty<ConnectionState>,
+}
+
+impl UIState {
+    pub fn new() -> Self {
+        Self {
+            attachment_state: Dirty::new(AttachmentState::Detached),
+            connection_state: Dirty::new(ConnectionState::Disconnected),
+        }
+    }
+}
+
+#[derive(Error, Debug)]
+#[error("???")]
+struct UIError;
+
+pub struct UIInner {
+    ui_state: UIState,
+    log_colors: HashMap<Level, cursive::theme::Color>,
+    cmdproc: Option<CommandProcessor>,
+    cb_sink: Sender<Box<dyn FnOnce(&mut Cursive) + 'static + Send>>,
+    cmd_history: VecDeque<String>,
+    cmd_history_position: usize,
+    cmd_history_max_size: usize,
+    connection_dialog_state: Option<ConnectionState>,
+}
+
+type Handle<T> = Rc<RefCell<T>>;
+
+#[derive(Clone)]
+pub struct UI {
+    siv: Handle<CursiveRunnable>,
+    inner: Handle<UIInner>,
+}
+
+impl UI {
+    pub fn new(node_log_scrollback: usize, settings: &Settings) -> Self {
+        cursive_flexi_logger_view::resize(node_log_scrollback);
+
+        // Instantiate the cursive runnable
+        /*
+        // reduces flicker, but it costs cpu
+        let mut runnable = CursiveRunnable::new(
+            || -> Result<Box<dyn cursive_buffered_backend::Backend>, UIError> {
+                let crossterm_backend = cursive::backends::crossterm::Backend::init().unwrap();
+                let buffered_backend =
+                    cursive_buffered_backend::BufferedBackend::new(crossterm_backend);
+
+                Ok(Box::new(buffered_backend))
+            },
+        );
+        */
+        let runnable = cursive::default();
+        // Make the callback mechanism easily reachable
+        let cb_sink = runnable.cb_sink().clone();
+
+        // Create the UI object
+        let this = Self {
+            siv: Rc::new(RefCell::new(runnable)),
+            inner: Rc::new(RefCell::new(UIInner {
+                ui_state: UIState::new(),
+                log_colors: Default::default(),
+                cmdproc: None,
+                cmd_history: {
+                    let mut vd = VecDeque::new();
+                    vd.push_back("".to_string());
+                    vd
+                },
+                cmd_history_position: 0,
+                cmd_history_max_size: settings.interface.command_line.history_size,
+                connection_dialog_state: None,
+                cb_sink: cb_sink,
+            })),
+        };
+
+        let mut siv = this.siv.borrow_mut();
+        let mut inner = this.inner.borrow_mut();
+
+        // Make the inner object accessible in callbacks easily
+        siv.set_user_data(this.inner.clone());
+
+        // Create layouts
+        let mut mainlayout = LinearLayout::vertical().with_name("main-layout");
+        mainlayout.get_mut().add_child(
+            Panel::new(
+                FlexiLoggerView::new_scrollable()
+                    .with_name("node-events")
+                    .full_screen(),
+            )
+            .title_position(HAlign::Left)
+            .title("Node Events"),
+        );
+        mainlayout.get_mut().add_child(
+            Panel::new(ScrollView::new(
+                TextView::new("Peer Table")
+                    .with_name("peers")
+                    .fixed_height(8)
+                    .scrollable(),
+            ))
+            .title_position(HAlign::Left)
+            .title("Peers"),
+        );
+        let mut command = StyledString::new();
+        command.append_styled("Command> ", ColorStyle::title_primary());
+        //
+        mainlayout.get_mut().add_child(
+            LinearLayout::horizontal()
+                .child(TextView::new(command))
+                .child(
+                    EditView::new()
+                        .on_submit(|s, text| {
+                            UI::on_command_line_entered(s, text);
+                        })
+                        .on_edit(|s, text, cursor| UI::on_command_line_edit(s, text, cursor))
+                        .on_up_down(|s, dir| {
+                            UI::on_command_line_history(s, dir);
+                        })
+                        .style(ColorStyle::new(
+                            PaletteColor::Background,
+                            PaletteColor::Secondary,
+                        ))
+                        .with_name("command-line")
+                        .full_screen()
+                        .fixed_height(1),
+                )
+                .child(
+                    Button::new("Attach", |s| {
+                        UI::on_button_attach_pressed(s);
+                    })
+                    .with_name("button-attach"),
+                ),
+        );
+        let mut version = StyledString::new();
+        version.append_styled(
+            concat!(" | veilid-cli v", env!("CARGO_PKG_VERSION")),
+            ColorStyle::highlight_inactive(),
+        );
+
+        mainlayout.get_mut().add_child(
+            LinearLayout::horizontal()
+                .color(Some(ColorStyle::highlight_inactive()))
+                .child(
+                    TextView::new("")
+                        .with_name("status-bar")
+                        .full_screen()
+                        .fixed_height(1),
+                )
+                .child(TextView::new(version)),
+        );
+
+        siv.add_fullscreen_layer(mainlayout);
+
+        UI::setup_colors(&mut siv, &mut inner, &settings);
+        UI::setup_quit_handler(&mut siv);
+
+        drop(inner);
+        drop(siv);
+
+        this
+    }
+
+    fn command_processor(s: &mut Cursive) -> CommandProcessor {
+        let inner = Self::inner(s);
+        inner.cmdproc.as_ref().unwrap().clone()
+    }
+
+    fn inner(s: &mut Cursive) -> std::cell::Ref<'_, UIInner> {
+        s.user_data::<Handle<UIInner>>().unwrap().borrow()
+    }
+    fn inner_mut(s: &mut Cursive) -> std::cell::RefMut<'_, UIInner> {
+        s.user_data::<Handle<UIInner>>().unwrap().borrow_mut()
+    }
+
+    fn setup_colors(siv: &mut CursiveRunnable, inner: &mut UIInner, settings: &Settings) {
+        // Make colors
+        let mut theme = cursive::theme::load_default();
+        theme.shadow = settings.interface.theme.shadow;
+        theme.borders = BorderStyle::from(&settings.interface.theme.borders);
+        theme.palette.set_color(
+            "background",
+            Color::parse(settings.interface.theme.colors.background.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "shadow",
+            Color::parse(settings.interface.theme.colors.shadow.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "view",
+            Color::parse(settings.interface.theme.colors.view.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "primary",
+            Color::parse(settings.interface.theme.colors.primary.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "secondary",
+            Color::parse(settings.interface.theme.colors.secondary.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "tertiary",
+            Color::parse(settings.interface.theme.colors.tertiary.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "title_primary",
+            Color::parse(settings.interface.theme.colors.title_primary.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "title_secondary",
+            Color::parse(settings.interface.theme.colors.title_secondary.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "highlight",
+            Color::parse(settings.interface.theme.colors.highlight.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "highlight_inactive",
+            Color::parse(settings.interface.theme.colors.highlight_inactive.as_str()).unwrap(),
+        );
+        theme.palette.set_color(
+            "highlight_text",
+            Color::parse(settings.interface.theme.colors.highlight_text.as_str()).unwrap(),
+        );
+        siv.set_theme(theme);
+
+        // Make log colors
+        let mut colors = HashMap::<Level, cursive::theme::Color>::new();
+        colors.insert(
+            Level::Trace,
+            Color::parse(settings.interface.theme.log_colors.trace.as_str()).unwrap(),
+        );
+        colors.insert(
+            Level::Debug,
+            Color::parse(settings.interface.theme.log_colors.debug.as_str()).unwrap(),
+        );
+        colors.insert(
+            Level::Info,
+            Color::parse(settings.interface.theme.log_colors.info.as_str()).unwrap(),
+        );
+        colors.insert(
+            Level::Warn,
+            Color::parse(settings.interface.theme.log_colors.warn.as_str()).unwrap(),
+        );
+        colors.insert(
+            Level::Error,
+            Color::parse(settings.interface.theme.log_colors.error.as_str()).unwrap(),
+        );
+        inner.log_colors = colors;
+    }
+    fn setup_quit_handler(siv: &mut Cursive) {
+        siv.clear_global_callbacks(cursive::event::Event::CtrlChar('c'));
+
+        siv.set_on_pre_event(cursive::event::Event::CtrlChar('c'), UI::quit_handler);
+        siv.set_global_callback(cursive::event::Event::Key(Key::Esc), UI::quit_handler);
+    }
+
+    fn quit_handler(siv: &mut Cursive) {
+        siv.add_layer(
+            Dialog::text("Do you want to exit?")
+                .button("Yes", |s| s.quit())
+                .button("No", |mut s| {
+                    s.pop_layer();
+                    UI::setup_quit_handler(&mut s);
+                }),
+        );
+        siv.set_on_pre_event(cursive::event::Event::CtrlChar('c'), |s| {
+            s.quit();
+        });
+        siv.set_global_callback(cursive::event::Event::Key(Key::Esc), |mut s| {
+            s.pop_layer();
+            UI::setup_quit_handler(&mut s);
+        });
+    }
+
+    pub fn cursive_flexi_logger(&mut self) -> Box<CursiveLogWriter> {
+        let mut flv =
+            cursive_flexi_logger_view::cursive_flexi_logger(self.siv.borrow().cb_sink().clone());
+        flv.set_colors(self.inner.borrow().log_colors.clone());
+        flv
+    }
+    pub fn set_command_processor(&mut self, cmdproc: CommandProcessor) {
+        let mut inner = self.inner.borrow_mut();
+        inner.cmdproc = Some(cmdproc);
+        let _ = inner.cb_sink.send(Box::new(UI::update_cb));
+    }
+    pub fn set_attachment_state(&mut self, state: AttachmentState) {
+        let mut inner = self.inner.borrow_mut();
+        inner.ui_state.attachment_state.set(state);
+        let _ = inner.cb_sink.send(Box::new(UI::update_cb));
+    }
+    pub fn set_connection_state(&mut self, state: ConnectionState) {
+        let mut inner = self.inner.borrow_mut();
+        inner.ui_state.connection_state.set(state);
+        let _ = inner.cb_sink.send(Box::new(UI::update_cb));
+    }
+    pub fn add_node_event(&mut self, event: &str) {
+        let inner = self.inner.borrow_mut();
+        let color = inner.log_colors.get(&Level::Info).unwrap().clone();
+        for line in event.lines() {
+            cursive_flexi_logger_view::push_to_log(StyledString::styled(line, color));
+        }
+        let _ = inner.cb_sink.send(Box::new(UI::update_cb));
+    }
+    pub fn quit(&mut self) {
+        let inner = self.inner.borrow_mut();
+        let _ = inner.cb_sink.send(Box::new(|s| {
+            s.quit();
+        }));
+    }
+    // Note: Cursive is not re-entrant, can't borrow_mut self.siv again after this
+    pub async fn run_async(&mut self) {
+        let mut siv = self.siv.borrow_mut();
+        siv.run_async().await;
+    }
+    // pub fn run(&mut self) {
+    //     let mut siv = self.siv.borrow_mut();
+    //     siv.run();
+    // }
+
+    /////////////////////////////////////////////////////////////////////////////////////
+    // Private functions
+    // fn main_layout(s: &mut Cursive) -> ViewRef<LinearLayout> {
+    //     s.find_name("main-layout").unwrap()
+    // }
+    // fn column_layout(s: &mut Cursive) -> ViewRef<LinearLayout> {
+    //     s.find_name("column-layout").unwrap()
+    // }
+    // fn button_layout(s: &mut Cursive) -> ViewRef<LinearLayout> {
+    //     s.find_name("button-layout").unwrap()
+    // }
+    // fn peers(s: &mut Cursive) -> ViewRef<TextView> {
+    //     s.find_name("peers").unwrap()
+    // }
+    // fn node_events(s: &mut Cursive) -> ViewRef<FlexiLoggerView> {
+    //     s.find_name("node-events").unwrap()
+    // }
+    fn command_line(s: &mut Cursive) -> ViewRef<EditView> {
+        s.find_name("command-line").unwrap()
+    }
+    fn button_attach(s: &mut Cursive) -> ViewRef<Button> {
+        s.find_name("button-attach").unwrap()
+    }
+    fn status_bar(s: &mut Cursive) -> ViewRef<TextView> {
+        s.find_name("status-bar").unwrap()
+    }
+    fn render_attachment_state<'a>(inner: &mut UIInner) -> &'a str {
+        match inner.ui_state.attachment_state.get() {
+            AttachmentState::Detached => " Detached [----]",
+            AttachmentState::Attaching => "Attaching [/   ]",
+            AttachmentState::AttachedWeak => " Attached [|   ]",
+            AttachmentState::AttachedGood => " Attached [||  ]",
+            AttachmentState::AttachedStrong => " Attached [||| ]",
+            AttachmentState::FullyAttached => " Attached [||||]",
+            AttachmentState::OverAttached => " Attached [++++]",
+            AttachmentState::Detaching => "Detaching [////]",
+        }
+    }
+    fn render_button_attach<'a>(inner: &mut UIInner) -> (&'a str, bool) {
+        if let ConnectionState::Connected(_, _) = inner.ui_state.connection_state.get() {
+            match inner.ui_state.attachment_state.get() {
+                AttachmentState::Detached => ("Attach", true),
+                AttachmentState::Attaching => ("Detach", true),
+                AttachmentState::AttachedWeak => ("Detach", true),
+                AttachmentState::AttachedGood => ("Detach", true),
+                AttachmentState::AttachedStrong => ("Detach", true),
+                AttachmentState::FullyAttached => ("Detach", true),
+                AttachmentState::OverAttached => ("Detach", true),
+                AttachmentState::Detaching => ("Detach", false),
+            }
+        } else {
+            (" ---- ", false)
+        }
+    }
+
+    fn on_command_line_edit(s: &mut Cursive, text: &str, _pos: usize) {
+        let mut inner = Self::inner_mut(s);
+
+        // save edited command to newest history slot
+        let hlen = inner.cmd_history.len();
+        inner.cmd_history_position = hlen - 1;
+        inner.cmd_history[hlen - 1] = text.to_owned();
+    }
+
+    pub fn enable_command_ui(s: &mut Cursive, enabled: bool) {
+        Self::command_line(s).set_enabled(enabled);
+        Self::button_attach(s).set_enabled(enabled);
+    }
+
+    fn run_command(s: &mut Cursive, text: &str) -> Result<(), String> {
+        // disable ui
+        Self::enable_command_ui(s, false);
+        // run command
+        let cmdproc = Self::command_processor(s);
+        cmdproc.run_command(
+            text,
+            Box::new(|ui: UI| {
+                let _ = ui
+                    .inner
+                    .borrow()
+                    .cb_sink
+                    .send(Box::new(|s| {
+                        Self::enable_command_ui(s, true);
+                    }))
+                    .unwrap();
+            }),
+        )
+    }
+
+    fn on_command_line_entered(s: &mut Cursive, text: &str) {
+        if text.trim().is_empty() {
+            return;
+        }
+        // run command
+        cursive_flexi_logger_view::push_to_log(StyledString::styled(
+            format!("> {}", text),
+            ColorStyle::primary(),
+        ));
+        match Self::run_command(s, text) {
+            Ok(_) => {}
+            Err(e) => {
+                let color = Self::inner_mut(s)
+                    .log_colors
+                    .get(&Level::Error)
+                    .unwrap()
+                    .clone();
+
+                cursive_flexi_logger_view::push_to_log(StyledString::styled(
+                    format!("> {}", text),
+                    color,
+                ));
+                cursive_flexi_logger_view::push_to_log(StyledString::styled(
+                    format!("  Error: {}", e),
+                    color,
+                ));
+                return;
+            }
+        }
+        // save to history unless it's a duplicate
+        {
+            let mut inner = Self::inner_mut(s);
+
+            let hlen = inner.cmd_history.len();
+            inner.cmd_history[hlen - 1] = text.to_owned();
+
+            if hlen >= 2 && inner.cmd_history[hlen - 1] == inner.cmd_history[hlen - 2] {
+                inner.cmd_history[hlen - 1] = "".to_string();
+            } else {
+                if hlen == inner.cmd_history_max_size {
+                    inner.cmd_history.pop_front();
+                }
+                inner.cmd_history.push_back("".to_string());
+            }
+            let hlen = inner.cmd_history.len();
+            inner.cmd_history_position = hlen - 1;
+        }
+
+        // Clear the edit field
+        let mut cmdline = Self::command_line(s);
+        cmdline.set_content("");
+    }
+
+    fn on_command_line_history(s: &mut Cursive, dir: bool) {
+        let mut cmdline = Self::command_line(s);
+        let mut inner = Self::inner_mut(s);
+        // if at top of buffer or end of buffer, ignore
+        if (!dir && inner.cmd_history_position == 0)
+            || (dir && inner.cmd_history_position == (inner.cmd_history.len() - 1))
+        {
+            return;
+        }
+
+        // move the history position
+        if dir {
+            inner.cmd_history_position += 1;
+        } else {
+            inner.cmd_history_position -= 1;
+        }
+
+        // replace text with current line
+        let hlen = inner.cmd_history_position;
+        cmdline.set_content(inner.cmd_history[hlen].as_str());
+    }
+
+    fn on_button_attach_pressed(s: &mut Cursive) {
+        let action: Option<bool> = match Self::inner_mut(s).ui_state.attachment_state.get() {
+            AttachmentState::Detached => Some(true),
+            AttachmentState::Attaching => Some(false),
+            AttachmentState::AttachedWeak => Some(false),
+            AttachmentState::AttachedGood => Some(false),
+            AttachmentState::AttachedStrong => Some(false),
+            AttachmentState::FullyAttached => Some(false),
+            AttachmentState::OverAttached => Some(false),
+            AttachmentState::Detaching => None,
+        };
+        let mut cmdproc = Self::command_processor(s);
+        if let Some(a) = action {
+            if a {
+                cmdproc.attach();
+            } else {
+                cmdproc.detach();
+            }
+        }
+    }
+
+    fn refresh_button_attach(s: &mut Cursive) {
+        let mut button_attach = UI::button_attach(s);
+        let mut inner = Self::inner_mut(s);
+
+        let (button_text, button_enable) = UI::render_button_attach(&mut inner);
+
+        button_attach.set_label(button_text);
+        button_attach.set_enabled(button_enable);
+    }
+
+    fn submit_connection_address(s: &mut Cursive) {
+        let edit = s.find_name::<EditView>("connection-address").unwrap();
+        let addr = (*edit.get_content()).clone();
+        let sa = match addr.parse::<std::net::SocketAddr>() {
+            Ok(sa) => Some(sa),
+            Err(_) => {
+                s.add_layer(Dialog::text("Invalid address").button("Close", |s| {
+                    s.pop_layer();
+                }));
+                return;
+            }
+        };
+        Self::command_processor(s).set_server_address(sa);
+        Self::command_processor(s).start_connection();
+    }
+
+    fn show_connection_dialog(s: &mut Cursive, state: ConnectionState) -> bool {
+        let mut inner = Self::inner_mut(s);
+
+        let mut show: bool = false;
+        let mut hide: bool = false;
+        let mut reset: bool = false;
+        match state {
+            ConnectionState::Disconnected => {
+                if inner.connection_dialog_state == None
+                    || inner
+                        .connection_dialog_state
+                        .as_ref()
+                        .unwrap()
+                        .is_connected()
+                {
+                    show = true;
+                } else if inner
+                    .connection_dialog_state
+                    .as_ref()
+                    .unwrap()
+                    .is_retrying()
+                {
+                    reset = true;
+                }
+            }
+            ConnectionState::Connected(_, _) => {
+                if inner.connection_dialog_state != None
+                    && !inner
+                        .connection_dialog_state
+                        .as_ref()
+                        .unwrap()
+                        .is_connected()
+                {
+                    hide = true;
+                }
+            }
+            ConnectionState::Retrying(_, _) => {
+                if inner.connection_dialog_state == None
+                    || inner
+                        .connection_dialog_state
+                        .as_ref()
+                        .unwrap()
+                        .is_connected()
+                {
+                    show = true;
+                } else if inner
+                    .connection_dialog_state
+                    .as_ref()
+                    .unwrap()
+                    .is_disconnected()
+                {
+                    reset = true;
+                }
+            }
+        }
+        inner.connection_dialog_state = Some(state);
+        drop(inner);
+        if hide {
+            s.pop_layer();
+            return true;
+        }
+        if show {
+            s.add_layer(
+                Dialog::around(
+                    LinearLayout::vertical().child(
+                        LinearLayout::horizontal()
+                            .child(TextView::new("Address:"))
+                            .child(
+                                EditView::new()
+                                    .on_submit(|s, _| Self::submit_connection_address(s))
+                                    .with_name("connection-address")
+                                    .fixed_height(1)
+                                    .min_width(40),
+                            ),
+                    ),
+                )
+                .title("Connect to server")
+                .with_name("connection-dialog"),
+            );
+
+            return true;
+        }
+        if reset {
+            let mut dlg = s.find_name::<Dialog>("connection-dialog").unwrap();
+            dlg.clear_buttons();
+            return true;
+        }
+
+        return false;
+    }
+
+    fn refresh_connection_dialog(s: &mut Cursive) {
+        let new_state = Self::inner(s).ui_state.connection_state.get().clone();
+
+        if !Self::show_connection_dialog(s, new_state.clone()) {
+            return;
+        }
+
+        match new_state {
+            ConnectionState::Disconnected => {
+                let addr = match Self::command_processor(s).get_server_address() {
+                    None => "".to_owned(),
+                    Some(addr) => addr.to_string(),
+                };
+                debug!("address is {}", addr);
+                let mut edit = s.find_name::<EditView>("connection-address").unwrap();
+                edit.set_content(addr.to_string());
+                edit.set_enabled(true);
+                let mut dlg = s.find_name::<Dialog>("connection-dialog").unwrap();
+                dlg.add_button("Connect", Self::submit_connection_address);
+            }
+            ConnectionState::Connected(_, _) => {
+                return;
+            }
+            ConnectionState::Retrying(addr, _) => {
+                //
+                let mut edit = s.find_name::<EditView>("connection-address").unwrap();
+                debug!("address is {}", addr);
+                edit.set_content(addr.to_string());
+                edit.set_enabled(false);
+                let mut dlg = s.find_name::<Dialog>("connection-dialog").unwrap();
+                dlg.add_button("Cancel", |s| {
+                    Self::command_processor(s).cancel_reconnect();
+                });
+            }
+        }
+    }
+
+    fn refresh_statusbar(s: &mut Cursive) {
+        let mut statusbar = UI::status_bar(s);
+
+        let mut inner = Self::inner_mut(s);
+
+        let mut status = StyledString::new();
+
+        match inner.ui_state.connection_state.get() {
+            ConnectionState::Disconnected => {
+                status.append_styled(format!("Disconnected "), ColorStyle::highlight_inactive());
+                status.append_styled("|", ColorStyle::highlight_inactive());
+            }
+            ConnectionState::Retrying(addr, _) => {
+                status.append_styled(
+                    format!("Reconnecting to {} ", addr.to_string()),
+                    ColorStyle::highlight_inactive(),
+                );
+                status.append_styled("|", ColorStyle::highlight_inactive());
+            }
+            ConnectionState::Connected(addr, _) => {
+                status.append_styled(
+                    format!("Connected to {} ", addr.to_string()),
+                    ColorStyle::highlight_inactive(),
+                );
+                status.append_styled("|", ColorStyle::highlight_inactive());
+                // Add attachment state
+                status.append_styled(
+                    format!(" {} ", UI::render_attachment_state(&mut inner)),
+                    ColorStyle::highlight_inactive(),
+                );
+                status.append_styled("|", ColorStyle::highlight_inactive());
+                // Add bandwidth status
+                status.append_styled(
+                    " Down: 0.0KB/s Up: 0.0KB/s ",
+                    ColorStyle::highlight_inactive(),
+                );
+                status.append_styled("|", ColorStyle::highlight_inactive());
+                // Add tunnel status
+                status.append_styled(" No Tunnels ", ColorStyle::highlight_inactive());
+                status.append_styled("|", ColorStyle::highlight_inactive());
+            }
+        };
+
+        statusbar.set_content(status);
+    }
+
+    fn update_cb(s: &mut Cursive) {
+        let mut inner = Self::inner_mut(s);
+
+        let mut refresh_statusbar = false;
+        let mut refresh_button_attach = false;
+        let mut refresh_connection_dialog = false;
+        if inner.ui_state.attachment_state.take_dirty() {
+            refresh_statusbar = true;
+            refresh_button_attach = true;
+        }
+        if inner.ui_state.connection_state.take_dirty() {
+            refresh_statusbar = true;
+            refresh_button_attach = true;
+            refresh_connection_dialog = true;
+        }
+
+        drop(inner);
+
+        if refresh_statusbar {
+            Self::refresh_statusbar(s);
+        }
+        if refresh_button_attach {
+            Self::refresh_button_attach(s);
+        }
+        if refresh_connection_dialog {
+            Self::refresh_connection_dialog(s);
+        }
+    }
+}

veilid-core/.gitignore

@@ -0,0 +1,6 @@
+/target
+**/*.rs.bk
+bin/
+pkg/
+wasm-pack.log
+/tests/tmp

veilid-core/Cargo.toml

@@ -0,0 +1,143 @@
+[package]
+name = "veilid-core"
+version = "0.1.0"
+authors = ["John Smith <nobody@example.com>"]
+edition = "2018"
+build = "build.rs"
+license = "LGPL-2.0-or-later OR MPL-2.0 OR (MIT AND BSD-3-Clause)"
+
+[lib]
+crate-type = ["cdylib", "staticlib", "rlib"]
+
+[features]
+android_tests = []
+ios_tests = [ "simplelog", "backtrace" ]
+
+[dependencies]
+capnp = { version = "^0", default_features = false }
+rust-fsm = "^0"
+static_assertions = "^1"
+log = "^0"
+cfg-if = "^0"
+anyhow = "^1"
+thiserror = "^1"
+
+blake3 = { version = "^1", default_features = false }
+hex = "^0"
+digest = "^0"
+generic-array = "^0"
+secrecy = "^0"
+chacha20poly1305 = "^0"
+uluru = "^3"
+serde-big-array = "^0"
+futures-util = { version = "^0", default_features = false, features = ["alloc"] }
+parking_lot = "^0"
+lazy_static = "^1"
+directories = "^3"
+once_cell = "^1"
+
+ed25519-dalek = { version = "^1", default_features = false, features = ["alloc", "u64_backend"] }
+x25519-dalek = { package = "x25519-dalek-ng", version = "^1", default_features = false, features = ["u64_backend"] }
+curve25519-dalek = { package = "curve25519-dalek-ng", version = "^4", default_features = false, features = ["alloc", "u64_backend"] }
+# ed25519-dalek needs rand 0.7 until it updates itself
+rand = "0.7"
+
+# Dependencies for native builds only
+# Linux, Windows, Mac, iOS, Android
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+async-std = { version = "^1", features = ["unstable"] }
+async-tungstenite = { version = "^0", features = ["async-std-runtime", "async-tls"] }
+maplit = "^1"
+config = { version = "^0", features = ["yaml"] }
+keyring = { path = "../external/keyring-rs" }
+lru = "^0"
+async-tls = "^0.11"
+webpki = "^0"
+webpki-roots = "^0"
+rustls = "^0.19"
+rustls-pemfile = "^0.2"
+num_cpus = "^1"
+futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std"] }
+keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
+data-encoding = { version = "^2" }
+serde = { version = "^1", features = ["derive" ] }
+serde_cbor = { version = "^0" }
+if-addrs = { path = "../external/if-addrs" }
+async_executors = { version = "^0", features = [ "async_std" ]}
+socket2 = "^0"
+
+# Dependencies for WASM builds only
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-bindgen = "^0"
+console_error_panic_hook = "^0"
+wee_alloc = "^0"
+js-sys = "^0"
+wasm-bindgen-futures = "^0"
+wasm-logger = "^0"
+hashbrown = "^0"
+lru = {version = "^0", features = ["hashbrown"] }
+no-std-net = "^0"
+keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
+data-encoding = { version = "^2", default_features = false, features = ["alloc"] }
+serde = { version = "^1", default-features = false, features = ["derive", "alloc"] }
+serde_cbor = { version = "^0", default-features = false, features = ["alloc"] }
+getrandom = { version = "^0", features = ["js"] }
+ws_stream_wasm = "^0"
+async_executors = { version = "^0", features = [ "bindgen" ]}
+
+# Configuration for WASM32 'web-sys' crate
+[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
+version = "^0"
+features = [
+#  'Document',
+#  'Element',
+#  'HtmlElement',
+#  'Node',
+  'IdbFactory',
+  'IdbOpenDbRequest',
+  'Storage',
+  'Location',
+  'Window',
+]
+
+# Dependencies specifically for Android
+[target.'cfg(target_os = "android")'.dependencies]
+jni = "^0"
+jni-sys = "^0"
+ndk = { version = "^0", features = ["trace"] }
+ndk-glue = { version = "^0", features = ["logger"] }
+android_logger = { version = "^0" }
+backtrace = { version = "^0" }
+
+[target.'cfg(target_os = "ios")'.dependencies]
+simplelog = { version = "^0", optional = true }
+backtrace = { version = "^0", optional = true }
+
+# Rusqlite configuration to ensure platforms that don't come with sqlite get it bundled
+# Except WASM which doesn't use sqlite
+[target.'cfg(all(not(target_os = "ios"),not(target_os = "android"),not(target_arch = "wasm32")))'.dependencies.rusqlite]
+version = "^0"
+features = ["bundled"]
+
+### DEV DEPENDENCIES
+
+[dev-dependencies]
+serial_test = "^0"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+simplelog = { version = "^0", features=["test"] }
+
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+wasm-bindgen-test = "^0"
+
+### BUILD OPTIONS
+
+[build-dependencies]
+capnpc = "^0"
+
+[profile.release]
+opt-level = "s"
+lto = true
+
+[package.metadata.wasm-pack.profile.release]
+wasm-opt = ["-O", "--enable-mutable-globals"]

veilid-core/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    ::capnpc::CompilerCommand::new()
+        .file("proto/veilid.capnp")
+        .run()
+        .expect("compiling schema");
+}

veilid-core/ios_build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+CARGO_MANIFEST_PATH=$SCRIPTDIR/Cargo.toml
+
+if [ "$CONFIGURATION" == "Debug" ]; then 
+    EXTRA_CARGO_OPTIONS="$@"
+else
+    EXTRA_CARGO_OPTIONS="$@ --release"
+fi
+ARCHS=${ARCHS:=arm64}
+for arch in $ARCHS
+do
+    if [ "$arch" == "arm64" ]; then
+        echo arm64
+        CARGO_TARGET=aarch64-apple-ios
+        CARGO_TOOLCHAIN=+ios-arm64-nightly-2021-06-12
+        #CARGO_TOOLCHAIN=
+    elif [ "$arch" == "x86_64" ]; then
+        echo x86_64
+        CARGO_TARGET=x86_64-apple-ios
+        CARGO_TOOLCHAIN=
+    else
+        echo Unsupported ARCH: $arch
+        continue
+    fi
+    env -i PATH=/usr/bin:/bin:/usr/local/bin ~/.cargo/bin/cargo $CARGO_TOOLCHAIN build $EXTRA_CARGO_OPTIONS --target $CARGO_TARGET --manifest-path $CARGO_MANIFEST_PATH
+done
+

veilid-core/proto/veilid.capnp

@@ -0,0 +1,420 @@
+@0x8ffce8033734ab02;
+
+# IDs And Hashes
+##############################
+
+struct Curve25519PublicKey {
+    u0                      @0  :UInt64;
+    u1                      @1  :UInt64;
+    u2                      @2  :UInt64;
+    u3                      @3  :UInt64;
+}
+
+struct Ed25519Signature {
+    u0                      @0  :UInt64;
+    u1                      @1  :UInt64;
+    u2                      @2  :UInt64;
+    u3                      @3  :UInt64;
+    u4                      @4  :UInt64;
+    u5                      @5  :UInt64;
+    u6                      @6  :UInt64;
+    u7                      @7  :UInt64;
+}
+
+struct XChaCha20Poly1305Nonce {
+    u0                      @0  :UInt64;
+    u1                      @1  :UInt64;
+    u2                      @2  :UInt64;
+}
+
+struct BLAKE3Hash {
+    u0                      @0  :UInt64;
+    u1                      @1  :UInt64;
+    u2                      @2  :UInt64;
+    u3                      @3  :UInt64;
+}
+
+using NodeID = Curve25519PublicKey;
+using RoutePublicKey = Curve25519PublicKey;
+using ValueID = Curve25519PublicKey;
+using Nonce = XChaCha20Poly1305Nonce;
+using Signature = Ed25519Signature;
+using BlockID = BLAKE3Hash;
+using TunnelID = UInt64;
+
+# Node Dial Info
+################################################################
+
+struct AddressIPV4 {
+    addr                    @0  :UInt32;                # Address in big endian format
+}
+
+struct AddressIPV6 {
+    addr0                   @0  :UInt32;                # \ 
+    addr1                   @1  :UInt32;                #  \ Address in big 
+    addr2                   @2  :UInt32;                #  / endian format
+    addr3                   @3  :UInt32;                # / 
+}
+
+struct Address {
+    union {
+        ipv4                @0  :AddressIPV4;
+        ipv6                @1  :AddressIPV6;
+        hostname            @2  :Text;
+    }
+}
+
+struct SocketAddress {
+    union {
+        ipv4                @0  :AddressIPV4;
+        ipv6                @1  :AddressIPV6;
+    }
+    port                    @2  :UInt16;
+}
+
+enum ProtocolKind {
+    udp                     @0;
+    ws                      @1;
+    wss                     @2;
+    tcp                     @3;
+}
+
+struct DialInfoUDP {
+    address                 @0  :Address;
+    port                    @1  :UInt16;
+}
+
+struct DialInfoTCP {
+    address                 @0  :Address;
+    port                    @1  :UInt16;
+}
+
+struct DialInfoWS {
+    fqdn                    @0  :Text;
+    port                    @1  :UInt16;
+    path                    @2  :Text;
+}
+
+struct DialInfoWSS {
+    fqdn                    @0  :Text;
+    port                    @1  :UInt16;
+    path                    @2  :Text;
+}
+
+struct DialInfo {
+    union {
+        udp                 @0  :DialInfoUDP;
+        tcp                 @1  :DialInfoTCP;
+        ws                  @2  :DialInfoWS;
+        wss                 @3  :DialInfoWSS;
+    }
+}
+
+struct NodeDialInfoSingle {
+    nodeId                  @0  :NodeID;                # node id
+    dialInfo                @1  :DialInfo;              # how to get to the node
+}
+
+# Private Routes
+##############################
+
+struct RouteHopData {         
+    nonce                   @0  :Nonce;                 # nonce for encrypted blob
+    blob                    @1  :Data;                  # encrypted blob with ENC(nonce,DH(PK,SK))
+                                                        #   can be one of: 
+                                                        #     if more hops remain in this route: RouteHop (0 byte appended as key)
+                                                        #     if end of safety route and starting private route: PrivateRoute (1 byte appended as key)
+}
+
+struct RouteHop {
+    dialInfo                @0  :NodeDialInfoSingle;    # dial info for this hop
+    nextHop                 @1  :RouteHopData;          # Optional: next hop in encrypted blob 
+                                                        # Null means no next hop, at destination (only used in private route, safety routes must enclose a stub private route)
+}
+
+struct PrivateRoute {
+    publicKey               @0  :RoutePublicKey;        # private route public key (unique per private route)
+    hopCount                @1  :UInt8;                 # Count of hops left in the private route
+    firstHop                @2  :RouteHop;              # Optional: first hop in the private route
+}
+
+struct SafetyRoute {
+    publicKey               @0  :RoutePublicKey;        # safety route public key (unique per safety route)
+    hopCount                @1  :UInt8;                 # Count of hops left in the safety route
+    hops :union {
+        data                @2  :RouteHopData;          # safety route has more hops
+        private             @3  :PrivateRoute;          # safety route has ended and private route follows
+    }
+}
+
+# Values
+##############################
+
+using ValueSeqNum = UInt32;                             # sequence numbers for values
+
+struct ValueKey {
+    publicKey               @0  :ValueID;               # the location of the value
+    subkey                  @1  :Text;                  # the name of the subkey (or empty if the whole key)
+}
+
+struct ValueKeySeq {
+    key                     @0  :ValueKey;              # the location of the value
+    seq                     @1  :ValueSeqNum;           # the sequence number of the value subkey
+}
+
+struct ValueData {
+    data                    @0  :Data;                  # value or subvalue contents in CBOR format
+    seq                     @1  :ValueSeqNum;           # sequence number of value
+}
+
+# Operations
+##############################
+
+struct OperationInfoQ {
+}
+
+struct NodeInfo {
+    canRoute                @0  :Bool;  
+    willRoute               @1  :Bool;
+    
+    canTunnel               @2  :Bool;  
+    willTunnel              @3  :Bool;
+    
+    canSignalLease          @4  :Bool;
+    willSignalLease         @5  :Bool;
+    
+    canRelayLease           @6  :Bool;
+    willRelayLease          @7  :Bool;
+
+    canValidateDialInfo     @8  :Bool;
+    willValidateDialInfo    @9  :Bool;
+}
+
+struct SenderInfo {
+    socketAddress           @0  :SocketAddress;         # socket address was available for peer
+}
+
+struct OperationInfoA {
+    nodeInfo                @0  :NodeInfo;              # returned node information
+    senderInfo              @1  :SenderInfo;            # info about InfoQ sender
+}
+
+struct OperationValidateDialInfo {
+    dialInfo                @0  :DialInfo;              # dial info to use for the receipt
+    receipt                 @1  :Data;                  # receipt to return to dial info to prove it is reachable
+    redirect                @2  :Bool;                  # request a different node do the validate
+    alternatePort           @3  :Bool;                  # return receipt from a different source port than the default
+}
+
+struct OperationReturnReceipt {
+    receipt                 @0  :Data;                  # receipt being returned to its origin
+}
+
+struct OperationFindNodeQ {    
+    nodeId                  @0  :NodeID;                # node id to locate
+    peerInfo                @1  :PeerInfo;              # The peer info for node asking the question
+}
+
+struct PeerInfo {
+    nodeId                  @0  :NodeID;                # node id or 'closer peer'
+    dialInfoList            @1  :List(DialInfo);        # dial info for 'closer peer'
+}
+
+struct OperationFindNodeA {
+    peers                   @0  :List(PeerInfo);        # returned 'closer peer' information
+}
+
+struct RoutedOperation {
+    signatures              @0  :List(Signature);       # signatures from nodes that have handled the private route
+    nonce                   @1  :Nonce;                 # nonce Xmsg 
+    data                    @2  :Data;                  # Operation encrypted with ENC(Xmsg,DH(PKapr,SKbsr))
+}
+
+struct OperationRoute {
+    safetyRoute             @0  :SafetyRoute;           # Where this should go
+    operation               @1  :RoutedOperation;       # The operation to be routed
+}
+
+struct OperationGetValueQ {
+    key                     @0  :ValueKey;              # key for value to get
+}
+
+struct OperationGetValueA {
+    union {
+        data                @0  :ValueData;             # the value if successful
+        peers               @1  :List(PeerInfo);        # returned 'closer peer' information if not successful       
+    }
+}
+
+struct OperationSetValueQ {
+    key                     @0  :ValueKey;              # key for value to update
+    value                   @1  :ValueData;             # value or subvalue contents in CBOR format (older or equal seq number gets dropped)
+}
+
+struct OperationSetValueA {
+    union {
+        data                @0  :ValueData;             # the new value if successful, may be a different value than what was set if the seq number was lower or equal
+        peers               @1  :List(PeerInfo);        # returned 'closer peer' information if not successful       
+    }
+}
+
+struct OperationWatchValueQ {
+    key                     @0  :ValueKey;              # key for value to watch
+}
+
+struct OperationWatchValueA {
+    expiration              @0  :UInt64;                # timestamp when this watch will expire in usec since epoch (0 if watch failed)
+    peers                   @1  :List(PeerInfo);        # returned list of other nodes to ask that could propagate watches
+}
+
+struct OperationValueChanged {
+    key                     @0  :ValueKey;              # key for value that changed
+    value                   @1  :ValueData;             # value or subvalue contents in CBOR format with sequence number
+}
+
+struct OperationSupplyBlockQ {
+    blockId                 @0  :BlockID;               # hash of the block we can supply
+}
+
+struct OperationSupplyBlockA {
+    union {
+        expiration          @0  :UInt64;                # when the block supplier entry will need to be refreshed
+        peers               @1  :List(PeerInfo);        # returned 'closer peer' information if not successful       
+    }
+}
+
+struct OperationFindBlockQ {
+    blockId                 @0  :BlockID;               # hash of the block we can supply
+}
+
+struct OperationFindBlockA {
+    data                    @0  :Data;                  # Optional: the actual block data if we have that block ourselves
+                                                        # null if we don't have a block to return
+    suppliers               @1  :List(PeerInfo);        # returned list of suppliers if we have them
+    peers                   @2  :List(PeerInfo);        # returned 'closer peer' information 
+}
+
+struct OperationSignalQ {
+    data                    @0  :Data;                  # the signalling system request
+}
+
+struct OperationSignalA {
+    data                    @0  :Data;                  # the signalling system response
+}
+
+enum TunnelEndpointMode {
+    raw                     @0;                         # raw tunnel
+    turn                    @1;                         # turn tunnel
+}
+
+enum TunnelError {
+    badId                   @0;                         # Tunnel ID was rejected
+    noEndpoint              @1;                         # Endpoint was unreachable
+    rejectedMode            @2;                         # Endpoint couldn't provide mode
+    noCapacity              @3;                         # Endpoint is full
+}
+
+struct TunnelEndpoint {
+    nodeId                  @0  :NodeID;                # node id
+    dialInfoList            @1  :List(DialInfo);        # how to reach the node
+    mode                    @2  :TunnelEndpointMode;    # what kind of endpoint this is
+}
+
+struct FullTunnel {
+    id                      @0  :TunnelID;              # tunnel id to use everywhere
+    timeout                 @1  :UInt64;                # duration from last data when this expires if no data is sent or received
+    local                   @2  :TunnelEndpoint;        # local endpoint
+    remote                  @3  :TunnelEndpoint;        # remote endpoint
+}
+
+struct PartialTunnel {
+    id                      @0  :TunnelID;              # tunnel id to use everywhere
+    timeout                 @1  :UInt64;                # timestamp when this expires if not completed
+    local                   @2  :TunnelEndpoint;        # local endpoint
+}
+
+struct OperationStartTunnelQ {
+    id                      @0  :TunnelID;              # tunnel id to use everywhere
+    localMode               @1  :TunnelEndpointMode;    # what kind of local endpoint mode is being requested
+    depth                   @2  :UInt8;                 # the number of nodes in the tunnel
+}
+
+struct OperationStartTunnelA {
+    union {
+        partial             @0  :PartialTunnel;         # the first half of the tunnel
+        error               @1  :TunnelError;           # if we didn't start the tunnel, why not
+    }
+}
+
+struct OperationCompleteTunnelQ {
+    id                      @0  :TunnelID;              # tunnel id to use everywhere
+    localMode               @1  :TunnelEndpointMode;    # what kind of local endpoint mode is being requested
+    depth                   @2  :UInt8;                 # the number of nodes in the tunnel
+    endpoint                @3  :TunnelEndpoint;        # the remote endpoint to complete
+}
+
+struct OperationCompleteTunnelA {
+    union {
+        tunnel              @0  :FullTunnel;            # the tunnel description
+        error               @1  :TunnelError;           # if we didn't complete the tunnel, why not
+    }
+}
+
+struct OperationCancelTunnelQ {
+    tunnel                  @0  :TunnelID;              # the tunnel id to cancel
+}
+
+struct OperationCancelTunnelA {
+    union {
+        tunnel              @0  :TunnelID;              # the tunnel id that was cancelled
+        error               @1  :TunnelError;           # if we couldn't cancel, why not
+    }
+}
+
+struct Operation {
+    opId                    @0  :UInt64;                # Random RPC ID. Must be random to foil reply forgery attacks. 
+
+    respondTo :union {
+        none                @1  :Void;                  # no response is desired
+        sender              @2  :Void;                  # envelope sender node id to be used for reply
+                                                        # possibly through a relay if the request arrived that way
+        privateRoute        @3  :PrivateRoute;          # embedded private route to be used for reply
+    }                              
+
+    detail :union {
+        # Direct operations
+        infoQ               @4  :OperationInfoQ;
+        infoA               @5  :OperationInfoA;
+        validateDialInfo    @6  :OperationValidateDialInfo;
+        findNodeQ           @7  :OperationFindNodeQ;
+        findNodeA           @8  :OperationFindNodeA;
+        route               @9  :OperationRoute;
+        
+        # Routable operations
+        getValueQ           @10  :OperationGetValueQ;
+        getValueA           @11 :OperationGetValueA;
+        setValueQ           @12 :OperationSetValueQ;
+        setValueA           @13 :OperationSetValueA;
+        watchValueQ         @14 :OperationWatchValueQ;
+        watchValueA         @15 :OperationWatchValueA;
+        valueChanged        @16 :OperationValueChanged;
+        
+        supplyBlockQ        @17 :OperationSupplyBlockQ;
+        supplyBlockA        @18 :OperationSupplyBlockA; 
+        findBlockQ          @19 :OperationFindBlockQ;
+        findBlockA          @20 :OperationFindBlockA; 
+    
+        signalQ             @21 :OperationSignalQ;
+        signalA             @22 :OperationSignalA;
+        
+        returnReceipt       @23  :OperationReturnReceipt;
+        
+        # Tunnel operations
+        startTunnelQ        @24 :OperationStartTunnelQ;
+        startTunnelA        @25 :OperationStartTunnelA;
+        completeTunnelQ     @26 :OperationCompleteTunnelQ;
+        completeTunnelA     @27 :OperationCompleteTunnelA;
+        cancelTunnelQ       @28 :OperationCancelTunnelQ; 
+        cancelTunnelA       @29 :OperationCancelTunnelA;
+    }
+}

veilid-core/src/attachment_manager.rs

@@ -0,0 +1,398 @@
+use crate::callback_state_machine::*;
+use crate::dht::crypto::Crypto;
+use crate::intf::*;
+use crate::network_manager::*;
+use crate::xx::*;
+use crate::*;
+use core::convert::TryFrom;
+
+state_machine! {
+    derive(Debug, PartialEq, Eq, Clone, Copy)
+    pub Attachment(Detached)
+//---
+    Detached(AttachRequested) => Attaching [StartAttachment],
+    Attaching => {
+        AttachmentStopped => Detached,
+        WeakPeers => AttachedWeak,
+        GoodPeers => AttachedGood,
+        StrongPeers => AttachedStrong,
+        FullPeers => FullyAttached,
+        TooManyPeers => OverAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    AttachedWeak => {
+        NoPeers => Attaching,
+        GoodPeers => AttachedGood,
+        StrongPeers => AttachedStrong,
+        FullPeers => FullyAttached,
+        TooManyPeers => OverAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    AttachedGood => {
+        NoPeers => Attaching,
+        WeakPeers => AttachedWeak,
+        StrongPeers => AttachedStrong,
+        FullPeers => FullyAttached,
+        TooManyPeers => OverAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    AttachedStrong => {
+        NoPeers => Attaching,
+        WeakPeers => AttachedWeak,
+        GoodPeers => AttachedGood,
+        FullPeers => FullyAttached,
+        TooManyPeers => OverAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    FullyAttached => {
+        NoPeers => Attaching,
+        WeakPeers => AttachedWeak,
+        GoodPeers => AttachedGood,
+        StrongPeers => AttachedStrong,
+        TooManyPeers => OverAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    OverAttached => {
+        NoPeers => Attaching,
+        WeakPeers => AttachedWeak,
+        GoodPeers => AttachedGood,
+        StrongPeers => AttachedStrong,
+        FullPeers => FullyAttached,
+        DetachRequested => Detaching [StopAttachment]
+    },
+    Detaching => {
+        AttachmentStopped => Detached,
+    },
+}
+
+impl ToString for AttachmentState {
+    fn to_string(&self) -> String {
+        match self {
+            AttachmentState::Attaching => "attaching".to_owned(),
+            AttachmentState::AttachedWeak => "attached_weak".to_owned(),
+            AttachmentState::AttachedGood => "attached_good".to_owned(),
+            AttachmentState::AttachedStrong => "attached_strong".to_owned(),
+            AttachmentState::FullyAttached => "fully_attached".to_owned(),
+            AttachmentState::OverAttached => "over_attached".to_owned(),
+            AttachmentState::Detaching => "detaching".to_owned(),
+            AttachmentState::Detached => "detached".to_owned(),
+        }
+    }
+}
+
+impl TryFrom<String> for AttachmentState {
+    type Error = ();
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        Ok(match s.as_str() {
+            "attaching" => AttachmentState::Attaching,
+            "attached_weak" => AttachmentState::AttachedWeak,
+            "attached_good" => AttachmentState::AttachedGood,
+            "attached_strong" => AttachmentState::AttachedStrong,
+            "fully_attached" => AttachmentState::FullyAttached,
+            "over_attached" => AttachmentState::OverAttached,
+            "detaching" => AttachmentState::Detaching,
+            "detached" => AttachmentState::Detached,
+            _ => return Err(()),
+        })
+    }
+}
+
+pub struct AttachmentManagerInner {
+    config: VeilidConfig,
+    table_store: TableStore,
+    crypto: Crypto,
+    attachment_machine: CallbackStateMachine<Attachment>,
+    network_manager: NetworkManager,
+    maintain_peers: bool,
+    peer_count: u32,
+    attach_timestamp: Option<u64>,
+    attachment_maintainer_jh: Option<JoinHandle<()>>,
+}
+
+#[derive(Clone)]
+pub struct AttachmentManager {
+    inner: Arc<Mutex<AttachmentManagerInner>>,
+}
+
+impl AttachmentManager {
+    fn new_inner(
+        config: VeilidConfig,
+        table_store: TableStore,
+        crypto: Crypto,
+    ) -> AttachmentManagerInner {
+        AttachmentManagerInner {
+            config: config.clone(),
+            table_store: table_store.clone(),
+            crypto: crypto.clone(),
+            attachment_machine: CallbackStateMachine::new(),
+            network_manager: NetworkManager::new(
+                config.clone(),
+                table_store.clone(),
+                crypto.clone(),
+            ),
+            maintain_peers: false,
+            peer_count: 0,
+            attach_timestamp: None,
+            attachment_maintainer_jh: None,
+        }
+    }
+    pub fn new(config: VeilidConfig, table_store: TableStore, crypto: Crypto) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(config, table_store, crypto))),
+        }
+    }
+
+    pub fn config(&self) -> VeilidConfig {
+        self.inner.lock().config.clone()
+    }
+
+    pub fn table_store(&self) -> TableStore {
+        self.inner.lock().table_store.clone()
+    }
+
+    pub fn crypto(&self) -> Crypto {
+        self.inner.lock().crypto.clone()
+    }
+
+    pub fn network_manager(&self) -> NetworkManager {
+        self.inner.lock().network_manager.clone()
+    }
+
+    pub fn is_attached(&self) -> bool {
+        let s = self.inner.lock().attachment_machine.state();
+        match s {
+            AttachmentState::Attaching => true,
+            AttachmentState::AttachedWeak => true,
+            AttachmentState::AttachedGood => true,
+            AttachmentState::AttachedStrong => true,
+            AttachmentState::FullyAttached => true,
+            AttachmentState::OverAttached => true,
+            _ => false,
+        }
+    }
+    pub fn is_detached(&self) -> bool {
+        let s = self.inner.lock().attachment_machine.state();
+        match s {
+            AttachmentState::Detached => true,
+            _ => false,
+        }
+    }
+
+    pub fn get_attach_timestamp(&self) -> Option<u64> {
+        self.inner.lock().attach_timestamp
+    }
+
+    pub fn get_peer_count(&self) -> u32 {
+        self.inner.lock().peer_count
+    }
+
+    fn translate_peer_input(cur: u32, max: u32) -> AttachmentInput {
+        if cur > max {
+            return AttachmentInput::TooManyPeers;
+        }
+        match cmp::min(4, 4 * cur / max) {
+            4 => AttachmentInput::FullPeers,
+            3 => AttachmentInput::StrongPeers,
+            2 => AttachmentInput::GoodPeers,
+            1 => AttachmentInput::WeakPeers,
+            0 => AttachmentInput::NoPeers,
+            _ => panic!("Invalid state"),
+        }
+    }
+    fn translate_peer_state(state: &AttachmentState) -> AttachmentInput {
+        match state {
+            AttachmentState::OverAttached => AttachmentInput::TooManyPeers,
+            AttachmentState::FullyAttached => AttachmentInput::FullPeers,
+            AttachmentState::AttachedStrong => AttachmentInput::StrongPeers,
+            AttachmentState::AttachedGood => AttachmentInput::GoodPeers,
+            AttachmentState::AttachedWeak => AttachmentInput::WeakPeers,
+            AttachmentState::Attaching => AttachmentInput::NoPeers,
+            _ => panic!("Invalid state"),
+        }
+    }
+
+    async fn update_peer_count(&self) {
+        let new_peer_state_input = {
+            let inner = self.inner.lock();
+
+            let old_peer_state_input =
+                AttachmentManager::translate_peer_state(&inner.attachment_machine.state());
+
+            let max_connections = inner.config.get().network.max_connections;
+
+            // get active peer count from routing table
+
+            let new_peer_state_input =
+                AttachmentManager::translate_peer_input(inner.peer_count, max_connections);
+
+            if old_peer_state_input == new_peer_state_input {
+                None
+            } else {
+                Some(new_peer_state_input)
+            }
+        };
+        if let Some(next_input) = new_peer_state_input {
+            let _ = self.process_input(&next_input).await;
+        }
+    }
+
+    async fn attachment_maintainer(self) {
+        trace!("attachment starting");
+        let netman = {
+            let mut inner = self.inner.lock();
+            inner.attach_timestamp = Some(intf::get_timestamp());
+            inner.network_manager.clone()
+        };
+
+        trace!("starting network");
+        let mut started = true;
+        if let Err(err) = netman.startup().await {
+            error!("network startup failed: {}", err);
+            started = false;
+        }
+
+        if started {
+            trace!("started maintaining peers");
+            while self.inner.lock().maintain_peers {
+                // tick network manager
+                if let Err(err) = netman.tick().await {
+                    error!("Error in network manager: {}", err);
+                    self.inner.lock().maintain_peers = false;
+                    break;
+                }
+
+                // xxx: ?update peer count?
+                self.update_peer_count().await;
+
+                // sleep should be at the end in case maintain_peers changes state
+                intf::sleep(1000).await;
+            }
+            trace!("stopped maintaining peers");
+
+            trace!("stopping network");
+            netman.shutdown().await;
+        }
+
+        trace!("stopping attachment");
+        let attachment_machine = self.inner.lock().attachment_machine.clone();
+        let _output = attachment_machine
+            .consume(&AttachmentInput::AttachmentStopped)
+            .await;
+        trace!("attachment stopped");
+        self.inner.lock().attach_timestamp = None;
+    }
+
+    pub async fn init(
+        &self,
+        state_change_callback: StateChangeCallback<Attachment>,
+    ) -> Result<(), String> {
+        let inner = self.inner.lock();
+        inner
+            .attachment_machine
+            .set_state_change_callback(state_change_callback);
+
+        inner.network_manager.init().await?;
+
+        Ok(())
+    }
+    pub async fn terminate(&self) {
+        // Ensure we detached
+        self.detach().await;
+        let inner = self.inner.lock();
+        inner.network_manager.terminate().await;
+    }
+
+    fn attach(&self) {
+        trace!("attach");
+        // Create long-running connection maintenance routine
+        let this = self.clone();
+        self.inner.lock().maintain_peers = true;
+        self.inner.lock().attachment_maintainer_jh =
+            Some(intf::spawn(this.attachment_maintainer()));
+    }
+
+    async fn detach(&self) {
+        trace!("detach");
+        let attachment_maintainer_jh = self.inner.lock().attachment_maintainer_jh.take();
+        if let Some(jh) = attachment_maintainer_jh {
+            // Terminate long-running connection maintenance routine
+            self.inner.lock().maintain_peers = false;
+            jh.await;
+        }
+    }
+
+    async fn handle_output(&self, output: &AttachmentOutput) {
+        match output {
+            AttachmentOutput::StartAttachment => self.attach(),
+            AttachmentOutput::StopAttachment => self.detach().await,
+        }
+    }
+
+    async fn process_input(&self, input: &AttachmentInput) -> bool {
+        let attachment_machine = self.inner.lock().attachment_machine.clone();
+        let output = attachment_machine.consume(input).await;
+        match output {
+            Err(_) => {
+                error!("invalid input for state machine: {:?}", input);
+                false
+            }
+            Ok(v) => {
+                if let Some(o) = v {
+                    self.handle_output(&o).await;
+                }
+                true
+            }
+        }
+    }
+
+    pub async fn send_state_update(&self) {
+        let attachment_machine = self.inner.lock().attachment_machine.clone();
+        attachment_machine.send_state_update().await;
+    }
+
+    pub async fn request_attach(&self) {
+        if !self.is_detached() {
+            trace!("attach request ignored");
+            return;
+        }
+        if self.process_input(&AttachmentInput::AttachRequested).await {
+            trace!("attach requested");
+        } else {
+            error!("attach request failed");
+        }
+    }
+
+    pub async fn request_detach(&self) {
+        if !self.is_attached() {
+            trace!("detach request ignored");
+            return;
+        }
+        if self.process_input(&AttachmentInput::DetachRequested).await {
+            trace!("detach requested");
+        } else {
+            error!("detach request failed");
+        }
+    }
+
+    pub fn get_state(&self) -> AttachmentState {
+        let attachment_machine = self.inner.lock().attachment_machine.clone();
+        attachment_machine.state()
+    }
+
+    pub async fn wait_for_state(&self, state: AttachmentState) {
+        loop {
+            let (current_state, eventual) = self
+                .inner
+                .lock()
+                .attachment_machine
+                .state_eventual_instance();
+            if current_state == state {
+                break;
+            }
+            if eventual.await == state {
+                break;
+            }
+        }
+    }
+}

veilid-core/src/callback_state_machine.rs

@@ -0,0 +1,125 @@
+use crate::xx::*;
+pub use rust_fsm::*;
+
+cfg_if! {
+    if #[cfg(target_arch = "wasm32")] {
+        pub type StateChangeCallback<T> = Arc<
+            dyn Fn(<T as StateMachineImpl>::State, <T as StateMachineImpl>::State) -> SystemPinBoxFuture<()>
+                + 'static,
+        >;
+    } else {
+        pub type StateChangeCallback<T> = Arc<
+            dyn Fn(<T as StateMachineImpl>::State, <T as StateMachineImpl>::State) -> SystemPinBoxFuture<()>
+                + Send
+                + Sync
+                + 'static,
+        >;
+    }
+}
+
+struct CallbackStateMachineInner<T>
+where
+    T: StateMachineImpl,
+    T::State: Copy + Unpin + core::fmt::Debug,
+{
+    state: T::State,
+    callback: Option<StateChangeCallback<T>>,
+    eventual: EventualValueClone<T::State>,
+}
+
+impl<T> core::fmt::Debug for CallbackStateMachineInner<T>
+where
+    T: StateMachineImpl,
+    T::State: Copy + Unpin + core::fmt::Debug,
+{
+    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+        write!(f, "CallbackStateMachineInner(state: {:?})", self.state)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct CallbackStateMachine<T>
+where
+    T: StateMachineImpl,
+    T::State: Copy + Unpin + core::fmt::Debug,
+{
+    inner: Arc<Mutex<CallbackStateMachineInner<T>>>,
+}
+
+impl<T> CallbackStateMachine<T>
+where
+    T: StateMachineImpl,
+    T::State: Copy + Unpin + core::fmt::Debug,
+{
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(CallbackStateMachineInner {
+                state: T::INITIAL_STATE,
+                callback: None,
+                eventual: EventualValueClone::new(),
+            })),
+        }
+    }
+
+    pub fn set_state_change_callback(&self, callback: StateChangeCallback<T>) {
+        self.inner.lock().callback = Some(callback);
+    }
+
+    // pub fn clear_state_change_callback(&self) {
+    //     self.inner.lock().callback = None;
+    // }
+    pub fn state_eventual_instance(&self) -> (T::State, EventualValueCloneFuture<T::State>) {
+        let inner = self.inner.lock();
+        (inner.state, inner.eventual.instance())
+    }
+
+    pub async fn send_state_update(&self) {
+        let (state, callback, eventual) = {
+            let mut inner = self.inner.lock();
+            let eventual =
+                core::mem::replace(&mut inner.eventual, EventualValueClone::<T::State>::new());
+            (inner.state, inner.callback.clone(), eventual)
+        };
+        if let Some(cb) = callback {
+            cb(state, state).await;
+        }
+        eventual.resolve(state).await;
+    }
+
+    pub async fn consume(&self, input: &T::Input) -> Result<Option<T::Output>, ()> {
+        let current_state = self.inner.lock().state;
+
+        if let Some(new_state) = T::transition(&current_state, &input) {
+            let output = T::output(&current_state, input);
+            let old_state = current_state;
+            let (callback, eventual) = {
+                let mut inner = self.inner.lock();
+                inner.state = new_state;
+                let eventual =
+                    core::mem::replace(&mut inner.eventual, EventualValueClone::<T::State>::new());
+                (inner.callback.clone(), eventual)
+            };
+            if let Some(cb) = callback {
+                cb(old_state, new_state).await;
+            }
+            eventual.resolve(new_state).await;
+            Ok(output)
+        } else {
+            Err(())
+        }
+    }
+
+    pub fn state(&self) -> T::State {
+        self.inner.lock().state
+    }
+}
+
+impl<T> Default for CallbackStateMachine<T>
+where
+    T: StateMachineImpl,
+    T::State: Copy + Unpin + core::fmt::Debug,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}

veilid-core/src/connection_table.rs

@@ -0,0 +1,116 @@
+use crate::intf::*;
+use crate::xx::*;
+use crate::*;
+
+#[derive(Clone, Debug)]
+pub struct ConnectionTableEntry {
+    pub conn: NetworkConnection,
+    pub established_time: u64,
+    pub last_message_sent_time: Option<u64>,
+    pub last_message_recv_time: Option<u64>,
+    pub stopper: Eventual,
+}
+
+impl PartialEq for ConnectionTableEntry {
+    fn eq(&self, other: &ConnectionTableEntry) -> bool {
+        if self.conn != other.conn {
+            return false;
+        }
+        if self.established_time != other.established_time {
+            return false;
+        }
+        if self.last_message_sent_time != other.last_message_sent_time {
+            return false;
+        }
+        if self.last_message_recv_time != other.last_message_recv_time {
+            return false;
+        }
+        return true;
+    }
+}
+
+#[derive(Debug)]
+pub struct ConnectionTableInner {
+    conn_by_addr: BTreeMap<ConnectionDescriptor, ConnectionTableEntry>,
+}
+
+#[derive(Clone)]
+pub struct ConnectionTable {
+    inner: Arc<Mutex<ConnectionTableInner>>,
+}
+impl core::fmt::Debug for ConnectionTable {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.debug_struct("ConnectionTable")
+            .field("inner", &*self.inner.lock())
+            .finish()
+    }
+}
+
+impl ConnectionTable {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(ConnectionTableInner {
+                conn_by_addr: BTreeMap::new(),
+            })),
+        }
+    }
+
+    pub fn add_connection(
+        &self,
+        descriptor: ConnectionDescriptor,
+        conn: NetworkConnection,
+    ) -> Result<ConnectionTableEntry, ()> {
+        assert_ne!(
+            descriptor.protocol_type(),
+            ProtocolType::UDP,
+            "Only connection oriented protocols go in the table!"
+        );
+
+        let mut inner = self.inner.lock();
+        if inner.conn_by_addr.contains_key(&descriptor) {
+            return Err(());
+        }
+
+        let timestamp = get_timestamp();
+
+        let entry = ConnectionTableEntry {
+            conn: conn,
+            established_time: timestamp,
+            last_message_sent_time: None,
+            last_message_recv_time: None,
+            stopper: Eventual::new(),
+        };
+        let res = inner.conn_by_addr.insert(descriptor, entry.clone());
+        assert!(res.is_none());
+        Ok(entry)
+    }
+
+    pub fn get_connection(
+        &self,
+        descriptor: &ConnectionDescriptor,
+    ) -> Option<ConnectionTableEntry> {
+        let inner = self.inner.lock();
+        match inner.conn_by_addr.get(&descriptor) {
+            Some(v) => Some(v.clone()),
+            None => None,
+        }
+    }
+
+    pub fn connection_count(&self) -> usize {
+        let inner = self.inner.lock();
+        inner.conn_by_addr.len()
+    }
+
+    pub fn remove_connection(
+        &self,
+        descriptor: &ConnectionDescriptor,
+    ) -> Result<ConnectionTableEntry, ()> {
+        let mut inner = self.inner.lock();
+
+        let res = inner.conn_by_addr.remove(&descriptor);
+        match res {
+            Some(v) => Ok(v.clone()),
+            None => Err(()),
+        }
+    }
+}

veilid-core/src/dht/crypto.rs

@@ -0,0 +1,287 @@
+use super::key::*;
+use crate::intf::*;
+use crate::xx::*;
+use crate::*;
+use chacha20poly1305 as ch;
+use chacha20poly1305::aead::{AeadInPlace, NewAead};
+use core::convert::TryInto;
+use curve25519_dalek as cd;
+use ed25519_dalek as ed;
+use serde::{Deserialize, Serialize};
+use serde_big_array::*;
+use uluru;
+use x25519_dalek as xd;
+
+pub type SharedSecret = [u8; 32];
+pub type Nonce = [u8; 24];
+
+const DH_CACHE_SIZE: usize = 1024;
+pub const ENCRYPTION_OVERHEAD: usize = 16;
+
+big_array! {
+    BigArray;
+    DH_CACHE_SIZE
+}
+
+type DHCache = uluru::LRUCache<DHCacheEntry, DH_CACHE_SIZE>;
+
+#[derive(Serialize, Deserialize)]
+struct DHCacheEntry {
+    key: DHTKey,
+    secret: DHTKeySecret,
+    shared_secret: SharedSecret,
+}
+
+fn cache_to_bytes(cache: &DHCache) -> Vec<u8> {
+    let cnt: usize = cache.len();
+    let mut out: Vec<u8> = Vec::with_capacity(cnt * (32 + 32 + 32));
+    for e in cache.iter() {
+        out.extend(&e.key.bytes);
+        out.extend(&e.secret.bytes);
+        out.extend(&e.shared_secret);
+    }
+    let mut rev: Vec<u8> = Vec::with_capacity(out.len());
+    for d in out.chunks(32 + 32 + 32).rev() {
+        rev.extend(d);
+    }
+    rev
+}
+
+fn bytes_to_cache(bytes: &[u8], cache: &mut DHCache) {
+    for d in bytes.chunks(32 + 32 + 32) {
+        let e = DHCacheEntry {
+            key: DHTKey::new(d[0..32].try_into().expect("asdf")),
+            secret: DHTKeySecret::new(d[32..64].try_into().expect("asdf")),
+            shared_secret: d[64..96].try_into().expect("asdf"),
+        };
+        cache.insert(e);
+    }
+}
+
+struct CryptoInner {
+    table_store: TableStore,
+    node_id: DHTKey,
+    node_id_secret: DHTKeySecret,
+    dh_cache: DHCache,
+    flush_future: Option<SystemPinBoxFuture<()>>,
+}
+
+#[derive(Clone)]
+pub struct Crypto {
+    config: VeilidConfig,
+    inner: Arc<Mutex<CryptoInner>>,
+}
+
+impl Crypto {
+    fn new_inner(table_store: TableStore) -> CryptoInner {
+        CryptoInner {
+            table_store: table_store,
+            node_id: Default::default(),
+            node_id_secret: Default::default(),
+            dh_cache: DHCache::default(),
+            flush_future: None,
+        }
+    }
+
+    pub fn new(config: VeilidConfig, table_store: TableStore) -> Self {
+        Self {
+            config: config,
+            inner: Arc::new(Mutex::new(Self::new_inner(table_store))),
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        trace!("Crypto::init");
+
+        // make local copy of node id for easy access
+        let mut inner = self.inner.lock();
+        let c = self.config.get();
+        inner.node_id = c.network.node_id;
+        inner.node_id_secret = c.network.node_id_secret;
+
+        // load caches if they are valid for this node id
+        let mut db = inner.table_store.open("crypto_caches", 1).await?;
+        let caches_valid = match db.load(0, b"node_id").await? {
+            Some(v) => v.as_slice() == inner.node_id.bytes,
+            None => false,
+        };
+        if caches_valid {
+            match db.load(0, b"dh_cache").await? {
+                Some(b) => {
+                    bytes_to_cache(&b, &mut inner.dh_cache);
+                }
+                None => (),
+            };
+        } else {
+            drop(db);
+            inner.table_store.delete("crypto_caches").await?;
+            db = inner.table_store.open("crypto_caches", 1).await?;
+            db.store(0, b"node_id", &inner.node_id.bytes).await?;
+        }
+
+        // Schedule flushing
+        let this = self.clone();
+        inner.flush_future = Some(Box::pin(interval(60000, move || {
+            let this = this.clone();
+            async move {
+                if let Err(e) = this.flush().await {
+                    warn!("flush failed: {}", e);
+                }
+            }
+        })));
+
+        Ok(())
+    }
+
+    pub async fn flush(&self) -> Result<(), String> {
+        //trace!("Crypto::flush");
+        let (table_store, cache_bytes) = {
+            let inner = self.inner.lock();
+            let cache_bytes = cache_to_bytes(&inner.dh_cache);
+            (inner.table_store.clone(), cache_bytes)
+        };
+
+        let db = table_store.open("crypto_caches", 1).await?;
+        db.store(0, b"dh_cache", &cache_bytes).await?;
+
+        Ok(())
+    }
+
+    pub async fn terminate(&self) {
+        trace!("Crypto::terminate");
+        let flush_future = self.inner.lock().flush_future.take();
+        if let Some(f) = flush_future {
+            f.await;
+        }
+        trace!("starting termination flush");
+        match self.flush().await {
+            Ok(_) => {
+                trace!("finished termination flush");
+                ()
+            }
+            Err(e) => {
+                error!("failed termination flush: {}", e);
+                ()
+            }
+        };
+    }
+
+    fn ed25519_to_x25519_pk(key: &ed::PublicKey) -> Result<xd::PublicKey, ()> {
+        let bytes = key.to_bytes();
+        let compressed = cd::edwards::CompressedEdwardsY(bytes);
+        let point = compressed.decompress().ok_or(())?;
+        let mp = point.to_montgomery();
+        Ok(xd::PublicKey::from(mp.to_bytes()))
+    }
+    fn ed25519_to_x25519_sk(key: &ed::SecretKey) -> Result<xd::StaticSecret, ()> {
+        let exp = ed::ExpandedSecretKey::from(key);
+        let bytes: [u8; ed::EXPANDED_SECRET_KEY_LENGTH] = exp.to_bytes();
+        let lowbytes: [u8; 32] = bytes[0..32].try_into().map_err(drop)?;
+        Ok(xd::StaticSecret::from(lowbytes))
+    }
+
+    pub fn cached_dh(&self, key: &DHTKey, secret: &DHTKeySecret) -> Result<SharedSecret, ()> {
+        if let Some(c) = self
+            .inner
+            .lock()
+            .dh_cache
+            .find(|entry| entry.key == *key && entry.secret == *secret)
+        {
+            return Ok(c.shared_secret);
+        }
+
+        let ss = Self::compute_dh(key, secret)?;
+        self.inner.lock().dh_cache.insert(DHCacheEntry {
+            key: key.clone(),
+            secret: secret.clone(),
+            shared_secret: ss.clone(),
+        });
+        Ok(ss)
+    }
+
+    ///////////
+    // These are safe to use regardless of initialization status
+
+    pub fn compute_dh(key: &DHTKey, secret: &DHTKeySecret) -> Result<SharedSecret, ()> {
+        assert!(key.valid);
+        assert!(secret.valid);
+        let pk_ed = match ed::PublicKey::from_bytes(&key.bytes) {
+            Ok(v) => v,
+            Err(e) => {
+                trace!("compute_dh error: {:?}", e);
+                return Err(());
+            }
+        };
+        let pk_xd = Self::ed25519_to_x25519_pk(&pk_ed)?;
+        let sk_ed = match ed::SecretKey::from_bytes(&secret.bytes) {
+            Ok(v) => v,
+            Err(e) => {
+                trace!("compute_dh error: {:?}", e);
+                return Err(());
+            }
+        };
+        let sk_xd = Self::ed25519_to_x25519_sk(&sk_ed)?;
+        Ok(sk_xd.diffie_hellman(&pk_xd).to_bytes())
+    }
+
+    pub fn get_random_nonce() -> Nonce {
+        let mut nonce = [0u8; 24];
+        let _ = random_bytes(&mut nonce).unwrap();
+        nonce
+    }
+
+    pub fn get_random_secret() -> SharedSecret {
+        let mut s = [0u8; 32];
+        let _ = random_bytes(&mut s).unwrap();
+        s
+    }
+
+    pub fn decrypt_in_place(
+        body: &mut Vec<u8>,
+        nonce: &Nonce,
+        shared_secret: &SharedSecret,
+        associated_data: Option<&[u8]>,
+    ) -> Result<(), ()> {
+        let key = ch::Key::from(shared_secret.clone());
+        let xnonce = ch::XNonce::from(nonce.clone());
+        let aead = ch::XChaCha20Poly1305::new(&key);
+        aead.decrypt_in_place(&xnonce, associated_data.unwrap_or(b""), body)
+            .map_err(|e| trace!("decryption failure: {}", e))
+    }
+
+    pub fn decrypt(
+        body: &[u8],
+        nonce: &Nonce,
+        shared_secret: &SharedSecret,
+        associated_data: Option<&[u8]>,
+    ) -> Result<Vec<u8>, ()> {
+        let mut out = body.to_vec();
+        let _ = Self::decrypt_in_place(&mut out, nonce, shared_secret, associated_data)?;
+        Ok(out)
+    }
+
+    pub fn encrypt_in_place(
+        body: &mut Vec<u8>,
+        nonce: &Nonce,
+        shared_secret: &SharedSecret,
+        associated_data: Option<&[u8]>,
+    ) -> Result<(), ()> {
+        let key = ch::Key::from(shared_secret.clone());
+        let xnonce = ch::XNonce::from(nonce.clone());
+        let aead = ch::XChaCha20Poly1305::new(&key);
+
+        aead.encrypt_in_place(&xnonce, associated_data.unwrap_or(b""), body)
+            .map_err(|e| trace!("encryption failure: {}", e))
+    }
+
+    pub fn encrypt(
+        body: &[u8],
+        nonce: &Nonce,
+        shared_secret: &SharedSecret,
+        associated_data: Option<&[u8]>,
+    ) -> Result<Vec<u8>, ()> {
+        let mut out = body.to_vec();
+        let _ = Self::encrypt_in_place(&mut out, nonce, shared_secret, associated_data)?;
+        Ok(out)
+    }
+}

veilid-core/src/dht/envelope.rs

@@ -0,0 +1,267 @@
+use super::crypto::*;
+use super::key::*;
+use crate::xx::*;
+use core::convert::TryInto;
+
+// #[repr(C, packed)]
+// struct EnvelopeHeader {
+//     // Size is at least 8 bytes. Depending on the version specified, the size may vary and should be case to the appropriate struct
+//     magic: [u8; 4],              // 0x00: 0x56 0x4C 0x49 0x44 ("VLID")
+//     version: u8,                 // 0x04: 0 = EnvelopeV0
+//     min_version: u8,             // 0x05: 0 = EnvelopeV0
+//     max_version: u8,             // 0x06: 0 = EnvelopeV0
+//     reserved: u8,                // 0x07: Reserved for future use
+// }
+
+// #[repr(C, packed)]
+// struct EnvelopeV0 {
+//     // Size is 106 bytes.
+//     magic: [u8; 4],              // 0x00: 0x56 0x4C 0x49 0x44 ("VLID")
+//     version: u8,                 // 0x04: 0 = EnvelopeV0
+//     min_version: u8,             // 0x05: 0 = EnvelopeV0
+//     max_version: u8,             // 0x06: 0 = EnvelopeV0
+//     reserved: u8,                // 0x07: Reserved for future use
+//     size: u16,                   // 0x08: Total size of the envelope including the encrypted operations message. Maximum size is 65,507 bytes, which is the data size limit for a single UDP message on IPv4.
+//     timestamp: u64,              // 0x0A: Duration since UNIX_EPOCH in microseconds when this message is sent. Messages older than 10 seconds are dropped.
+//     nonce: [u8; 24],             // 0x12: Random nonce for replay protection and for x25519
+//     sender_id: [u8; 32],         // 0x2A: Node ID of the message source, which is the Ed25519 public key of the sender (must be verified with find_node if this is a new node_id/address combination)
+//     recipient_id: [u8; 32],      // 0x4A: Node ID of the intended recipient, which is the Ed25519 public key of the recipient (must be the receiving node, or a relay lease holder)
+//                                  // 0x6A: message is appended (operations)
+//                                  //       encrypted by XChaCha20Poly1305(nonce,x25519(recipient_id, sender_secret_key))
+//                                  //       decryptable by XChaCha20Poly1305(nonce,x25519(sender_id, recipient_secret_key))
+//                                  //       entire header needs to be included in message digest, relays are not allowed to modify the envelope without invalidating the signature.
+// }
+
+pub const MAX_ENVELOPE_SIZE: usize = 65507;
+pub const MIN_ENVELOPE_SIZE: usize = 106;
+pub const AEAD_ADDITIONAL_SIZE: usize = 16;
+pub const ENVELOPE_MAGIC: &[u8; 4] = b"VLID";
+pub const MIN_VERSION: u8 = 0u8;
+pub const MAX_VERSION: u8 = 0u8;
+pub type EnvelopeNonce = [u8; 24];
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct Envelope {
+    version: u8,
+    min_version: u8,
+    max_version: u8,
+    timestamp: u64,
+    nonce: EnvelopeNonce,
+    sender_id: DHTKey,
+    recipient_id: DHTKey,
+}
+
+impl Envelope {
+    pub fn new(
+        version: u8,
+        timestamp: u64,
+        nonce: EnvelopeNonce,
+        sender_id: DHTKey,
+        recipient_id: DHTKey,
+    ) -> Self {
+        assert!(sender_id.valid);
+        assert!(recipient_id.valid);
+
+        assert!(version >= MIN_VERSION);
+        assert!(version <= MAX_VERSION);
+        Self {
+            version: version,
+            min_version: MIN_VERSION,
+            max_version: MAX_VERSION,
+            timestamp: timestamp,
+            nonce: nonce,
+            sender_id: sender_id,
+            recipient_id: recipient_id,
+        }
+    }
+
+    pub fn from_data(data: &[u8]) -> Result<Envelope, ()> {
+        // Ensure we are at least the length of the envelope
+        if data.len() < MIN_ENVELOPE_SIZE {
+            trace!("envelope too small: len={}", data.len());
+            return Err(());
+        }
+
+        // Verify magic number
+        let magic: [u8; 4] = data[0x00..0x04].try_into().map_err(drop)?;
+        if magic != *ENVELOPE_MAGIC {
+            trace!("bad magic number: len={:?}", magic);
+            return Err(());
+        }
+
+        // Check version
+        let version = data[0x04];
+        if version > MAX_VERSION || version < MIN_VERSION {
+            trace!("unsupported protocol version: version={}", version);
+            return Err(());
+        }
+
+        // Get min version
+        let min_version = data[0x05];
+        if min_version > version {
+            trace!(
+                "invalid version information in envelope: min_version={}, version={}",
+                min_version,
+                version,
+            );
+            return Err(());
+        }
+
+        // Get max version
+        let max_version = data[0x06];
+        if version > max_version || min_version > max_version {
+            trace!(
+                "invalid version information in envelope: min_version={}, version={}, max_version={}",
+                min_version,
+                version,
+                max_version
+            );
+            return Err(());
+        }
+
+        // Get size and ensure it matches the size of the envelope and is less than the maximum message size
+        let size: u16 = u16::from_le_bytes(data[0x08..0x0A].try_into().map_err(drop)?);
+        if (size as usize) > MAX_ENVELOPE_SIZE {
+            trace!("envelope size is too large: size={}", size);
+            return Err(());
+        }
+        if (size as usize) != data.len() {
+            trace!(
+                "size doesn't match envelope size: size={} data.len()={}",
+                size,
+                data.len()
+            );
+            return Err(());
+        }
+
+        // Get the timestamp
+        let timestamp: u64 = u64::from_le_bytes(data[0x0A..0x12].try_into().map_err(drop)?);
+
+        // Get nonce and sender node id
+        let nonce: EnvelopeNonce = data[0x12..0x2A].try_into().map_err(drop)?;
+        let sender_id: [u8; 32] = data[0x2A..0x4A].try_into().map_err(drop)?;
+        let recipient_id: [u8; 32] = data[0x4A..0x6A].try_into().map_err(drop)?;
+        let sender_id_dhtkey = DHTKey::new(sender_id);
+        let recipient_id_dhtkey = DHTKey::new(recipient_id);
+
+        // Ensure sender_id and recipient_id are not the same
+        if sender_id_dhtkey == recipient_id_dhtkey {
+            trace!(
+                "sender_id should not be same as recipient_id: {}",
+                recipient_id_dhtkey.encode()
+            );
+            return Err(());
+        }
+
+        // Return envelope
+        Ok(Self {
+            version: version,
+            min_version: min_version,
+            max_version: max_version,
+            timestamp: timestamp,
+            nonce: nonce,
+            sender_id: sender_id_dhtkey,
+            recipient_id: recipient_id_dhtkey,
+        })
+    }
+
+    pub fn decrypt_body(
+        &self,
+        crypto: Crypto,
+        data: &[u8],
+        node_id_secret: &DHTKeySecret,
+    ) -> Result<Vec<u8>, ()> {
+        // Get DH secret
+        let dh_secret = crypto.cached_dh(&self.sender_id, node_id_secret)?;
+
+        // Decrypt message and authenticate, including the envelope header as associated data to authenticate
+        let body = Crypto::decrypt(
+            &data[0x6A..],
+            &self.nonce,
+            &dh_secret,
+            Some(&data[0..MIN_ENVELOPE_SIZE]),
+        )?;
+
+        Ok(body)
+    }
+
+    pub fn to_encrypted_data(
+        &self,
+        crypto: Crypto,
+        body: &[u8],
+        node_id_secret: &DHTKeySecret,
+    ) -> Result<Vec<u8>, ()> {
+        // Ensure sender node id is valid
+        if !self.sender_id.valid {
+            return Err(());
+        }
+        // Ensure recipient node id is valid
+        if !self.recipient_id.valid {
+            return Err(());
+        }
+
+        // Ensure body isn't too long
+        let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE + AEAD_ADDITIONAL_SIZE;
+        if envelope_size > MAX_ENVELOPE_SIZE {
+            return Err(());
+        }
+        let mut data: Vec<u8> = Vec::with_capacity(envelope_size);
+        data.resize(envelope_size, 0u8);
+
+        // Write magic
+        data[0x00..0x04].copy_from_slice(ENVELOPE_MAGIC);
+        // Write version
+        data[0x04] = self.version;
+        // Write min version
+        data[0x05] = self.min_version;
+        // Write max version
+        data[0x06] = self.max_version;
+        // Write size
+        data[0x08..0x0A].copy_from_slice(&(envelope_size as u16).to_le_bytes());
+        // Write timestamp
+        data[0x0A..0x12].copy_from_slice(&self.timestamp.to_le_bytes());
+        // Write nonce
+        data[0x12..0x2A].copy_from_slice(&self.nonce);
+        // Write sender node id
+        data[0x2A..0x4A].copy_from_slice(&self.sender_id.bytes);
+        // Write recipient node id
+        data[0x4A..0x6A].copy_from_slice(&self.recipient_id.bytes);
+
+        // Generate dh secret
+        let dh_secret = crypto
+            .cached_dh(&self.recipient_id, node_id_secret)
+            .map_err(drop)?;
+
+        // Encrypt and authenticate message
+        let encrypted_body =
+            Crypto::encrypt(body, &self.nonce, &dh_secret, Some(&data[0..0x6A])).map_err(drop)?;
+
+        // Write body
+        data[0x6A..].copy_from_slice(encrypted_body.as_slice());
+
+        Ok(data)
+    }
+
+    pub fn get_version(&self) -> u8 {
+        self.version
+    }
+
+    pub fn get_min_max_version(&self) -> (u8, u8) {
+        (self.min_version, self.max_version)
+    }
+
+    pub fn get_timestamp(&self) -> u64 {
+        self.timestamp
+    }
+
+    pub fn get_nonce(&self) -> EnvelopeNonce {
+        self.nonce
+    }
+
+    pub fn get_sender_id(&self) -> DHTKey {
+        self.sender_id
+    }
+    pub fn get_recipient_id(&self) -> DHTKey {
+        self.recipient_id
+    }
+}

veilid-core/src/dht/key.rs

@@ -0,0 +1,432 @@
+use crate::xx::*;
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::convert::{TryFrom, TryInto};
+use core::fmt;
+use hex;
+
+use crate::veilid_rng::*;
+use ed25519_dalek::{Keypair, PublicKey, Signature};
+use serde::{Deserialize, Serialize};
+
+use data_encoding::BASE64URL_NOPAD;
+use digest::generic_array::typenum::U64;
+use digest::{Digest, Output};
+use generic_array::GenericArray;
+
+//////////////////////////////////////////////////////////////////////
+
+#[allow(dead_code)]
+pub const DHT_KEY_LENGTH: usize = 32;
+#[allow(dead_code)]
+pub const DHT_KEY_LENGTH_ENCODED: usize = 43;
+#[allow(dead_code)]
+pub const DHT_KEY_SECRET_LENGTH: usize = 32;
+#[allow(dead_code)]
+pub const DHT_KEY_SECRET_LENGTH_ENCODED: usize = 43;
+#[allow(dead_code)]
+pub const DHT_SIGNATURE_LENGTH: usize = 64;
+#[allow(dead_code)]
+pub const DHT_SIGNATURE_LENGTH_ENCODED: usize = 86;
+
+//////////////////////////////////////////////////////////////////////
+
+macro_rules! byte_array_type {
+    ($name:ident, $size:expr) => {
+        #[derive(Clone, Copy)]
+        pub struct $name {
+            pub bytes: [u8; $size],
+            pub valid: bool,
+        }
+
+        impl Serialize for $name {
+            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+            where
+                S: serde::Serializer,
+            {
+                let s: String;
+                if self.valid {
+                    s = self.encode();
+                } else {
+                    s = "".to_owned();
+                }
+                s.serialize(serializer)
+            }
+        }
+
+        impl<'de> Deserialize<'de> for $name {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: serde::Deserializer<'de>,
+            {
+                let s = String::deserialize(deserializer)?;
+                if s == "" {
+                    return Ok($name::default());
+                }
+                $name::try_decode(s.as_str()).map_err(|e| serde::de::Error::custom(e))
+            }
+        }
+
+        impl $name {
+            pub fn new(bytes: [u8; $size]) -> Self {
+                Self {
+                    bytes: bytes,
+                    valid: true,
+                }
+            }
+
+            pub fn try_from_vec(v: Vec<u8>) -> Result<Self, String> {
+                let mut this = Self {
+                    bytes: [0u8; $size],
+                    valid: true,
+                };
+
+                if v.len() != $size {
+                    return Err(format!(
+                        "Expected a Vec of length {} but it was {}",
+                        $size,
+                        v.len()
+                    ));
+                }
+
+                for n in 0..v.len() {
+                    this.bytes[n] = v[n];
+                }
+
+                Ok(this)
+            }
+
+            pub fn bit(&self, index: usize) -> bool {
+                assert!(index < ($size * 8));
+                let bi = index / 8;
+                let ti = 7 - (index % 8);
+                ((self.bytes[bi] >> ti) & 1) != 0
+            }
+
+            pub fn first_nonzero_bit(&self) -> Option<usize> {
+                for i in 0..$size {
+                    let b = self.bytes[i];
+                    if b != 0 {
+                        for n in 0..8 {
+                            if ((b >> (7 - n)) & 1u8) != 0u8 {
+                                return Some((i * 8) + n);
+                            }
+                        }
+                        panic!("wtf")
+                    }
+                }
+                None
+            }
+
+            pub fn nibble(&self, index: usize) -> u8 {
+                assert!(index < ($size * 2));
+                let bi = index / 2;
+                if index & 1 == 0 {
+                    (self.bytes[bi] >> 4) & 0xFu8
+                } else {
+                    self.bytes[bi] & 0xFu8
+                }
+            }
+
+            pub fn first_nonzero_nibble(&self) -> Option<(usize, u8)> {
+                for i in 0..($size * 2) {
+                    let n = self.nibble(i);
+                    if n != 0 {
+                        return Some((i, n));
+                    }
+                }
+                None
+            }
+
+            pub fn encode(&self) -> String {
+                assert!(self.valid);
+                BASE64URL_NOPAD.encode(&self.bytes)
+            }
+
+            pub fn try_decode(input: &str) -> Result<Self, String> {
+                let mut bytes = [0u8; $size];
+
+                let res = BASE64URL_NOPAD.decode_len(input.len());
+                match res {
+                    Ok(v) => {
+                        if v != $size {
+                            return Err("Incorrect length in decode".to_owned());
+                        }
+                    }
+                    Err(_) => {
+                        return Err("Failed to decode".to_owned());
+                    }
+                }
+
+                let res = BASE64URL_NOPAD.decode_mut(input.as_bytes(), &mut bytes);
+                match res {
+                    Ok(_) => Ok(Self::new(bytes)),
+                    Err(_) => Err("Failed to decode".to_owned()),
+                }
+            }
+        }
+        impl PartialOrd for $name {
+            fn partial_cmp(&self, other: &$name) -> Option<Ordering> {
+                Some(self.cmp(other))
+            }
+        }
+        impl Ord for $name {
+            fn cmp(&self, other: &$name) -> Ordering {
+                if !self.valid && !other.valid {
+                    return Ordering::Equal;
+                }
+                if !self.valid && other.valid {
+                    return Ordering::Less;
+                }
+                if self.valid && !other.valid {
+                    return Ordering::Greater;
+                }
+
+                for n in 0..$size {
+                    if self.bytes[n] < other.bytes[n] {
+                        return Ordering::Less;
+                    }
+                    if self.bytes[n] > other.bytes[n] {
+                        return Ordering::Greater;
+                    }
+                }
+                Ordering::Equal
+            }
+        }
+        impl PartialEq<$name> for $name {
+            fn eq(&self, other: &$name) -> bool {
+                if self.valid != other.valid {
+                    return false;
+                }
+                for n in 0..$size {
+                    if self.bytes[n] != other.bytes[n] {
+                        return false;
+                    }
+                }
+                true
+            }
+        }
+        impl Eq for $name {}
+        impl Default for $name {
+            fn default() -> Self {
+                let mut this = $name::new([0u8; $size]);
+                this.valid = false;
+                this
+            }
+        }
+        impl fmt::Display for $name {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                write!(f, "{}", String::from(self))
+            }
+        }
+
+        impl fmt::Debug for $name {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                write!(f, concat!(stringify!($name), "("))?;
+                write!(f, "{}", String::from(self))?;
+                write!(f, ")")
+            }
+        }
+
+        impl From<&$name> for String {
+            fn from(value: &$name) -> Self {
+                if !value.valid {
+                    return "".to_owned();
+                }
+                let mut s = String::new();
+                for n in 0..($size / 8) {
+                    let b: [u8; 8] = value.bytes[n * 8..(n + 1) * 8].try_into().unwrap();
+                    s.push_str(hex::encode(b).as_str());
+                }
+                s
+            }
+        }
+
+        impl TryFrom<String> for $name {
+            type Error = String;
+            fn try_from(value: String) -> Result<Self, Self::Error> {
+                $name::try_from(value.as_str())
+            }
+        }
+
+        impl TryFrom<&str> for $name {
+            type Error = String;
+            fn try_from(value: &str) -> Result<Self, Self::Error> {
+                let mut out = $name::default();
+                if value == "" {
+                    return Ok(out);
+                }
+                if value.len() != ($size * 2) {
+                    return Err(concat!(stringify!($name), " is incorrect length").to_owned());
+                }
+                match hex::decode_to_slice(value, &mut out.bytes) {
+                    Ok(_) => {
+                        out.valid = true;
+                        Ok(out)
+                    }
+                    Err(err) => Err(format!("{}", err)),
+                }
+            }
+        }
+    };
+}
+
+byte_array_type!(DHTKey, DHT_KEY_LENGTH);
+byte_array_type!(DHTKeySecret, DHT_KEY_SECRET_LENGTH);
+byte_array_type!(DHTSignature, DHT_SIGNATURE_LENGTH);
+byte_array_type!(DHTKeyDistance, DHT_KEY_LENGTH);
+
+/////////////////////////////////////////
+
+struct Blake3Digest512 {
+    dig: blake3::Hasher,
+}
+
+impl Digest for Blake3Digest512 {
+    type OutputSize = U64;
+
+    fn new() -> Self {
+        Self {
+            dig: blake3::Hasher::new(),
+        }
+    }
+
+    fn update(&mut self, data: impl AsRef<[u8]>) {
+        self.dig.update(data.as_ref());
+    }
+
+    fn chain(mut self, data: impl AsRef<[u8]>) -> Self
+    where
+        Self: Sized,
+    {
+        self.update(data);
+        self
+    }
+
+    fn finalize(self) -> Output<Self> {
+        let mut b = [0u8; 64];
+        self.dig.finalize_xof().fill(&mut b);
+        let mut out = GenericArray::<u8, U64>::default();
+        for n in 0..64 {
+            out[n] = b[n];
+        }
+        out
+    }
+
+    fn finalize_reset(&mut self) -> Output<Self> {
+        let mut b = [0u8; 64];
+        self.dig.finalize_xof().fill(&mut b);
+        let mut out = GenericArray::<u8, U64>::default();
+        for n in 0..64 {
+            out[n] = b[n];
+        }
+        self.reset();
+        out
+    }
+
+    fn reset(&mut self) {
+        self.dig.reset();
+    }
+
+    fn output_size() -> usize {
+        64
+    }
+
+    fn digest(data: &[u8]) -> Output<Self> {
+        let mut dig = blake3::Hasher::new();
+        dig.update(data);
+        let mut b = [0u8; 64];
+        dig.finalize_xof().fill(&mut b);
+        let mut out = GenericArray::<u8, U64>::default();
+        for n in 0..64 {
+            out[n] = b[n];
+        }
+        out
+    }
+}
+
+/////////////////////////////////////////
+
+pub fn generate_secret() -> (DHTKey, DHTKeySecret) {
+    let mut csprng = VeilidRng {};
+    let keypair = Keypair::generate(&mut csprng);
+    let dht_key = DHTKey::new(keypair.public.to_bytes());
+    let dht_key_secret = DHTKeySecret::new(keypair.secret.to_bytes());
+
+    (dht_key, dht_key_secret)
+}
+
+pub fn sign(
+    dht_key: &DHTKey,
+    dht_key_secret: &DHTKeySecret,
+    data: &[u8],
+) -> Result<DHTSignature, String> {
+    assert!(dht_key.valid);
+    assert!(dht_key_secret.valid);
+
+    let mut kpb: [u8; DHT_KEY_SECRET_LENGTH + DHT_KEY_LENGTH] =
+        [0u8; DHT_KEY_SECRET_LENGTH + DHT_KEY_LENGTH];
+
+    kpb[..DHT_KEY_SECRET_LENGTH].copy_from_slice(&dht_key_secret.bytes);
+    kpb[DHT_KEY_SECRET_LENGTH..].copy_from_slice(&dht_key.bytes);
+    let keypair = Keypair::from_bytes(&kpb).map_err(|_| "Keypair is invalid".to_owned())?;
+
+    let mut dig = Blake3Digest512::new();
+    dig.update(data);
+
+    let sig = keypair
+        .sign_prehashed(dig, None)
+        .map_err(|_| "Signature failed".to_owned())?;
+
+    let dht_sig = DHTSignature::new(sig.to_bytes().clone());
+    Ok(dht_sig)
+}
+
+pub fn verify(dht_key: &DHTKey, data: &[u8], signature: &DHTSignature) -> Result<(), String> {
+    assert!(dht_key.valid);
+    assert!(signature.valid);
+    let pk =
+        PublicKey::from_bytes(&dht_key.bytes).map_err(|_| "Public key is invalid".to_owned())?;
+    let sig =
+        Signature::from_bytes(&signature.bytes).map_err(|_| "Signature is invalid".to_owned())?;
+
+    let mut dig = Blake3Digest512::new();
+    dig.update(data);
+
+    pk.verify_prehashed(dig, None, &sig)
+        .map_err(|_| "Verification failed".to_owned())?;
+    Ok(())
+}
+
+pub fn generate_hash(data: &[u8]) -> DHTKey {
+    DHTKey::new(*blake3::hash(data).as_bytes())
+}
+
+pub fn validate_hash(data: &[u8], dht_key: &DHTKey) -> bool {
+    assert!(dht_key.valid);
+    let bytes = *blake3::hash(data).as_bytes();
+
+    bytes == dht_key.bytes
+}
+
+pub fn validate_key(dht_key: &DHTKey, dht_key_secret: &DHTKeySecret) -> bool {
+    let data = vec![0u8; 512];
+    let sig = match sign(&dht_key, &dht_key_secret, &data) {
+        Ok(s) => s,
+        Err(_) => {
+            return false;
+        }
+    };
+    verify(&dht_key, &data, &sig).is_ok()
+}
+
+pub fn distance(key1: &DHTKey, key2: &DHTKey) -> DHTKeyDistance {
+    assert!(key1.valid);
+    assert!(key2.valid);
+    let mut bytes = [0u8; DHT_KEY_LENGTH];
+
+    for n in 0..DHT_KEY_LENGTH {
+        bytes[n] = key1.bytes[n] ^ key2.bytes[n];
+    }
+
+    DHTKeyDistance::new(bytes)
+}

veilid-core/src/dht/mod.rs

@@ -0,0 +1,11 @@
+pub mod crypto;
+pub mod envelope;
+pub mod key;
+pub mod receipt;
+pub mod value;
+
+pub use crypto::*;
+pub use envelope::*;
+pub use key::*;
+pub use receipt::*;
+pub use value::*;

veilid-core/src/dht/receipt.rs

@@ -0,0 +1,170 @@
+use super::envelope::{MAX_VERSION, MIN_VERSION};
+use super::key::*;
+use crate::xx::*;
+use core::convert::TryInto;
+
+// #[repr(C, packed)]
+// struct ReceiptHeader {
+//     // Size is at least 8 bytes. Depending on the version specified, the size may vary and should be case to the appropriate struct
+//     magic: [u8; 4],              // 0x00: 0x52 0x43 0x50 0x54 ("RCPT")
+//     version: u8,                 // 0x04: 0 = ReceiptV0
+//     reserved: u8,                // 0x05: Reserved for future use
+// }
+
+// #[repr(C, packed)]
+// struct ReceiptV0 {
+//     // Size is 106 bytes.
+//     magic: [u8; 4],              // 0x00: 0x52 0x43 0x50 0x54 ("RCPT")
+//     version: u8,                 // 0x04: 0 = ReceiptV0
+//     reserved: u8,                // 0x05: Reserved for future use
+//     size: u16,                   // 0x06: Total size of the receipt including the extra data and the signature. Maximum size is 1152 bytes.
+//     nonce: [u8; 24],             // 0x08: Randomly chosen bytes that represent a unique receipt. Could be used to encrypt the extra data, but it's not required.
+//     sender_id: [u8; 32],         // 0x20: Node ID of the message source, which is the Ed25519 public key of the sender (must be verified with find_node if this is a new node_id/address combination)
+//     extra_data: [u8; ??],        // 0x40: Extra data is appended (arbitrary extra data, not encrypted by receipt itself, maximum size is 1024 bytes)
+//     signature: [u8; 64],         // 0x?? (end-0x40): Ed25519 signature of the entire receipt including header and extra data is appended to the packet
+// }
+
+pub const MAX_RECEIPT_SIZE: usize = 1152;
+pub const MAX_EXTRA_DATA_SIZE: usize = 1024;
+pub const MIN_RECEIPT_SIZE: usize = 128;
+pub const RECEIPT_MAGIC: &[u8; 4] = b"RCPT";
+pub type ReceiptNonce = [u8; 24];
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct Receipt {
+    version: u8,
+    nonce: ReceiptNonce,
+    sender_id: DHTKey,
+    extra_data: Vec<u8>,
+}
+
+impl Receipt {
+    pub fn try_new<D: AsRef<[u8]>>(
+        version: u8,
+        nonce: ReceiptNonce,
+        sender_id: DHTKey,
+        extra_data: D,
+    ) -> Result<Self, String> {
+        assert!(sender_id.valid);
+        if extra_data.as_ref().len() > MAX_EXTRA_DATA_SIZE {
+            return Err("extra data too large for receipt".to_owned());
+        }
+        Ok(Self {
+            version: version,
+            nonce: nonce,
+            sender_id: sender_id,
+            extra_data: Vec::from(extra_data.as_ref()),
+        })
+    }
+
+    pub fn from_signed_data(data: &[u8]) -> Result<Receipt, ()> {
+        // Ensure we are at least the length of the envelope
+        if data.len() < MIN_RECEIPT_SIZE {
+            trace!("receipt too small: len={}", data.len());
+            return Err(());
+        }
+
+        // Verify magic number
+        let magic: [u8; 4] = data[0x00..0x04].try_into().map_err(drop)?;
+        if magic != *RECEIPT_MAGIC {
+            trace!("bad magic number: len={:?}", magic);
+            return Err(());
+        }
+
+        // Check version
+        let version = data[0x04];
+        if version > MAX_VERSION || version < MIN_VERSION {
+            trace!("unsupported protocol version: version={}", version);
+            return Err(());
+        }
+
+        // Get size and ensure it matches the size of the envelope and is less than the maximum message size
+        let size: u16 = u16::from_le_bytes(data[0x06..0x08].try_into().map_err(drop)?);
+        if (size as usize) > MAX_RECEIPT_SIZE {
+            trace!("receipt size is too large: size={}", size);
+            return Err(());
+        }
+        if (size as usize) != data.len() {
+            trace!(
+                "size doesn't match receipt size: size={} data.len()={}",
+                size,
+                data.len()
+            );
+            return Err(());
+        }
+
+        // Get sender id
+        let sender_id_dhtkey = DHTKey::new(data[0x20..0x40].try_into().map_err(drop)?);
+        // Get signature
+        let signature = DHTSignature::new(data[(data.len() - 64)..].try_into().map_err(drop)?);
+
+        // Validate signature
+        verify(&sender_id_dhtkey, &data[0..(data.len() - 64)], &signature).map_err(drop)?;
+
+        // Get nonce
+        let nonce: ReceiptNonce = data[0x08..0x20].try_into().map_err(drop)?;
+
+        // Get extra data and signature
+        let extra_data: Vec<u8> = Vec::from(&data[0x40..(data.len() - 64)]);
+
+        // Return receipt
+        Ok(Self {
+            version: version,
+            nonce: nonce,
+            sender_id: sender_id_dhtkey,
+            extra_data: extra_data,
+        })
+    }
+
+    pub fn to_signed_data(&self, secret: &DHTKeySecret) -> Result<Vec<u8>, ()> {
+        // Ensure sender node id is valid
+        if !self.sender_id.valid {
+            return Err(());
+        }
+
+        // Ensure extra data isn't too long
+        let receipt_size: usize = self.extra_data.len() + MIN_RECEIPT_SIZE;
+        if receipt_size > MAX_RECEIPT_SIZE {
+            return Err(());
+        }
+        let mut data: Vec<u8> = Vec::with_capacity(receipt_size);
+        data.resize(receipt_size, 0u8);
+
+        // Write magic
+        data[0x00..0x04].copy_from_slice(RECEIPT_MAGIC);
+        // Write version
+        data[0x04] = self.version;
+        // Write size
+        data[0x06..0x08].copy_from_slice(&(receipt_size as u16).to_le_bytes());
+        // Write nonce
+        data[0x08..0x20].copy_from_slice(&self.nonce);
+        // Write sender node id
+        data[0x20..0x40].copy_from_slice(&self.sender_id.bytes);
+        // Write extra data
+        if self.extra_data.len() > 0 {
+            data[0x40..(receipt_size - 64)].copy_from_slice(self.extra_data.as_slice());
+        }
+        // Sign the receipt
+        let signature =
+            sign(&self.sender_id, secret, &data[0..(receipt_size - 64)]).map_err(drop)?;
+        // Append the signature
+        data[(receipt_size - 64)..].copy_from_slice(&signature.bytes);
+
+        Ok(data)
+    }
+
+    pub fn get_version(&self) -> u8 {
+        self.version
+    }
+
+    pub fn get_nonce(&self) -> ReceiptNonce {
+        self.nonce
+    }
+
+    pub fn get_sender_id(&self) -> DHTKey {
+        self.sender_id
+    }
+    pub fn get_extra_data(&self) -> &[u8] {
+        &self.extra_data
+    }
+}

veilid-core/src/intf/mod.rs

@@ -0,0 +1,54 @@
+mod table_db;
+
+use crate::xx::*;
+use data_encoding::BASE64URL_NOPAD;
+
+#[cfg(target_arch = "wasm32")]
+mod wasm;
+#[cfg(target_arch = "wasm32")]
+pub use wasm::*;
+#[cfg(not(target_arch = "wasm32"))]
+mod native;
+#[cfg(not(target_arch = "wasm32"))]
+pub use native::*;
+
+pub async fn save_user_secret(namespace: &str, key: &str, value: &[u8]) -> Result<bool, String> {
+    let mut s = BASE64URL_NOPAD.encode(value);
+    s.push('!');
+
+    save_user_secret_string(namespace, key, s.as_str()).await
+}
+
+pub async fn load_user_secret(namespace: &str, key: &str) -> Result<Option<Vec<u8>>, String> {
+    let mut s = match load_user_secret_string(namespace, key).await? {
+        Some(s) => s,
+        None => {
+            return Ok(None);
+        }
+    };
+
+    if s.pop() != Some('!') {
+        return Err("User secret is not a buffer".to_owned());
+    }
+
+    let mut bytes = Vec::<u8>::new();
+    let res = BASE64URL_NOPAD.decode_len(s.len());
+    match res {
+        Ok(l) => {
+            bytes.resize(l, 0u8);
+        }
+        Err(_) => {
+            return Err("Failed to decode".to_owned());
+        }
+    }
+
+    let res = BASE64URL_NOPAD.decode_mut(s.as_bytes(), &mut bytes);
+    match res {
+        Ok(_) => Ok(Some(bytes)),
+        Err(_) => Err("Failed to decode".to_owned()),
+    }
+}
+
+pub async fn remove_user_secret(namespace: &str, key: &str) -> Result<bool, String> {
+    remove_user_secret_string(namespace, key).await
+}

veilid-core/src/intf/native/block_store.rs

@@ -0,0 +1,2 @@
+//use crate::intf::*;
+//use crate::xx::*;

veilid-core/src/intf/native/mod.rs

@@ -0,0 +1,12 @@
+mod block_store;
+mod network;
+mod protected_store;
+mod system;
+pub mod table_store;
+pub mod utils;
+
+pub use block_store::*;
+pub use network::*;
+pub use protected_store::*;
+pub use system::*;
+pub use table_store::*;

veilid-core/src/intf/native/network/listener_state.rs

@@ -0,0 +1,54 @@
+use crate::intf::*;
+use crate::network_manager::*;
+use utils::async_peek_stream::*;
+
+use async_std::net::*;
+use async_tls::TlsAcceptor;
+
+pub trait TcpProtocolHandler: TcpProtocolHandlerClone + Send + Sync {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SendPinBoxFuture<Result<bool, ()>>;
+}
+
+pub trait TcpProtocolHandlerClone {
+    fn clone_box(&self) -> Box<dyn TcpProtocolHandler>;
+}
+
+impl<T> TcpProtocolHandlerClone for T
+where
+    T: 'static + TcpProtocolHandler + Clone,
+{
+    fn clone_box(&self) -> Box<dyn TcpProtocolHandler> {
+        Box::new(self.clone())
+    }
+}
+impl Clone for Box<dyn TcpProtocolHandler> {
+    fn clone(&self) -> Box<dyn TcpProtocolHandler> {
+        self.clone_box()
+    }
+}
+
+pub type NewTcpProtocolHandler =
+    dyn Fn(NetworkManager, bool, SocketAddr) -> Box<dyn TcpProtocolHandler> + Send;
+
+/////////////////////////////////////////////////////////////////
+
+#[derive(Clone)]
+pub struct ListenerState {
+    pub protocol_handlers: Vec<Box<dyn TcpProtocolHandler + 'static>>,
+    pub tls_protocol_handlers: Vec<Box<dyn TcpProtocolHandler + 'static>>,
+    pub tls_acceptor: Option<TlsAcceptor>,
+}
+
+impl ListenerState {
+    pub fn new() -> Self {
+        Self {
+            protocol_handlers: Vec::new(),
+            tls_protocol_handlers: Vec::new(),
+            tls_acceptor: None,
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/mod.rs

@@ -0,0 +1,63 @@
+pub mod tcp;
+pub mod udp;
+pub mod wrtc;
+pub mod ws;
+
+use super::listener_state::*;
+use crate::veilid_api::ProtocolType;
+use crate::xx::*;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DummyNetworkConnection {}
+
+impl DummyNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        ProtocolType::UDP
+    }
+    pub fn send(&self, _message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        Box::pin(async { Ok(()) })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        Box::pin(async { Ok(Vec::new()) })
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum NetworkConnection {
+    Dummy(DummyNetworkConnection),
+    RawTcp(tcp::RawTcpNetworkConnection),
+    WSAccepted(ws::WebSocketNetworkConnectionAccepted),
+    WS(ws::WebsocketNetworkConnectionWS),
+    WSS(ws::WebsocketNetworkConnectionWSS),
+    //WebRTC(wrtc::WebRTCNetworkConnection),
+}
+
+impl NetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        match self {
+            Self::Dummy(d) => d.protocol_type(),
+            Self::RawTcp(t) => t.protocol_type(),
+            Self::WSAccepted(w) => w.protocol_type(),
+            Self::WS(w) => w.protocol_type(),
+            Self::WSS(w) => w.protocol_type(),
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        match self {
+            Self::Dummy(d) => d.send(message),
+            Self::RawTcp(t) => t.send(message),
+            Self::WSAccepted(w) => w.send(message),
+            Self::WS(w) => w.send(message),
+            Self::WSS(w) => w.send(message),
+        }
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        match self {
+            Self::Dummy(d) => d.recv(),
+            Self::RawTcp(t) => t.recv(),
+            Self::WSAccepted(w) => w.recv(),
+            Self::WS(w) => w.recv(),
+            Self::WSS(w) => w.recv(),
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/tcp.rs

@@ -0,0 +1,234 @@
+use super::*;
+use crate::intf::native::utils::async_peek_stream::*;
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::net::*;
+use async_std::prelude::*;
+use async_std::sync::Mutex as AsyncMutex;
+use socket2::{Domain, Protocol, Socket, Type};
+use std::fmt;
+
+struct RawTcpNetworkConnectionInner {
+    stream: AsyncPeekStream,
+}
+
+#[derive(Clone)]
+pub struct RawTcpNetworkConnection {
+    inner: Arc<AsyncMutex<RawTcpNetworkConnectionInner>>,
+}
+
+impl fmt::Debug for RawTcpNetworkConnection {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", std::any::type_name::<Self>())
+    }
+}
+
+impl PartialEq for RawTcpNetworkConnection {
+    fn eq(&self, other: &Self) -> bool {
+        Arc::as_ptr(&self.inner) == Arc::as_ptr(&other.inner)
+    }
+}
+
+impl Eq for RawTcpNetworkConnection {}
+
+impl RawTcpNetworkConnection {
+    fn new_inner(stream: AsyncPeekStream) -> RawTcpNetworkConnectionInner {
+        RawTcpNetworkConnectionInner { stream: stream }
+    }
+
+    pub fn new(stream: AsyncPeekStream) -> Self {
+        Self {
+            inner: Arc::new(AsyncMutex::new(Self::new_inner(stream))),
+        }
+    }
+}
+
+impl RawTcpNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        ProtocolType::TCP
+    }
+
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            if message.len() > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+            let len = message.len() as u16;
+            let header = [b'V', b'L', len as u8, (len >> 8) as u8];
+
+            let mut inner = inner.lock().await;
+            inner.stream.write_all(&header).await.map_err(drop)?;
+            inner.stream.write_all(&message).await.map_err(drop)
+        })
+    }
+
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            let mut header = [0u8; 4];
+            let mut inner = inner.lock().await;
+
+            inner.stream.read_exact(&mut header).await.map_err(drop)?;
+            if header[0] != b'V' || header[1] != b'L' {
+                return Err(());
+            }
+            let len = ((header[3] as usize) << 8) | (header[2] as usize);
+            if len > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+
+            let mut out: Vec<u8> = Vec::with_capacity(len);
+            out.resize(len, 0u8);
+            inner.stream.read_exact(&mut out).await.map_err(drop)?;
+            Ok(out)
+        })
+    }
+}
+
+///////////////////////////////////////////////////////////
+///
+
+struct RawTcpProtocolHandlerInner {
+    network_manager: NetworkManager,
+    local_address: SocketAddr,
+}
+
+#[derive(Clone)]
+pub struct RawTcpProtocolHandler
+where
+    Self: TcpProtocolHandler,
+{
+    inner: Arc<Mutex<RawTcpProtocolHandlerInner>>,
+}
+
+impl RawTcpProtocolHandler {
+    fn new_inner(
+        network_manager: NetworkManager,
+        local_address: SocketAddr,
+    ) -> RawTcpProtocolHandlerInner {
+        RawTcpProtocolHandlerInner {
+            network_manager: network_manager,
+            local_address: local_address,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager, local_address: SocketAddr) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager, local_address))),
+        }
+    }
+
+    pub async fn on_accept_async(
+        self,
+        stream: AsyncPeekStream,
+        socket_addr: SocketAddr,
+    ) -> Result<bool, ()> {
+        let mut peekbuf: [u8; PEEK_DETECT_LEN] = [0u8; PEEK_DETECT_LEN];
+        let peeklen = stream.peek(&mut peekbuf).await.map_err(drop)?;
+        assert_eq!(peeklen, PEEK_DETECT_LEN);
+
+        let conn = NetworkConnection::RawTcp(RawTcpNetworkConnection::new(stream));
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(socket_addr),
+            socket_addr.port(),
+            ProtocolType::TCP,
+        );
+        let (network_manager, local_address) = {
+            let inner = self.inner.lock();
+            (inner.network_manager.clone(), inner.local_address.clone())
+        };
+        network_manager
+            .on_new_connection(ConnectionDescriptor::new(peer_addr, local_address), conn)
+            .await?;
+        Ok(true)
+    }
+
+    pub async fn connect(
+        network_manager: NetworkManager,
+        preferred_local_address: Option<SocketAddr>,
+        remote_socket_addr: SocketAddr,
+    ) -> Result<NetworkConnection, ()> {
+        // Make a low level socket that can connect to the remote socket address
+        // and attempt to reuse the local address that our listening socket uses
+        // for hole-punch compatibility
+        let domain = Domain::for_address(remote_socket_addr);
+        let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).map_err(drop)?;
+        if let Err(e) = socket.set_linger(None) {
+            warn!("Couldn't set TCP linger: {}", e);
+        }
+        if let Err(e) = socket.set_nodelay(true) {
+            warn!("Couldn't set TCP nodelay: {}", e);
+        }
+        if let Err(e) = socket.set_reuse_address(true) {
+            warn!("Couldn't set reuse address: {}", e);
+        }
+        cfg_if! {
+            if #[cfg(unix)] {
+                if let Err(e) = socket.set_reuse_port(true) {
+                    warn!("Couldn't set reuse port: {}", e);
+                }
+            }
+        }
+
+        // Try to bind it to the preferred local address
+        if let Some(some_local_addr) = preferred_local_address {
+            let socket2_addr = socket2::SockAddr::from(some_local_addr);
+            if let Err(e) = socket.bind(&socket2_addr) {
+                warn!("failed to bind TCP socket: {}", e);
+            }
+        }
+
+        // Connect to the remote address
+        let remote_socket2_addr = socket2::SockAddr::from(remote_socket_addr);
+        socket.connect(&remote_socket2_addr).map_err(drop)?;
+        let std_stream: std::net::TcpStream = socket.into();
+        let ts = TcpStream::from(std_stream);
+
+        // See what local address we ended up with and turn this into a stream
+        let local_address = ts.local_addr().map_err(drop)?.clone();
+        let ps = AsyncPeekStream::new(ts);
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(remote_socket_addr),
+            remote_socket_addr.port(),
+            ProtocolType::TCP,
+        );
+
+        // Wrap the stream in a network connection and register it
+        let conn = NetworkConnection::RawTcp(RawTcpNetworkConnection::new(ps));
+        network_manager
+            .on_new_connection(
+                ConnectionDescriptor::new(peer_addr, local_address),
+                conn.clone(),
+            )
+            .await?;
+        Ok(conn)
+    }
+
+    pub async fn send_unbound_message(data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+        trace!(
+            "sending unbound message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        let mut stream = TcpStream::connect(socket_addr).await.map_err(drop)?;
+        stream.write_all(&data).await.map_err(drop)
+    }
+}
+
+impl TcpProtocolHandler for RawTcpProtocolHandler {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SendPinBoxFuture<Result<bool, ()>> {
+        Box::pin(self.clone().on_accept_async(stream, peer_addr))
+    }
+}

veilid-core/src/intf/native/network/protocol/udp.rs

@@ -0,0 +1,109 @@
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::net::*;
+
+struct RawUdpProtocolHandlerInner {
+    network_manager: NetworkManager,
+    socket: Arc<UdpSocket>,
+}
+
+#[derive(Clone)]
+pub struct RawUdpProtocolHandler {
+    inner: Arc<Mutex<RawUdpProtocolHandlerInner>>,
+}
+
+impl RawUdpProtocolHandler {
+    fn new_inner(
+        network_manager: NetworkManager,
+        socket: Arc<UdpSocket>,
+    ) -> RawUdpProtocolHandlerInner {
+        RawUdpProtocolHandlerInner {
+            network_manager: network_manager,
+            socket: socket,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager, socket: Arc<UdpSocket>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager, socket))),
+        }
+    }
+
+    pub async fn on_message(&self, data: &[u8], remote_addr: SocketAddr) -> Result<bool, ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+
+        trace!(
+            "receiving message of length {} from {}",
+            data.len(),
+            remote_addr
+        );
+
+        // Process envelope
+        let (network_manager, socket) = {
+            let inner = self.inner.lock();
+            (inner.network_manager.clone(), inner.socket.clone())
+        };
+
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(remote_addr),
+            remote_addr.port(),
+            ProtocolType::UDP,
+        );
+        let local_socket_addr = socket.local_addr().map_err(drop)?;
+        network_manager
+            .on_recv_envelope(
+                data,
+                &ConnectionDescriptor::new(peer_addr, local_socket_addr),
+            )
+            .await
+    }
+
+    pub async fn send_message(&self, data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+
+        trace!(
+            "sending message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        let socket = self.inner.lock().socket.clone();
+        let len = socket.send_to(&data, socket_addr).await.map_err(drop)?;
+        if len != data.len() {
+            Err(())
+        } else {
+            Ok(())
+        }
+    }
+
+    pub async fn send_unbound_message(data: Vec<u8>, socket_addr: SocketAddr) -> Result<(), ()> {
+        if data.len() > MAX_MESSAGE_SIZE {
+            return Err(());
+        }
+        trace!(
+            "sending unbound message of length {} to {}",
+            data.len(),
+            socket_addr
+        );
+
+        // get local wildcard address for bind
+        let local_socket_addr = match socket_addr {
+            SocketAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
+            SocketAddr::V6(_) => {
+                SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), 0)
+            }
+        };
+        let socket = UdpSocket::bind(local_socket_addr).await.map_err(drop)?;
+        let len = socket.send_to(&data, socket_addr).await.map_err(drop)?;
+        if len != data.len() {
+            Err(())
+        } else {
+            Ok(())
+        }
+    }
+}

veilid-core/src/intf/native/network/protocol/ws.rs

@@ -0,0 +1,302 @@
+use super::*;
+use crate::intf::native::utils::async_peek_stream::*;
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use async_std::io;
+use async_std::net::*;
+use async_std::sync::Mutex as AsyncMutex;
+use async_tls::TlsConnector;
+use async_tungstenite::tungstenite::protocol::Message;
+use async_tungstenite::{accept_async, client_async, WebSocketStream};
+use futures_util::sink::SinkExt;
+use futures_util::stream::StreamExt;
+use std::fmt;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub type WebSocketNetworkConnectionAccepted = WebsocketNetworkConnection<AsyncPeekStream>;
+pub type WebsocketNetworkConnectionWSS =
+    WebsocketNetworkConnection<async_tls::client::TlsStream<async_std::net::TcpStream>>;
+pub type WebsocketNetworkConnectionWS = WebsocketNetworkConnection<async_std::net::TcpStream>;
+
+struct WebSocketNetworkConnectionInner<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    ws_stream: WebSocketStream<T>,
+}
+
+pub struct WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    tls: bool,
+    inner: Arc<AsyncMutex<WebSocketNetworkConnectionInner<T>>>,
+}
+
+impl<T> Clone for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn clone(&self) -> Self {
+        Self {
+            tls: self.tls,
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<T> fmt::Debug for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", std::any::type_name::<Self>())
+    }
+}
+
+impl<T> PartialEq for WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.tls == other.tls && Arc::as_ptr(&self.inner) == Arc::as_ptr(&other.inner)
+    }
+}
+
+impl<T> Eq for WebsocketNetworkConnection<T> where T: io::Read + io::Write + Send + Unpin + 'static {}
+
+impl<T> WebsocketNetworkConnection<T>
+where
+    T: io::Read + io::Write + Send + Unpin + 'static,
+{
+    pub fn new(tls: bool, ws_stream: WebSocketStream<T>) -> Self {
+        Self {
+            tls: tls,
+            inner: Arc::new(AsyncMutex::new(WebSocketNetworkConnectionInner {
+                ws_stream: ws_stream,
+            })),
+        }
+    }
+
+    pub fn protocol_type(&self) -> ProtocolType {
+        if self.tls {
+            ProtocolType::WSS
+        } else {
+            ProtocolType::WS
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            if message.len() > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+            let mut inner = inner.lock().await;
+            inner
+                .ws_stream
+                .send(Message::binary(message))
+                .await
+                .map_err(drop)
+        })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        let inner = self.inner.clone();
+
+        Box::pin(async move {
+            let mut inner = inner.lock().await;
+
+            let out = match inner.ws_stream.next().await {
+                Some(Ok(Message::Binary(v))) => v,
+                _ => {
+                    trace!("websocket recv failed");
+                    return Err(());
+                }
+            };
+            if out.len() > MAX_MESSAGE_SIZE {
+                Err(())
+            } else {
+                Ok(out)
+            }
+        })
+    }
+}
+
+///////////////////////////////////////////////////////////
+///
+struct WebsocketProtocolHandlerInner {
+    tls: bool,
+    network_manager: NetworkManager,
+    local_address: SocketAddr,
+    request_path: Vec<u8>,
+    connection_initial_timeout: u64,
+}
+
+#[derive(Clone)]
+pub struct WebsocketProtocolHandler
+where
+    Self: TcpProtocolHandler,
+{
+    inner: Arc<WebsocketProtocolHandlerInner>,
+}
+impl WebsocketProtocolHandler {
+    pub fn new(network_manager: NetworkManager, tls: bool, local_address: SocketAddr) -> Self {
+        let config = network_manager.config();
+        let c = config.get();
+        let path = format!("GET {}", c.network.protocol.ws.path.trim_end_matches('/'));
+        let connection_initial_timeout = if tls {
+            c.network.tls.connection_initial_timeout
+        } else {
+            c.network.connection_initial_timeout
+        };
+
+        let inner = WebsocketProtocolHandlerInner {
+            tls: tls,
+            network_manager: network_manager,
+            local_address: local_address,
+            request_path: path.as_bytes().to_vec(),
+            connection_initial_timeout: connection_initial_timeout,
+        };
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    pub async fn on_accept_async(
+        self,
+        ps: AsyncPeekStream,
+        socket_addr: SocketAddr,
+    ) -> Result<bool, ()> {
+        let request_path_len = self.inner.request_path.len() + 2;
+        let mut peekbuf: Vec<u8> = Vec::with_capacity(request_path_len);
+        peekbuf.resize(request_path_len, 0u8);
+        match io::timeout(
+            Duration::from_micros(self.inner.connection_initial_timeout),
+            ps.peek_exact(&mut peekbuf),
+        )
+        .await
+        {
+            Ok(_) => (),
+            Err(e) => {
+                trace!("failed to peek stream: {:?}", e);
+                return Err(());
+            }
+        }
+        // Check for websocket path
+        let matches_path = &peekbuf[0..request_path_len - 2] == self.inner.request_path.as_slice()
+            && (peekbuf[request_path_len - 2] == b' '
+                || (peekbuf[request_path_len - 2] == b'/'
+                    && peekbuf[request_path_len - 1] == b' '));
+
+        if !matches_path {
+            trace!("not websocket");
+            return Ok(false);
+        }
+        trace!("found websocket");
+
+        let ws_stream = match accept_async(ps).await {
+            Ok(s) => s,
+            Err(e) => {
+                trace!("failed websockets handshake: {:?}", e);
+                return Err(());
+            }
+        };
+
+        // Wrap the websocket in a NetworkConnection and register it
+        let protocol_type = if self.inner.tls {
+            ProtocolType::WSS
+        } else {
+            ProtocolType::WS
+        };
+
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(socket_addr),
+            socket_addr.port(),
+            protocol_type,
+        );
+
+        let conn = NetworkConnection::WSAccepted(WebsocketNetworkConnection::new(
+            self.inner.tls,
+            ws_stream,
+        ));
+        self.inner
+            .network_manager
+            .clone()
+            .on_new_connection(
+                ConnectionDescriptor::new(peer_addr, self.inner.local_address.clone()),
+                conn,
+            )
+            .await?;
+        Ok(true)
+    }
+
+    pub async fn connect(
+        network_manager: NetworkManager,
+        dial_info: &DialInfo,
+    ) -> Result<NetworkConnection, ()> {
+        let (tls, request, domain, port, protocol_type) = match &dial_info {
+            DialInfo::WS(di) => (
+                false,
+                di.path.clone(),
+                di.fqdn.clone(),
+                di.port,
+                ProtocolType::WS,
+            ),
+            DialInfo::WSS(di) => (
+                true,
+                di.path.clone(),
+                di.fqdn.clone(),
+                di.port,
+                ProtocolType::WSS,
+            ),
+            _ => panic!("invalid dialinfo for WS/WSS protocol"),
+        };
+
+        let tcp_stream = TcpStream::connect(format!("{}:{}", &domain, &port))
+            .await
+            .map_err(drop)?;
+        let local_addr = tcp_stream.local_addr().map_err(drop)?;
+        let peer_socket_addr = tcp_stream.peer_addr().map_err(drop)?;
+        let peer_addr = PeerAddress::new(
+            Address::from_socket_addr(peer_socket_addr),
+            peer_socket_addr.port(),
+            protocol_type,
+        );
+
+        if tls {
+            let connector = TlsConnector::default();
+            let tls_stream = connector.connect(domain, tcp_stream).await.map_err(drop)?;
+            let (ws_stream, _response) = client_async(request, tls_stream).await.map_err(drop)?;
+            let conn = NetworkConnection::WSS(WebsocketNetworkConnection::new(tls, ws_stream));
+            network_manager
+                .on_new_connection(
+                    ConnectionDescriptor::new(peer_addr, local_addr),
+                    conn.clone(),
+                )
+                .await?;
+            Ok(conn)
+        } else {
+            let (ws_stream, _response) = client_async(request, tcp_stream).await.map_err(drop)?;
+            let conn = NetworkConnection::WS(WebsocketNetworkConnection::new(tls, ws_stream));
+            network_manager
+                .on_new_connection(
+                    ConnectionDescriptor::new(peer_addr, local_addr),
+                    conn.clone(),
+                )
+                .await?;
+            Ok(conn)
+        }
+    }
+}
+
+impl TcpProtocolHandler for WebsocketProtocolHandler {
+    fn on_accept(
+        &self,
+        stream: AsyncPeekStream,
+        peer_addr: SocketAddr,
+    ) -> SystemPinBoxFuture<Result<bool, ()>> {
+        Box::pin(self.clone().on_accept_async(stream, peer_addr))
+    }
+}

veilid-core/src/intf/native/network/public_dialinfo_discovery.rs

@@ -0,0 +1,222 @@
+use super::*;
+
+use crate::intf::*;
+use crate::network_manager::*;
+use crate::routing_table::*;
+use crate::*;
+
+use async_std::net::*;
+
+impl Network {
+    // Ask for a public address check from a particular noderef
+    async fn request_public_address(&self, node_ref: NodeRef) -> Option<SocketAddr> {
+        let routing_table = self.routing_table();
+        let rpc = routing_table.rpc_processor();
+        let info_answer = match rpc.rpc_call_info(node_ref.clone()).await {
+            Err(e) => {
+                trace!("failed to get info answer from {:?}: {:?}", node_ref, e);
+                return None;
+            }
+            Ok(ia) => ia,
+        };
+        info_answer.sender_info.socket_address
+    }
+
+    // find fast peers with a particular address type, and ask them to tell us what our external address is
+    async fn discover_external_address(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+        ignore_node: Option<DHTKey>,
+    ) -> Result<(SocketAddr, NodeRef), String> {
+        let routing_table = self.routing_table();
+        let peers = routing_table.get_fast_nodes_of_type(protocol_address_type);
+        if peers.len() == 0 {
+            return Err(format!("no peers of type '{:?}'", protocol_address_type));
+        }
+        for peer in peers {
+            if let Some(ignore_node) = ignore_node {
+                if peer.node_id() == ignore_node {
+                    continue;
+                }
+            }
+            if let Some(sa) = self.request_public_address(peer.clone()).await {
+                return Ok((sa, peer));
+            }
+        }
+        Err("no peers responded with an external address".to_owned())
+    }
+
+    fn discover_local_address(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Result<SocketAddr, String> {
+        let routing_table = self.routing_table();
+
+        match routing_table
+            .get_own_peer_info(PeerScope::Public)
+            .dial_infos
+            .iter()
+            .find_map(|di| {
+                if di.protocol_address_type() == protocol_address_type {
+                    if let Ok(addr) = di.to_socket_addr() {
+                        return Some(addr);
+                    }
+                }
+                None
+            }) {
+            None => Err(format!(
+                "no local address for protocol address type: {:?}",
+                protocol_address_type
+            )),
+            Some(addr) => Ok(addr),
+        }
+    }
+
+    async fn validate_dial_info(
+        &self,
+        node_ref: NodeRef,
+        dial_info: DialInfo,
+        redirect: bool,
+        alternate_port: bool,
+    ) -> bool {
+        let routing_table = self.routing_table();
+        let rpc = routing_table.rpc_processor();
+        match rpc
+            .rpc_call_validate_dial_info(node_ref.clone(), dial_info, redirect, alternate_port)
+            .await
+        {
+            Err(e) => {
+                error!(
+                    "failed to send validate_dial_info to {:?}: {:?}",
+                    node_ref, e
+                );
+                false
+            }
+            Ok(val) => val,
+        }
+    }
+
+    async fn try_port_mapping(
+        &self,
+        local_addr: SocketAddr,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Option<SocketAddr> {
+        //xxx
+        None
+    }
+
+    pub async fn update_udpv4_dialinfo_task_routine(self, l: u64, t: u64) -> Result<(), String> {
+        trace!("looking for udpv4 public dial info");
+        let routing_table = self.routing_table();
+
+        // Get our local address
+        let local1 = self.discover_local_address(ProtocolAddressType::UDPv4)?;
+        // Get our external address from some fast node, call it node B
+        let (external1, node_b) = self
+            .discover_external_address(ProtocolAddressType::UDPv4, None)
+            .await?;
+        let external1_dial_info = DialInfo::udp_from_socketaddr(external1);
+
+        // If local1 == external1 then there is no NAT in place
+        if local1 == external1 {
+            // No NAT
+            // Do a validate_dial_info on the external address from a routed node
+            if self
+                .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false)
+                .await
+            {
+                // Add public dial info with Server network class
+                routing_table.register_public_dial_info(
+                    external1_dial_info,
+                    Some(NetworkClass::Server),
+                    DialInfoOrigin::Discovered,
+                );
+            } else {
+                // UDP firewall?
+                warn!("UDP static public dial info not reachable. UDP firewall may be blocking inbound to {:?} for {:?}",external1_dial_info, node_b);
+            }
+        } else {
+            // There is -some NAT-
+            // Attempt a UDP port mapping via all available and enabled mechanisms
+            if let Some(external_mapped) = self
+                .try_port_mapping(local1.clone(), ProtocolAddressType::UDPv4)
+                .await
+            {
+                // Got a port mapping, let's use it
+                let external_mapped_dial_info = DialInfo::udp_from_socketaddr(external_mapped);
+                routing_table.register_public_dial_info(
+                    external_mapped_dial_info,
+                    Some(NetworkClass::Mapped),
+                    DialInfoOrigin::Mapped,
+                );
+            } else {
+                // Port mapping was not possible, let's see what kind of NAT we have
+
+                // Does a redirected dial info validation find us?
+                if self
+                    .validate_dial_info(node_b.clone(), external1_dial_info.clone(), true, false)
+                    .await
+                {
+                    // Yes, another machine can use the dial info directly, so Full Cone
+                    // Add public dial info with full cone NAT network class
+                    routing_table.register_public_dial_info(
+                        external1_dial_info,
+                        Some(NetworkClass::FullNAT),
+                        DialInfoOrigin::Discovered,
+                    );
+                } else {
+                    // No, we are restricted, determine what kind of restriction
+
+                    // Get our external address from some fast node, that is not node B, call it node D
+                    let (external2, node_d) = self
+                        .discover_external_address(
+                            ProtocolAddressType::UDPv4,
+                            Some(node_b.node_id()),
+                        )
+                        .await?;
+                    // If we have two different external addresses, then this is a symmetric NAT
+                    if external2 != external1 {
+                        // Symmetric NAT is outbound only, no public dial info will work
+                        self.inner.lock().network_class = Some(NetworkClass::OutboundOnly);
+                    } else {
+                        // Address is the same, so it's address or port restricted
+                        let external2_dial_info = DialInfo::udp_from_socketaddr(external2);
+                        // Do a validate_dial_info on the external address from a routed node
+                        if self
+                            .validate_dial_info(
+                                node_d.clone(),
+                                external2_dial_info.clone(),
+                                false,
+                                true,
+                            )
+                            .await
+                        {
+                            // Got a reply from a non-default port, which means we're only address restricted
+                            routing_table.register_public_dial_info(
+                                external1_dial_info,
+                                Some(NetworkClass::AddressRestrictedNAT),
+                                DialInfoOrigin::Discovered,
+                            );
+                        } else {
+                            // Didn't get a reply from a non-default port, which means we are also port restricted
+                            routing_table.register_public_dial_info(
+                                external1_dial_info,
+                                Some(NetworkClass::PortRestrictedNAT),
+                                DialInfoOrigin::Discovered,
+                            );
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn update_tcpv4_dialinfo_task_routine(self, l: u64, t: u64) -> Result<(), String> {
+        trace!("looking for tcpv4 public dial info");
+        // xxx
+        //Err("unimplemented".to_owned())
+        Ok(())
+    }
+}

veilid-core/src/intf/native/protected_store.rs

@@ -0,0 +1,58 @@
+use cfg_if::*;
+use keyring::{Keyring, KeyringError};
+
+fn keyring_name(namespace: &str) -> String {
+    if namespace.len() == 0 {
+        "veilid".to_owned()
+    } else {
+        format!("veilid_{}", namespace)
+    }
+}
+
+fn get_keyring<'a>(krname: &'a str, key: &'a str) -> Keyring<'a> {
+    cfg_if! {
+        if #[cfg(target_os = "android")] {
+            let agopt = super::utils::android::ANDROID_GLOBALS.lock();
+            let ag = agopt.as_ref().unwrap();
+            let vm = ag.vm.attach_current_thread().unwrap().get_java_vm().unwrap(); // cmon jni, no clone for javavm
+            let ctx = ag.ctx.clone();
+            Keyring::new("veilid", krname, key, (vm, ctx))
+        } else {
+            Keyring::new("veilid", krname, key)
+        }
+    }
+}
+
+pub async fn save_user_secret_string(
+    namespace: &str,
+    key: &str,
+    value: &str,
+) -> Result<bool, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    let existed = kr.get_password().is_ok();
+    let _ = kr
+        .set_password(value)
+        .map_err(|e| format!("Failed to save user secret: {}", e).to_owned())?;
+    Ok(existed)
+}
+
+pub async fn load_user_secret_string(namespace: &str, key: &str) -> Result<Option<String>, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    match kr.get_password() {
+        Ok(v) => Ok(Some(v)),
+        Err(KeyringError::NoPasswordFound) => Ok(None),
+        Err(e) => Err(format!("Failed to load user secret: {}", e).to_owned()),
+    }
+}
+
+pub async fn remove_user_secret_string(namespace: &str, key: &str) -> Result<bool, String> {
+    let krname = keyring_name(namespace);
+    let kr = get_keyring(krname.as_str(), key);
+    match kr.delete_password() {
+        Ok(_) => Ok(true),
+        Err(KeyringError::NoPasswordFound) => Ok(false),
+        Err(e) => Err(format!("Failed to remove user secret: {}", e).to_owned()),
+    }
+}

veilid-core/src/intf/native/system.rs

@@ -0,0 +1,102 @@
+use crate::xx::*;
+pub use async_executors::JoinHandle;
+use async_executors::{AsyncStd, LocalSpawnHandleExt, SpawnHandleExt};
+use rand::prelude::*;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+pub fn get_timestamp() -> u64 {
+    match SystemTime::now().duration_since(UNIX_EPOCH) {
+        Ok(n) => n.as_micros() as u64,
+        Err(_) => panic!("SystemTime before UNIX_EPOCH!"),
+    }
+}
+
+pub fn random_bytes(dest: &mut [u8]) -> Result<(), String> {
+    let mut rng = rand::thread_rng();
+    rng.try_fill_bytes(dest).map_err(|err| format!("{:?}", err))
+}
+
+pub fn get_random_u32() -> u32 {
+    let mut rng = rand::thread_rng();
+    rng.next_u32()
+}
+
+pub fn get_random_u64() -> u64 {
+    let mut rng = rand::thread_rng();
+    rng.next_u64()
+}
+
+pub async fn sleep(millis: u32) {
+    if millis == 0 {
+        async_std::task::yield_now().await;
+    } else {
+        async_std::task::sleep(Duration::from_millis(u64::from(millis))).await;
+    }
+}
+
+pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> JoinHandle<Out>
+where
+    Out: Send + 'static,
+{
+    AsyncStd
+        .spawn_handle(future)
+        .expect("async-std spawn should never error out")
+}
+
+pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> JoinHandle<Out>
+where
+    Out: 'static,
+{
+    AsyncStd
+        .spawn_handle_local(future)
+        .expect("async-std spawn_local should never error out")
+}
+
+pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SystemPinBoxFuture<()>
+where
+    F: Fn() -> FUT + Send + Sync + 'static,
+    FUT: Future<Output = ()> + Send,
+{
+    let e = Eventual::new();
+
+    let ie = e.clone();
+    let jh = spawn(async move {
+        while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
+            callback().await;
+        }
+    });
+
+    Box::pin(async move {
+        e.resolve().await;
+        jh.await;
+    })
+}
+
+pub use async_std::future::TimeoutError;
+
+pub async fn timeout<F, T>(dur_ms: u32, f: F) -> Result<T, TimeoutError>
+where
+    F: Future<Output = T>,
+{
+    async_std::future::timeout(Duration::from_millis(dur_ms as u64), f).await
+}
+
+pub fn get_concurrency() -> u32 {
+    num_cpus::get() as u32
+}
+
+/*
+pub fn async_callback<F, OF, EF, T, E>(fut: F, ok_fn: OF, err_fn: EF)
+where
+    F: Future<Output = Result<T, E>> + Send + 'static,
+    OF: FnOnce(T) + Send + 'static,
+    EF: FnOnce(E) + Send + 'static,
+{
+    spawn(Box::pin(async move {
+        match fut.await {
+            Ok(v) => ok_fn(v),
+            Err(e) => err_fn(e),
+        };
+    }));
+}
+*/

veilid-core/src/intf/native/table_store.rs

@@ -0,0 +1,121 @@
+use crate::intf::table_db::*;
+use crate::intf::*;
+use crate::*;
+use keyvaluedb_sqlite::*;
+use std::path::PathBuf;
+
+struct TableStoreInner {
+    config: VeilidConfig,
+    opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
+}
+
+#[derive(Clone)]
+pub struct TableStore {
+    inner: Arc<Mutex<TableStoreInner>>,
+}
+
+impl TableStore {
+    fn new_inner(config: VeilidConfig) -> TableStoreInner {
+        TableStoreInner {
+            config: config,
+            opened: BTreeMap::new(),
+        }
+    }
+    pub fn new(config: VeilidConfig) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(config))),
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        Ok(())
+    }
+
+    pub async fn terminate(&self) {
+        assert!(
+            self.inner.lock().opened.len() == 0,
+            "all open databases should have been closed"
+        );
+    }
+
+    pub fn on_table_db_drop(&self, table: String) {
+        let mut inner = self.inner.lock();
+        match inner.opened.remove(&table) {
+            Some(_) => (),
+            None => {
+                assert!(false, "should have removed an item");
+            }
+        }
+    }
+
+    fn get_dbpath(inner: &TableStoreInner, table: &str) -> Result<PathBuf, String> {
+        if !table
+            .chars()
+            .all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
+        {
+            return Err(format!("table name '{}' is invalid", table));
+        }
+        let c = inner.config.get();
+        let tablestoredir = c.tablestore.directory.clone();
+        std::fs::create_dir_all(&tablestoredir)
+            .map_err(|e| format!("failed to create tablestore path: {}", e))?;
+
+        let dbpath: PathBuf = [tablestoredir, String::from(table)].iter().collect();
+        Ok(dbpath)
+    }
+
+    fn get_table_name(inner: &TableStoreInner, table: &str) -> Result<String, String> {
+        if !table
+            .chars()
+            .all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
+        {
+            return Err(format!("table name '{}' is invalid", table));
+        }
+        let c = inner.config.get();
+        let namespace = c.namespace.clone();
+        Ok(if namespace.len() == 0 {
+            format!("{}", table)
+        } else {
+            format!("_ns_{}_{}", namespace, table)
+        })
+    }
+
+    pub async fn open(&self, name: &str, column_count: u32) -> Result<TableDB, String> {
+        let mut inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
+            match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) {
+                Some(tdb) => {
+                    return Ok(tdb);
+                }
+                None => {
+                    inner.opened.remove(&table_name);
+                }
+            };
+        }
+
+        let dbpath = Self::get_dbpath(&inner, &table_name)?;
+        let cfg = DatabaseConfig::with_columns(column_count);
+        let db =
+            Database::open(dbpath, cfg).map_err(|e| format!("failed to open tabledb: {}", e))?;
+
+        let table_db = TableDB::new(table_name.clone(), self.clone(), db);
+
+        inner.opened.insert(table_name, table_db.weak_inner());
+
+        Ok(table_db)
+    }
+
+    pub async fn delete(&self, name: &str) -> Result<bool, String> {
+        let inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if inner.opened.contains_key(&table_name) {
+            return Err("Not deleting table that is still opened".to_owned());
+        }
+        let dbpath = Self::get_dbpath(&inner, &table_name)?;
+        let ret = std::fs::remove_file(dbpath).is_ok();
+        Ok(ret)
+    }
+}

veilid-core/src/intf/native/utils/android/android_get_if_addrs.rs

@@ -0,0 +1,192 @@
+use super::*;
+use crate::xx::*;
+pub use if_addrs::{IfAddr, Ifv4Addr, Ifv6Addr, Interface};
+use jni::objects::JValue;
+use std::io;
+
+fn get_netmask_from_prefix_length_v4(out: &mut [u8; 4], mut plen: i16) {
+    for n in 0..4 {
+        out[n] = if plen >= 8 {
+            plen -= 8;
+            255u8
+        } else if plen <= 0 {
+            0u8
+        } else {
+            let v = 255u8 << (8 - plen);
+            plen = 0;
+            v
+        }
+    }
+}
+fn get_netmask_from_prefix_length_v6(out: &mut [u8; 16], mut plen: i16) {
+    for n in 0..16 {
+        out[n] = if plen >= 8 {
+            plen -= 8;
+            255u8
+        } else if plen == 0 {
+            0u8
+        } else {
+            let v = 255u8 << (8 - plen);
+            plen = 0;
+            v
+        }
+    }
+}
+
+fn convert_to_unsigned_4(x: [i8; 4]) -> [u8; 4] {
+    let mut out: [u8; 4] = [0u8; 4];
+    for i in 0..4 {
+        out[i] = x[i] as u8;
+    }
+    out
+}
+
+fn convert_to_unsigned_16(x: [i8; 16]) -> [u8; 16] {
+    let mut out: [u8; 16] = [0u8; 16];
+    for i in 0..16 {
+        out[i] = x[i] as u8;
+    }
+    out
+}
+
+macro_rules! call_method_checked {
+    ($env:expr, $obj:expr, $name:expr, $sig:expr, $args:expr, $kind:ident) => {
+        $env.call_method($obj, $name, $sig, $args)
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("call_method {} {} failed: {}", $name, $sig, e),
+                )
+            })?
+            .$kind()
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?
+    };
+}
+
+pub fn get_if_addrs() -> io::Result<Vec<Interface>> {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+
+    let niclass = env.find_class("java/net/NetworkInterface").unwrap();
+    let intfenum = env
+        .call_static_method(
+            niclass,
+            "getNetworkInterfaces",
+            "()Ljava/util/Enumeration;",
+            &[],
+        )
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let mut out: Vec<Interface> = Vec::new();
+    while call_method_checked!(env, intfenum, "hasMoreElements", "()Z", &[], z) {
+        let intf =
+            call_method_checked!(env, intfenum, "nextElement", "()Ljava/lang/Object;", &[], l);
+
+        let nameobj = call_method_checked!(env, intf, "getName", "()Ljava/lang/String;", &[], l);
+        let name_jstrval = env.get_string(JString::from(nameobj)).unwrap();
+        let name = String::from(name_jstrval.to_string_lossy());
+
+        let intfaddrs = call_method_checked!(
+            env,
+            intf,
+            "getInterfaceAddresses",
+            "()Ljava/util/List;",
+            &[],
+            l
+        );
+        let size = call_method_checked!(env, intfaddrs, "size", "()I", &[], i);
+        for i in 0..size {
+            let intfaddr = call_method_checked!(
+                env,
+                intfaddrs,
+                "get",
+                "(I)Ljava/lang/Object;",
+                &[JValue::Int(i)],
+                l
+            );
+
+            let ia_addr = call_method_checked!(
+                env,
+                intfaddr,
+                "getAddress",
+                "()Ljava/net/InetAddress;",
+                &[],
+                l
+            );
+            let ia_bcst = call_method_checked!(
+                env,
+                intfaddr,
+                "getBroadcast",
+                "()Ljava/net/InetAddress;",
+                &[],
+                l
+            );
+            let ia_plen =
+                call_method_checked!(env, intfaddr, "getNetworkPrefixLength", "()S", &[], s);
+
+            let ia_addr_bytearray =
+                call_method_checked!(env, ia_addr, "getAddress", "()[B", &[], l);
+            let ia_addr_bytearray_len = env.get_array_length(*ia_addr_bytearray).unwrap();
+            let addr: IfAddr;
+            if ia_addr_bytearray_len == 4 {
+                let mut ia_addr_bytes_v4 = [0i8; 4];
+                env.get_byte_array_region(*ia_addr_bytearray, 0, &mut ia_addr_bytes_v4)
+                    .unwrap();
+
+                let broadcast = if !env.is_same_object(ia_bcst, JObject::null()).unwrap() {
+                    let ia_bcst_bytearray =
+                        call_method_checked!(env, ia_bcst, "getAddress", "()[B", &[], l);
+                    let ia_bcst_bytearray_len = env.get_array_length(*ia_bcst_bytearray).unwrap();
+                    if ia_bcst_bytearray_len != 4 {
+                        warn!(
+                            "mismatched inet4 broadcast address length: {}",
+                            ia_bcst_bytearray_len
+                        );
+                        continue;
+                    }
+
+                    let mut ia_bsct_bytes_v4 = [0i8; 4];
+                    env.get_byte_array_region(*ia_bcst_bytearray, 0, &mut ia_bsct_bytes_v4)
+                        .unwrap();
+
+                    Some(Ipv4Addr::from(convert_to_unsigned_4(ia_bsct_bytes_v4)))
+                } else {
+                    None
+                };
+
+                let mut ia_netmask_bytes_v4 = [0u8; 4];
+                get_netmask_from_prefix_length_v4(&mut ia_netmask_bytes_v4, ia_plen);
+                addr = IfAddr::V4(Ifv4Addr {
+                    ip: Ipv4Addr::from(convert_to_unsigned_4(ia_addr_bytes_v4)),
+                    netmask: Ipv4Addr::from(ia_netmask_bytes_v4),
+                    broadcast: broadcast,
+                });
+            } else if ia_addr_bytearray_len == 16 {
+                let mut ia_addr_bytes_v6 = [0i8; 16];
+                env.get_byte_array_region(*ia_addr_bytearray, 0, &mut ia_addr_bytes_v6)
+                    .unwrap();
+
+                let mut ia_netmask_bytes_v6 = [0u8; 16];
+                get_netmask_from_prefix_length_v6(&mut ia_netmask_bytes_v6, ia_plen);
+                addr = IfAddr::V6(Ifv6Addr {
+                    ip: Ipv6Addr::from(convert_to_unsigned_16(ia_addr_bytes_v6)),
+                    netmask: Ipv6Addr::from(ia_netmask_bytes_v6),
+                    broadcast: None,
+                });
+            } else {
+                warn!("weird inet address length: {}", ia_addr_bytearray_len);
+                continue;
+            }
+            let elem = Interface {
+                name: name.clone(),
+                addr: addr,
+            };
+
+            out.push(elem);
+        }
+    }
+    Ok(out)
+}

veilid-core/src/intf/native/utils/android/get_directories.rs

@@ -0,0 +1,42 @@
+use super::*;
+use crate::xx::*;
+
+pub fn get_files_dir() -> String {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+    // context.getFilesDir().getAbsolutePath()
+    let file = env
+        .call_method(ag.ctx.as_obj(), "getFilesDir", "()Ljava/io/File;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+    let path = env
+        .call_method(file, "getAbsolutePath", "()Ljava/lang/String;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let jstrval = env.get_string(JString::from(path)).unwrap();
+    String::from(jstrval.to_string_lossy())
+}
+
+pub fn get_cache_dir() -> String {
+    let aglock = ANDROID_GLOBALS.lock();
+    let ag = aglock.as_ref().unwrap();
+    let env = ag.vm.attach_current_thread().unwrap();
+    // context.getCacheDir().getAbsolutePath()
+    let file = env
+        .call_method(ag.ctx.as_obj(), "getCacheDir", "()Ljava/io/File;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+    let path = env
+        .call_method(file, "getAbsolutePath", "()Ljava/lang/String;", &[])
+        .unwrap()
+        .l()
+        .unwrap();
+
+    let jstrval = env.get_string(JString::from(path)).unwrap();
+    String::from(jstrval.to_string_lossy())
+}

veilid-core/src/intf/native/utils/android/mod.rs

@@ -0,0 +1,66 @@
+mod android_get_if_addrs;
+mod get_directories;
+pub use android_get_if_addrs::*;
+pub use get_directories::*;
+
+use crate::xx::*;
+use android_logger::{Config, FilterBuilder};
+use backtrace::Backtrace;
+use jni::{objects::GlobalRef, objects::JObject, objects::JString, JNIEnv, JavaVM};
+use lazy_static::*;
+use log::*;
+use std::panic;
+
+pub struct AndroidGlobals {
+    pub vm: JavaVM,
+    pub ctx: GlobalRef,
+}
+
+lazy_static! {
+    pub static ref ANDROID_GLOBALS: Arc<Mutex<Option<AndroidGlobals>>> = Arc::new(Mutex::new(None));
+}
+
+pub fn veilid_core_setup_android<'a>(
+    env: JNIEnv<'a>,
+    ctx: JObject<'a>,
+    log_tag: &'a str,
+    log_level: Level,
+) {
+    android_logger::init_once(
+        Config::default()
+            .with_min_level(log_level)
+            .with_tag(log_tag)
+            .with_filter(
+                FilterBuilder::new()
+                    .filter(Some(log_tag), log_level.to_level_filter())
+                    .build(),
+            ),
+    );
+    panic::set_hook(Box::new(|panic_info| {
+        let bt = Backtrace::new();
+        if let Some(location) = panic_info.location() {
+            error!(
+                "panic occurred in file '{}' at line {}",
+                location.file(),
+                location.line(),
+            );
+        } else {
+            error!("panic occurred but can't get location information...");
+        }
+        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
+            error!("panic payload: {:?}", a);
+        } else {
+            error!("no panic payload");
+        }
+        error!("Backtrace:\n{:?}", bt);
+    }));
+
+    *ANDROID_GLOBALS.lock() = Some(AndroidGlobals {
+        vm: env.get_java_vm().unwrap(),
+        ctx: env.new_global_ref(ctx).unwrap(),
+    });
+}

veilid-core/src/intf/native/utils/async_peek_stream.rs

@@ -0,0 +1,171 @@
+use crate::xx::*;
+use async_std::io::{Read, ReadExt, Result, Write};
+use core::task::{Context, Poll};
+use std::pin::Pin;
+
+////////
+///
+trait SendStream: Read + Write + Send + Unpin {
+    fn clone_stream(&self) -> Box<dyn SendStream>;
+}
+impl<S> SendStream for S
+where
+    S: Read + Write + Send + Clone + Unpin + 'static,
+{
+    fn clone_stream(&self) -> Box<dyn SendStream> {
+        Box::new(self.clone())
+    }
+}
+
+/////////
+///
+struct AsyncPeekStreamInner {
+    stream: Box<dyn SendStream>,
+    peekbuf: Vec<u8>,
+    peekbuf_len: usize,
+}
+
+#[derive(Clone)]
+pub struct AsyncPeekStream
+where
+    Self: Read + Write + Send + Unpin,
+{
+    inner: Arc<Mutex<AsyncPeekStreamInner>>,
+}
+
+impl AsyncPeekStream {
+    pub fn new<S>(stream: S) -> Self
+    where
+        S: Read + Write + Send + Clone + Unpin + 'static,
+    {
+        Self {
+            inner: Arc::new(Mutex::new(AsyncPeekStreamInner {
+                stream: Box::new(stream),
+                peekbuf: Vec::new(),
+                peekbuf_len: 0,
+            })),
+        }
+    }
+
+    pub async fn peek(&'_ self, buf: &'_ mut [u8]) -> Result<usize> {
+        let (mut stream, mut peekbuf, mut peekbuf_len) = {
+            let inner = self.inner.lock();
+            (
+                inner.stream.clone_stream(),
+                inner.peekbuf.clone(),
+                inner.peekbuf_len,
+            )
+        };
+        //
+        let buf_len = buf.len();
+        let mut copy_len = buf_len;
+        if buf_len > peekbuf_len {
+            //
+            peekbuf.resize(buf_len, 0u8);
+            let read_len = stream
+                .read(&mut peekbuf.as_mut_slice()[peekbuf_len..buf_len])
+                .await?;
+            peekbuf_len += read_len;
+            copy_len = peekbuf_len;
+        }
+        buf[..copy_len].copy_from_slice(&peekbuf[..copy_len]);
+
+        let mut inner = self.inner.lock();
+        inner.peekbuf = peekbuf;
+        inner.peekbuf_len = peekbuf_len;
+        Ok(copy_len)
+    }
+
+    pub async fn peek_exact(&'_ self, buf: &'_ mut [u8]) -> Result<()> {
+        let (mut stream, mut peekbuf, mut peekbuf_len) = {
+            let inner = self.inner.lock();
+            (
+                inner.stream.clone_stream(),
+                inner.peekbuf.clone(),
+                inner.peekbuf_len,
+            )
+        };
+        //
+        let buf_len = buf.len();
+        if buf_len > peekbuf_len {
+            //
+            peekbuf.resize(buf_len, 0u8);
+            stream
+                .read_exact(&mut peekbuf.as_mut_slice()[peekbuf_len..buf_len])
+                .await?;
+            peekbuf_len = buf_len;
+        }
+        buf.copy_from_slice(&peekbuf[..buf_len]);
+
+        let mut inner = self.inner.lock();
+        inner.peekbuf = peekbuf;
+        inner.peekbuf_len = peekbuf_len;
+        Ok(())
+    }
+}
+
+impl Read for AsyncPeekStream {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        //
+        let buflen = buf.len();
+        let bufcopylen = cmp::min(buflen, inner.peekbuf_len);
+        let bufreadlen = if buflen > inner.peekbuf_len {
+            buflen - inner.peekbuf_len
+        } else {
+            0
+        };
+
+        if bufreadlen > 0 {
+            match Pin::new(&mut inner.stream).poll_read(cx, &mut buf[bufcopylen..buflen]) {
+                Poll::Ready(res) => {
+                    let readlen = res?;
+                    buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+                    inner.peekbuf_len = 0;
+                    Poll::Ready(Ok(bufcopylen + readlen))
+                }
+                Poll::Pending => {
+                    if bufcopylen > 0 {
+                        buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+                        inner.peekbuf_len = 0;
+                        Poll::Ready(Ok(bufcopylen))
+                    } else {
+                        Poll::Pending
+                    }
+                }
+            }
+        } else {
+            buf[0..bufcopylen].copy_from_slice(&inner.peekbuf[0..bufcopylen]);
+            if bufcopylen == inner.peekbuf_len {
+                inner.peekbuf_len = 0;
+            } else if bufcopylen != 0 {
+                // slide buffer over by bufcopylen
+                let tail = inner.peekbuf.split_off(bufcopylen);
+                inner.peekbuf = tail;
+                inner.peekbuf_len -= bufcopylen;
+            }
+            Poll::Ready(Ok(bufcopylen))
+        }
+    }
+}
+
+impl Write for AsyncPeekStream {
+    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_write(cx, buf)
+    }
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_flush(cx)
+    }
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut inner.stream).poll_close(cx)
+    }
+}
+
+impl std::marker::Unpin for AsyncPeekStream {}

veilid-core/src/intf/native/utils/channel.rs

@@ -0,0 +1,83 @@
+pub use async_std::channel;
+
+#[derive(Debug)]
+pub struct Sender<T> {
+    imp: channel::Sender<T>,
+}
+
+impl<T> Clone for Sender<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Receiver<T> {
+    imp: channel::Receiver<T>,
+}
+
+impl<T> Clone for Receiver<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+pub fn channel<T>(cap: usize) -> (Sender<T>, Receiver<T>) {
+    let imp = channel::bounded(cap);
+    (Sender { imp: imp.0 }, Receiver { imp: imp.1 })
+}
+
+pub use channel::SendError;
+pub use channel::TrySendError;
+
+#[allow(dead_code)]
+impl<T> Sender<T> {
+    // NOTE: This needs a timeout or you could block a very long time
+    // pub async fn send(&self, msg: T) -> Result<(), SendError<T>> {
+    //     self.imp.send(msg).await
+    // }
+    pub async fn try_send(&self, msg: T) -> Result<(), TrySendError<T>> {
+        self.imp.try_send(msg)
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.capacity().unwrap()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        self.imp.is_full()
+    }
+    pub fn len(&self) -> usize {
+        self.imp.len()
+    }
+}
+
+pub use channel::RecvError;
+pub use channel::TryRecvError;
+
+#[allow(dead_code)]
+impl<T> Receiver<T> {
+    pub async fn recv(&self) -> Result<T, RecvError> {
+        self.imp.recv().await
+    }
+    pub async fn try_recv(&self) -> Result<T, TryRecvError> {
+        self.imp.try_recv()
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.capacity().unwrap()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        self.imp.is_full()
+    }
+    pub fn len(&self) -> usize {
+        self.imp.len()
+    }
+}

veilid-core/src/intf/native/utils/clone_stream.rs

@@ -0,0 +1,66 @@
+use crate::xx::*;
+use async_std::io::{Read, Result, Write};
+use core::task::{Context, Poll};
+use std::pin::Pin;
+
+pub struct CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    inner: Arc<Mutex<T>>,
+}
+
+impl<T> Clone for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<T> CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    pub fn new(t: T) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(t)),
+        }
+    }
+}
+impl<T> Read for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_read(cx, buf)
+    }
+}
+
+impl<T> Write for CloneStream<T>
+where
+    T: Read + Write + Send + Unpin,
+{
+    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_write(cx, buf)
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_flush(cx)
+    }
+
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+        let mut inner = self.inner.lock();
+        Pin::new(&mut *inner).poll_close(cx)
+    }
+}

veilid-core/src/intf/native/utils/ios/mod.rs

@@ -0,0 +1,99 @@
+use crate::xx::*;
+use backtrace::Backtrace;
+use lazy_static::*;
+use log::*;
+use simplelog::*;
+use std::fs::OpenOptions;
+use std::panic;
+use std::path::{Path, PathBuf};
+
+pub struct IOSGlobals {}
+
+lazy_static! {
+    pub static ref IOS_GLOBALS: Arc<Mutex<Option<IOSGlobals>>> = Arc::new(Mutex::new(None));
+}
+
+pub fn veilid_core_setup_ios<'a>(
+    log_tag: &'a str,
+    terminal_log: Option<Level>,
+    file_log: Option<(Level, &Path)>,
+) {
+    if let Err(e) = veilid_core_setup_ios_internal(log_tag, terminal_log, file_log) {
+        panic!("failed to set up veilid-core: {}", e);
+    }
+}
+
+fn veilid_core_setup_ios_internal<'a>(
+    _log_tag: &'a str,
+    terminal_log: Option<Level>,
+    file_log: Option<(Level, &Path)>,
+) -> Result<(), String> {
+    let mut logs: Vec<Box<dyn SharedLogger>> = Vec::new();
+
+    let mut cb = ConfigBuilder::new();
+    cb.add_filter_ignore_str("async_std");
+    cb.add_filter_ignore_str("async_io");
+    cb.add_filter_ignore_str("polling");
+    cb.add_filter_ignore_str("rustls");
+    cb.add_filter_ignore_str("async_tungstenite");
+    cb.add_filter_ignore_str("tungstenite");
+
+    if let Some(level) = terminal_log {
+        logs.push(TermLogger::new(
+            level.to_level_filter(),
+            cb.build(),
+            TerminalMode::Mixed,
+            ColorChoice::Auto,
+        ))
+    }
+    if let Some((level, log_path)) = file_log {
+        let logfile = OpenOptions::new()
+            .truncate(true)
+            .create(true)
+            .write(true)
+            .open(log_path)
+            .map_err(|e| {
+                format!(
+                    "log open error: {} path={:?} all_dirs={:?}",
+                    e,
+                    log_path,
+                    std::fs::read_dir(std::env::var("HOME").unwrap())
+                        .unwrap()
+                        .map(|d| d.unwrap().path())
+                        .collect::<Vec<PathBuf>>()
+                )
+            })?;
+        logs.push(WriteLogger::new(
+            level.to_level_filter(),
+            cb.build(),
+            logfile,
+        ))
+    }
+    CombinedLogger::init(logs).map_err(|e| format!("logger init error: {}", e))?;
+
+    panic::set_hook(Box::new(|panic_info| {
+        let bt = Backtrace::new();
+        if let Some(location) = panic_info.location() {
+            error!(
+                "panic occurred in file '{}' at line {}",
+                location.file(),
+                location.line(),
+            );
+        } else {
+            error!("panic occurred but can't get location information...");
+        }
+        if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            error!("panic payload: {:?}", s);
+        } else if let Some(a) = panic_info.payload().downcast_ref::<std::fmt::Arguments>() {
+            error!("panic payload: {:?}", a);
+        } else {
+            error!("no panic payload");
+        }
+        error!("Backtrace:\n{:?}", bt);
+    }));
+
+    *IOS_GLOBALS.lock() = Some(IOSGlobals {});
+    Ok(())
+}

veilid-core/src/intf/native/utils/mod.rs

@@ -0,0 +1,8 @@
+#[cfg(target_os = "android")]
+pub mod android;
+pub mod async_peek_stream;
+pub mod channel;
+pub mod clone_stream;
+#[cfg(target_os = "ios")]
+pub mod ios;
+pub mod network_interfaces;

veilid-core/src/intf/native/utils/network_interfaces.rs

@@ -0,0 +1,110 @@
+#[cfg(target_os = "android")]
+pub use super::android::*;
+use crate::xx::*;
+#[cfg(not(target_os = "android"))]
+pub use if_addrs::*;
+
+#[derive(PartialEq, Eq, Clone)]
+pub struct NetworkInterface {
+    name: String,
+    is_loopback: bool,
+    addrs: Vec<IfAddr>,
+}
+
+#[allow(dead_code)]
+impl NetworkInterface {
+    pub fn new(name: String, is_loopback: bool) -> Self {
+        Self {
+            name: name,
+            is_loopback: is_loopback,
+            addrs: Vec::new(),
+        }
+    }
+    pub fn name(&self) -> String {
+        self.name.clone()
+    }
+    pub fn is_loopback(&self) -> bool {
+        self.is_loopback
+    }
+    pub fn primary_ipv4(&self) -> Option<Ipv4Addr> {
+        for x in self.addrs.iter() {
+            match x {
+                IfAddr::V4(a) => return Some(a.ip.clone()),
+                _ => (),
+            };
+        }
+        None
+    }
+    pub fn primary_ipv6(&self) -> Option<Ipv6Addr> {
+        for x in self.addrs.iter() {
+            match x {
+                IfAddr::V6(a) => return Some(a.ip.clone()),
+                _ => (),
+            };
+        }
+        None
+    }
+}
+
+pub struct NetworkInterfaces {
+    valid: bool,
+    interfaces: BTreeMap<String, NetworkInterface>,
+}
+
+#[allow(dead_code)]
+impl NetworkInterfaces {
+    pub fn new() -> Self {
+        Self {
+            valid: false,
+            interfaces: BTreeMap::new(),
+        }
+    }
+    pub fn is_valid(&self) -> bool {
+        self.valid
+    }
+    pub fn clear(&mut self) {
+        self.interfaces.clear();
+        self.valid = false;
+    }
+    // returns Ok(false) if refresh had no changes, Ok(true) if changes were present
+    pub fn refresh(&mut self) -> Result<bool, String> {
+        self.valid = false;
+
+        let last_interfaces = core::mem::take(&mut self.interfaces);
+
+        let mut intfs = match get_if_addrs() {
+            Err(e) => {
+                return Err(format!("failed to refresh network interfaces: {}", e));
+            }
+            Ok(v) => v,
+        };
+        intfs.sort();
+
+        // debug!("{} interfaces found", intfs.len());
+        for intf in intfs {
+            // trace!("interface {} at {}", &intf.name, &intf.addr.ip());
+            let ni = match self.interfaces.get_mut(&intf.name) {
+                None => {
+                    self.interfaces.insert(
+                        intf.name.clone(),
+                        NetworkInterface::new(intf.name.clone(), intf.is_loopback()),
+                    );
+                    self.interfaces.get_mut(&intf.name).unwrap()
+                }
+                Some(v) => v,
+            };
+
+            ni.addrs.push(intf.addr.clone());
+        }
+
+        self.valid = true;
+
+        Ok(last_interfaces != self.interfaces)
+    }
+    pub fn len(&self) -> usize {
+        self.interfaces.len()
+    }
+    pub fn iter(&self) -> std::collections::btree_map::Iter<String, NetworkInterface> {
+        self.interfaces.iter()
+    }
+}

veilid-core/src/intf/table_db.rs

@@ -0,0 +1,138 @@
+use crate::intf::*;
+use crate::xx::*;
+use serde::{Deserialize, Serialize};
+use serde_cbor;
+
+cfg_if! {
+    if #[cfg(target_arch = "wasm32")] {
+        use keyvaluedb_web::*;
+    } else {
+        use keyvaluedb_sqlite::*;
+    }
+}
+
+pub struct TableDBInner {
+    table: String,
+    table_store: TableStore,
+    database: Database,
+}
+
+impl Drop for TableDBInner {
+    fn drop(&mut self) {
+        self.table_store.on_table_db_drop(self.table.clone());
+    }
+}
+
+#[derive(Clone)]
+pub struct TableDB {
+    inner: Arc<Mutex<TableDBInner>>,
+}
+
+impl TableDB {
+    pub fn new(table: String, table_store: TableStore, database: Database) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(TableDBInner {
+                table: table,
+                table_store: table_store.clone(),
+                database: database,
+            })),
+        }
+    }
+
+    pub fn try_new_from_weak_inner(weak_inner: Weak<Mutex<TableDBInner>>) -> Option<Self> {
+        match weak_inner.upgrade() {
+            Some(table_db_inner) => Some(Self {
+                inner: table_db_inner,
+            }),
+            None => None,
+        }
+    }
+
+    pub fn weak_inner(&self) -> Weak<Mutex<TableDBInner>> {
+        Arc::downgrade(&self.inner)
+    }
+
+    pub async fn get_column_count(&self) -> Result<u32, String> {
+        let db = &self.inner.lock().database;
+        db.num_columns()
+            .map_err(|e| format!("failed to get column count: {}", e))
+    }
+
+    pub async fn get_keys(&self, col: u32) -> Result<Vec<Box<[u8]>>, String> {
+        let db = &self.inner.lock().database;
+        let mut out: Vec<Box<[u8]>> = Vec::new();
+        db.iter(col, None, &mut |kv| {
+            out.push(kv.0.clone().into_boxed_slice());
+            Ok(true)
+        })
+        .map_err(|e| format!("failed to get keys for column {}: {}", col, e))?;
+        Ok(out)
+    }
+
+    pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> Result<(), String> {
+        let db = &self.inner.lock().database;
+        let mut dbt = db.transaction();
+        dbt.put(col, key, value);
+        db.write(dbt)
+            .map_err(|e| format!("failed to store key {:?}: {}", key, e))
+    }
+
+    pub async fn store_cbor<T>(&self, col: u32, key: &[u8], value: &T) -> Result<(), String>
+    where
+        T: Serialize,
+    {
+        let v = serde_cbor::to_vec(value).map_err(|_| "couldn't store as CBOR".to_owned())?;
+
+        let db = &self.inner.lock().database;
+        let mut dbt = db.transaction();
+        dbt.put(col, key, v.as_slice());
+        db.write(dbt)
+            .map_err(|e| format!("failed to store key {:?}: {}", key, e))
+    }
+
+    pub async fn load(&self, col: u32, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
+        let db = &self.inner.lock().database;
+        db.get(col, key)
+            .map_err(|e| format!("failed to get key {:?}: {}", key, e))
+    }
+
+    pub async fn load_cbor<T>(&self, col: u32, key: &[u8]) -> Result<Option<T>, String>
+    where
+        T: for<'de> Deserialize<'de>,
+    {
+        let db = &self.inner.lock().database;
+        let out = db
+            .get(col, key)
+            .map_err(|e| format!("failed to get key {:?}: {}", key, e))?;
+        let b = match out {
+            Some(v) => v,
+            None => {
+                return Ok(None);
+            }
+        };
+        let obj = match serde_cbor::from_slice::<T>(&b) {
+            Ok(value) => value,
+            Err(e) => {
+                return Err(format!("failed to deserialize: {}", e));
+            }
+        };
+        Ok(Some(obj))
+    }
+
+    pub async fn delete(&self, col: u32, key: &[u8]) -> Result<bool, String> {
+        let db = &self.inner.lock().database;
+        let found = db
+            .get(col, key)
+            .map_err(|e| format!("failed to get key {:?}: {}", key, e))?;
+        match found {
+            None => Ok(false),
+            Some(_) => {
+                let mut dbt = db.transaction();
+                dbt.delete(col, key);
+                db.write(dbt)
+                    .map_err(|e| format!("failed to delete key {:?}: {}", key, e))?;
+                Ok(true)
+            }
+        }
+    }
+}

veilid-core/src/intf/wasm/block_store.rs

@@ -0,0 +1 @@
+

veilid-core/src/intf/wasm/mod.rs

@@ -0,0 +1,12 @@
+mod block_store;
+mod network;
+mod protected_store;
+mod system;
+mod table_store;
+pub mod utils;
+
+pub use block_store::*;
+pub use network::*;
+pub use protected_store::*;
+pub use system::*;
+pub use table_store::*;

veilid-core/src/intf/wasm/network/mod.rs

@@ -0,0 +1,179 @@
+mod protocol;
+
+use crate::intf::*;
+use crate::network_manager::*;
+use crate::routing_table::*;
+use crate::*;
+use protocol::ws::WebsocketProtocolHandler;
+pub use protocol::*;
+
+/////////////////////////////////////////////////////////////////
+
+struct NetworkInner {
+    network_manager: NetworkManager,
+    stop_network: Eventual,
+    network_needs_restart: bool,
+    //join_handle: TryJoin?
+}
+
+#[derive(Clone)]
+pub struct Network {
+    inner: Arc<Mutex<NetworkInner>>,
+}
+
+impl Network {
+    fn new_inner(network_manager: NetworkManager) -> NetworkInner {
+        NetworkInner {
+            network_manager: network_manager,
+            stop_network: Eventual::new(),
+            network_needs_restart: false,
+            //join_handle: None,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
+        }
+    }
+
+    /////////////////////////////////////////////////////////////////
+
+    async fn send_data_to_existing_connection(
+        &self,
+        descriptor: &ConnectionDescriptor,
+        data: Vec<u8>,
+    ) -> Result<Option<Vec<u8>>, String> {
+        match descriptor.protocol_type() {
+            ProtocolType::UDP => return Err("no support for udp protocol".to_owned()),
+            ProtocolType::TCP => return Err("no support for tcp protocol".to_owned()),
+            ProtocolType::WS | ProtocolType::WSS => {
+                // find an existing connection in the connection table if one exists
+                let network_manager = self.inner.lock().network_manager.clone();
+                if let Some(entry) = network_manager
+                    .connection_table()
+                    .get_connection(&descriptor)
+                {
+                    // connection exists, send over it
+                    entry
+                        .conn
+                        .send(data)
+                        .await
+                        .map_err(|_| "failed to send ws message".to_owned())?;
+                    // Data was consumed
+                    return Ok(None);
+                }
+            }
+        }
+        // connection or local socket didn't exist, we'll need to use dialinfo to create one
+        // Pass the data back out so we don't own it any more
+        Ok(Some(data))
+    }
+
+    pub async fn send_data_unbound_to_dial_info(
+        &self,
+        dial_info: &DialInfo,
+        data: Vec<u8>,
+    ) -> Result<(), String> {
+        let network_manager = self.inner.lock().network_manager.clone();
+
+        match &dial_info {
+            DialInfo::UDP(_) => return Err("no support for UDP protocol".to_owned()),
+            DialInfo::TCP(_) => return Err("no support for TCP protocol".to_owned()),
+            DialInfo::WS(_) => Err("WS protocol does not support unbound messages".to_owned()),
+            DialInfo::WSS(_) => Err("WSS protocol does not support unbound messages".to_owned()),
+        }
+    }
+    pub async fn send_data_to_dial_info(
+        &self,
+        dial_info: &DialInfo,
+        data: Vec<u8>,
+    ) -> Result<(), String> {
+        let network_manager = self.inner.lock().network_manager.clone();
+
+        let conn = match &dial_info {
+            DialInfo::UDP(_) => return Err("no support for UDP protocol".to_owned()),
+            DialInfo::TCP(_) => return Err("no support for TCP protocol".to_owned()),
+            DialInfo::WS(_) => WebsocketProtocolHandler::connect(network_manager, dial_info)
+                .await
+                .map_err(|_| "failed to connect to WS dial info".to_owned())?,
+            DialInfo::WSS(_) => WebsocketProtocolHandler::connect(network_manager, dial_info)
+                .await
+                .map_err(|_| "failed to connect to WSS dial info".to_owned())?,
+        };
+
+        conn.send(data)
+            .await
+            .map_err(|_| "failed to send data to dial info".to_owned())
+    }
+
+    pub async fn send_data(&self, node_ref: NodeRef, data: Vec<u8>) -> Result<(), String> {
+        let dial_info = node_ref.dial_info();
+        let descriptor = node_ref.last_connection();
+
+        // First try to send data to the last socket we've seen this peer on
+        let di_data = if let Some(descriptor) = descriptor {
+            match self
+                .clone()
+                .send_data_to_existing_connection(&descriptor, data)
+                .await?
+            {
+                None => {
+                    return Ok(());
+                }
+                Some(d) => d,
+            }
+        } else {
+            data
+        };
+
+        // If that fails, try to make a connection or reach out to the peer via its dial info
+        if let Some(di) = dial_info {
+            self.clone().send_data_to_dial_info(&di, di_data).await
+        } else {
+            Err("couldn't send data, no dial info or peer address".to_owned())
+        }
+    }
+
+    /////////////////////////////////////////////////////////////////
+
+    pub async fn startup(&self) -> Result<(), String> {
+        //let network_manager = self.inner.lock().network_manager.clone();
+        //let config_shared = network_manager.core().config();
+        //let config = config_shared.get();
+
+        Ok(())
+    }
+
+    pub fn needs_restart(&self) -> bool {
+        self.inner.lock().network_needs_restart
+    }
+
+    pub async fn shutdown(&self) {
+        trace!("stopping network");
+
+        // Reset state
+        let network_manager = self.inner.lock().network_manager.clone();
+        let routing_table = network_manager.routing_table();
+
+        // Drop all dial info
+        routing_table.clear_local_dial_info();
+        routing_table.clear_public_dial_info();
+
+        // Cancels all async background tasks by dropping join handles
+        *self.inner.lock() = Self::new_inner(network_manager);
+
+        trace!("network stopped");
+    }
+
+    //////////////////////////////////////////
+    pub fn get_network_class(&self) -> NetworkClass {
+        // xxx eventually detect tor browser?
+        return NetworkClass::WebApp;
+    }
+
+    //////////////////////////////////////////
+    pub async fn tick(&self) -> Result<(), String> {
+        Ok(())
+    }
+}

veilid-core/src/intf/wasm/network/protocol/mod.rs

@@ -0,0 +1,48 @@
+pub mod wrtc;
+pub mod ws;
+
+use crate::veilid_api::ProtocolType;
+use crate::xx::*;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DummyNetworkConnection {}
+
+impl DummyNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        ProtocolType::UDP
+    }
+    pub fn send(&self, _message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        Box::pin(async { Ok(()) })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        Box::pin(async { Ok(Vec::new()) })
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum NetworkConnection {
+    Dummy(DummyNetworkConnection),
+    WS(ws::WebsocketNetworkConnection),
+    //WebRTC(wrtc::WebRTCNetworkConnection),
+}
+
+impl NetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        match self {
+            Self::Dummy(d) => d.protocol_type(),
+            Self::WS(w) => w.protocol_type(),
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        match self {
+            Self::Dummy(d) => d.send(message),
+            Self::WS(w) => w.send(message),
+        }
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        match self {
+            Self::Dummy(d) => d.recv(),
+            Self::WS(w) => w.recv(),
+        }
+    }
+}

veilid-core/src/intf/wasm/network/protocol/ws.rs

@@ -0,0 +1,116 @@
+use crate::intf::*;
+use crate::network_manager::{NetworkManager, MAX_MESSAGE_SIZE};
+use crate::*;
+use alloc::fmt;
+use futures_util::stream::StreamExt;
+use web_sys::WebSocket;
+use ws_stream_wasm::*;
+
+struct WebsocketNetworkConnectionInner {
+    _ws_meta: WsMeta,
+    ws_stream: WsStream,
+    ws: WebSocket,
+}
+
+#[derive(Clone)]
+pub struct WebsocketNetworkConnection {
+    tls: bool,
+    inner: Arc<Mutex<WebsocketNetworkConnectionInner>>,
+}
+
+impl fmt::Debug for WebsocketNetworkConnection {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", core::any::type_name::<Self>())
+    }
+}
+
+impl PartialEq for WebsocketNetworkConnection {
+    fn eq(&self, other: &Self) -> bool {
+        self.tls == other.tls && Arc::as_ptr(&self.inner) == Arc::as_ptr(&other.inner)
+    }
+}
+
+impl Eq for WebsocketNetworkConnection {}
+
+impl WebsocketNetworkConnection {
+    pub fn new(tls: bool, ws_meta: WsMeta, ws_stream: WsStream) -> Self {
+        let ws = ws_stream.wrapped().clone();
+        Self {
+            tls: tls,
+            inner: Arc::new(Mutex::new(WebsocketNetworkConnectionInner {
+                _ws_meta: ws_meta,
+                ws_stream: ws_stream,
+                ws: ws,
+            })),
+        }
+    }
+}
+
+impl WebsocketNetworkConnection {
+    pub fn protocol_type(&self) -> ProtocolType {
+        if self.tls {
+            ProtocolType::WSS
+        } else {
+            ProtocolType::WS
+        }
+    }
+    pub fn send(&self, message: Vec<u8>) -> SystemPinBoxFuture<Result<(), ()>> {
+        let inner = self.inner.clone();
+        Box::pin(async move {
+            if message.len() > MAX_MESSAGE_SIZE {
+                return Err(());
+            }
+            inner.lock().ws.send_with_u8_array(&message).map_err(drop)
+        })
+    }
+    pub fn recv(&self) -> SystemPinBoxFuture<Result<Vec<u8>, ()>> {
+        let inner = self.inner.clone();
+        Box::pin(async move {
+            let out = match inner.lock().ws_stream.next().await {
+                Some(WsMessage::Binary(v)) => v,
+                _ => {
+                    trace!("websocket recv failed");
+                    return Err(());
+                }
+            };
+            if out.len() > MAX_MESSAGE_SIZE {
+                Err(())
+            } else {
+                Ok(out)
+            }
+        })
+    }
+}
+
+///////////////////////////////////////////////////////////
+///
+
+pub struct WebsocketProtocolHandler {}
+
+impl WebsocketProtocolHandler {
+    pub async fn connect(
+        network_manager: NetworkManager,
+        dial_info: &DialInfo,
+    ) -> Result<NetworkConnection, ()> {
+        let url = dial_info.to_url_string(None);
+        let (tls, host, port, protocol_type) = match dial_info {
+            DialInfo::WS(ws) => (false, ws.fqdn.clone(), ws.port, ProtocolType::WS),
+            DialInfo::WSS(wss) => (true, wss.fqdn.clone(), wss.port, ProtocolType::WSS),
+            _ => return Err(()),
+        };
+
+        let peer_addr = PeerAddress::new(Address::from_str(&host)?, port, protocol_type);
+
+        let (ws, wsio) = match WsMeta::connect(url, None).await {
+            Ok(conn) => conn,
+            Err(_) => return Err(()),
+        };
+
+        let conn = NetworkConnection::WS(WebsocketNetworkConnection::new(tls, ws, wsio));
+        network_manager
+            .on_new_connection(ConnectionDescriptor::new_no_local(peer_addr), conn.clone())
+            .await?;
+
+        Ok(conn)
+    }
+}

veilid-core/src/intf/wasm/protected_store.rs

@@ -0,0 +1,185 @@
+use super::utils;
+use crate::xx::*;
+use js_sys::*;
+use wasm_bindgen_futures::*;
+use web_sys::*;
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen(catch, js_name = setPassword, js_namespace = ["global", "wasmhost", "keytar"])]
+    fn keytar_setPassword(service: &str, account: &str, password: &str)
+        -> Result<Promise, JsValue>;
+    #[wasm_bindgen(catch, js_name = getPassword, js_namespace = ["global", "wasmhost", "keytar"])]
+    fn keytar_getPassword(service: &str, account: &str) -> Result<Promise, JsValue>;
+    #[wasm_bindgen(catch, js_name = deletePassword, js_namespace = ["global", "wasmhost", "keytar"])]
+    fn keytar_deletePassword(service: &str, account: &str) -> Result<Promise, JsValue>;
+}
+
+fn keyring_name(namespace: &str) -> String {
+    if namespace.len() == 0 {
+        "veilid".to_owned()
+    } else {
+        format!("veilid_{}", namespace)
+    }
+}
+
+fn browser_key_name(namespace: &str, key: &str) -> String {
+    if namespace.len() == 0 {
+        format!("__veilid_secret_{}", key)
+    } else {
+        format!("__veilid_{}_secret_{}", namespace, key)
+    }
+}
+
+pub async fn save_user_secret_string(
+    namespace: &str,
+    key: &str,
+    value: &str,
+) -> Result<bool, String> {
+    if utils::is_nodejs() {
+        let prev = match JsFuture::from(
+            keytar_getPassword(keyring_name(namespace).as_str(), key)
+                .map_err(|_| "exception thrown".to_owned())?,
+        )
+        .await
+        {
+            Ok(v) => v.is_truthy(),
+            Err(_) => false,
+        };
+
+        match JsFuture::from(
+            keytar_setPassword(keyring_name(namespace).as_str(), key, value)
+                .map_err(|_| "exception thrown".to_owned())?,
+        )
+        .await
+        {
+            Ok(_) => {}
+            Err(_) => return Err("Failed to set password".to_owned()),
+        }
+
+        Ok(prev)
+    } else if utils::is_browser() {
+        let win = match window() {
+            Some(w) => w,
+            None => {
+                return Err("failed to get window".to_owned());
+            }
+        };
+
+        let ls = match win
+            .local_storage()
+            .map_err(|_| "exception getting local storage".to_owned())?
+        {
+            Some(l) => l,
+            None => {
+                return Err("failed to get local storage".to_owned());
+            }
+        };
+
+        let vkey = browser_key_name(namespace, key);
+
+        let prev = match ls
+            .get_item(&vkey)
+            .map_err(|_| "exception_thrown".to_owned())?
+        {
+            Some(_) => true,
+            None => false,
+        };
+
+        ls.set_item(&vkey, value)
+            .map_err(|_| "exception_thrown".to_owned())?;
+
+        Ok(prev)
+    } else {
+        Err("unimplemented".to_owned())
+    }
+}
+
+pub async fn load_user_secret_string(namespace: &str, key: &str) -> Result<Option<String>, String> {
+    if utils::is_nodejs() {
+        let prev = match JsFuture::from(
+            keytar_getPassword(keyring_name(namespace).as_str(), key)
+                .map_err(|_| "exception thrown".to_owned())?,
+        )
+        .await
+        {
+            Ok(p) => p,
+            Err(_) => JsValue::UNDEFINED,
+        };
+
+        if prev.is_undefined() || prev.is_null() {
+            return Ok(None);
+        }
+
+        Ok(prev.as_string())
+    } else if utils::is_browser() {
+        let win = match window() {
+            Some(w) => w,
+            None => {
+                return Err("failed to get window".to_owned());
+            }
+        };
+
+        let ls = match win
+            .local_storage()
+            .map_err(|_| "exception getting local storage".to_owned())?
+        {
+            Some(l) => l,
+            None => {
+                return Err("failed to get local storage".to_owned());
+            }
+        };
+
+        let vkey = browser_key_name(namespace, key);
+
+        ls.get_item(&vkey)
+            .map_err(|_| "exception_thrown".to_owned())
+    } else {
+        Err("unimplemented".to_owned())
+    }
+}
+
+pub async fn remove_user_secret_string(namespace: &str, key: &str) -> Result<bool, String> {
+    if utils::is_nodejs() {
+        match JsFuture::from(
+            keytar_deletePassword("veilid", key).map_err(|_| "exception thrown".to_owned())?,
+        )
+        .await
+        {
+            Ok(v) => Ok(v.is_truthy()),
+            Err(_) => Err("Failed to delete".to_owned()),
+        }
+    } else if utils::is_browser() {
+        let win = match window() {
+            Some(w) => w,
+            None => {
+                return Err("failed to get window".to_owned());
+            }
+        };
+
+        let ls = match win
+            .local_storage()
+            .map_err(|_| "exception getting local storage".to_owned())?
+        {
+            Some(l) => l,
+            None => {
+                return Err("failed to get local storage".to_owned());
+            }
+        };
+
+        let vkey = browser_key_name(namespace, key);
+
+        match ls
+            .get_item(&vkey)
+            .map_err(|_| "exception_thrown".to_owned())?
+        {
+            Some(_) => {
+                ls.delete(&vkey)
+                    .map_err(|_| "exception_thrown".to_owned())?;
+                Ok(true)
+            }
+            None => Ok(false),
+        }
+    } else {
+        Err("unimplemented".to_owned())
+    }
+}

veilid-core/src/intf/wasm/system.rs

@@ -0,0 +1,154 @@
+use super::utils;
+use crate::xx::*;
+use crate::*;
+pub use async_executors::JoinHandle;
+use async_executors::{Bindgen, LocalSpawnHandleExt /*, SpawnHandleExt*/};
+use core::fmt;
+use futures_util::future::{select, Either};
+use js_sys::*;
+use wasm_bindgen_futures::*;
+use web_sys::*;
+
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen(catch, structural, js_namespace = global, js_name = setTimeout)]
+    fn nodejs_global_set_timeout_with_callback_and_timeout_and_arguments_0(
+        handler: &::js_sys::Function,
+        timeout: u32,
+    ) -> Result<JsValue, JsValue>;
+}
+
+pub fn get_timestamp() -> u64 {
+    if utils::is_browser() {
+        return (Date::now() * 1000.0f64) as u64;
+    } else if utils::is_nodejs() {
+        return (Date::now() * 1000.0f64) as u64;
+    } else {
+        panic!("WASM requires browser or nodejs environment");
+    }
+}
+
+pub fn random_bytes(dest: &mut [u8]) -> Result<(), String> {
+    let len = dest.len();
+    let u32len = len / 4;
+    let remlen = len % 4;
+
+    for n in 0..u32len {
+        let r = (Math::random() * (u32::max_value() as f64)) as u32;
+
+        dest[n * 4 + 0] = (r & 0xFF) as u8;
+        dest[n * 4 + 1] = ((r >> 8) & 0xFF) as u8;
+        dest[n * 4 + 2] = ((r >> 16) & 0xFF) as u8;
+        dest[n * 4 + 3] = ((r >> 24) & 0xFF) as u8;
+    }
+    if remlen > 0 {
+        let r = (Math::random() * (u32::max_value() as f64)) as u32;
+        for n in 0..remlen {
+            dest[u32len * 4 + n] = ((r >> (n * 8)) & 0xFF) as u8;
+        }
+    }
+
+    Ok(())
+}
+
+pub fn get_random_u32() -> u32 {
+    (Math::random() * (u32::max_value() as f64)) as u32
+}
+
+pub fn get_random_u64() -> u64 {
+    let v1: u32 = get_random_u32();
+    let v2: u32 = get_random_u32();
+    ((v1 as u64) << 32) | ((v2 as u32) as u64)
+}
+
+pub async fn sleep(millis: u32) {
+    if utils::is_browser() {
+        let wait_millis = if millis > u32::MAX {
+            i32::MAX
+        } else {
+            millis as i32
+        };
+        let promise = Promise::new(&mut |yes, _| {
+            let win = window().unwrap();
+            win.set_timeout_with_callback_and_timeout_and_arguments_0(&yes, wait_millis)
+                .unwrap();
+        });
+
+        JsFuture::from(promise).await.unwrap();
+    } else if utils::is_nodejs() {
+        let promise = Promise::new(&mut |yes, _| {
+            nodejs_global_set_timeout_with_callback_and_timeout_and_arguments_0(&yes, millis)
+                .unwrap();
+        });
+
+        JsFuture::from(promise).await.unwrap();
+    } else {
+        panic!("WASM requires browser or nodejs environment");
+    }
+}
+
+pub fn spawn<Out>(future: impl Future<Output = Out> + 'static) -> JoinHandle<Out>
+where
+    Out: Send + 'static,
+{
+    Bindgen
+        .spawn_handle_local(future)
+        .expect("wasm-bindgen-futures spawn should never error out")
+}
+
+pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> JoinHandle<Out>
+where
+    Out: 'static,
+{
+    Bindgen
+        .spawn_handle_local(future)
+        .expect("wasm-bindgen-futures spawn_local should never error out")
+}
+
+pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SystemPinBoxFuture<()>
+where
+    F: Fn() -> FUT + 'static,
+    FUT: Future<Output = ()>,
+{
+    let e = Eventual::new();
+
+    let ie = e.clone();
+    let jh = spawn_local(Box::pin(async move {
+        while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
+            callback().await;
+        }
+    }));
+
+    Box::pin(async move {
+        e.resolve().await;
+        jh.await;
+    })
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct TimeoutError {
+    _private: (),
+}
+
+//impl Error for TimeoutError {}
+
+impl fmt::Display for TimeoutError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        "future has timed out".fmt(f)
+    }
+}
+
+pub async fn timeout<F, T>(dur_ms: u32, f: F) -> Result<T, TimeoutError>
+where
+    F: Future<Output = T>,
+{
+    match select(Box::pin(intf::sleep(dur_ms)), Box::pin(f)).await {
+        Either::Left((_x, _b)) => Err(TimeoutError { _private: () }),
+        Either::Right((y, _a)) => Ok(y),
+    }
+}
+
+// xxx: for now until wasm threads are more stable, and/or we bother with web workers
+pub fn get_concurrency() -> u32 {
+    1
+}

veilid-core/src/intf/wasm/table_store.rs

@@ -0,0 +1,118 @@
+use crate::intf::table_db::*;
+use crate::intf::*;
+use crate::*;
+use keyvaluedb_web::*;
+
+struct TableStoreInner {
+    config: VeilidConfig,
+    opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
+}
+
+#[derive(Clone)]
+pub struct TableStore {
+    inner: Arc<Mutex<TableStoreInner>>,
+}
+
+impl TableStore {
+    fn new_inner(config: VeilidConfig) -> TableStoreInner {
+        TableStoreInner {
+            config: config,
+            opened: BTreeMap::new(),
+        }
+    }
+    pub fn new(config: VeilidConfig) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(config))),
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        Ok(())
+    }
+
+    pub async fn terminate(&self) {
+        assert!(
+            self.inner.lock().opened.len() == 0,
+            "all open databases should have been closed"
+        );
+    }
+
+    pub fn on_table_db_drop(&self, table: String) {
+        let mut inner = self.inner.lock();
+        match inner.opened.remove(&table) {
+            Some(_) => (),
+            None => {
+                assert!(false, "should have removed an item");
+            }
+        }
+    }
+
+    fn get_table_name(inner: &TableStoreInner, table: &str) -> Result<String, String> {
+        if !table
+            .chars()
+            .all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
+        {
+            return Err(format!("table name '{}' is invalid", table));
+        }
+        let c = inner.config.get();
+        let namespace = c.namespace.clone();
+        Ok(if namespace.len() == 0 {
+            format!("{}", table)
+        } else {
+            format!("_ns_{}_{}", namespace, table)
+        })
+    }
+
+    pub async fn open(&self, name: &str, column_count: u32) -> Result<TableDB, String> {
+        let mut inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
+            match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) {
+                Some(tdb) => {
+                    return Ok(tdb);
+                }
+                None => {
+                    inner.opened.remove(&table_name);
+                }
+            };
+        }
+        let db = Database::open(table_name.clone(), column_count)
+            .await
+            .map_err(|e| format!("failed to open tabledb at: {} ({})", table_name, e))?;
+
+        let table_db = TableDB::new(table_name.clone(), self.clone(), db);
+
+        inner.opened.insert(table_name, table_db.weak_inner());
+
+        Ok(table_db)
+    }
+
+    pub async fn delete(&self, name: &str) -> Result<bool, String> {
+        trace!("TableStore::delete {}", name);
+        let inner = self.inner.lock();
+        let table_name = Self::get_table_name(&*inner, name)?;
+
+        if inner.opened.contains_key(&table_name) {
+            trace!(
+                "TableStore::delete {}: Not deleting, still open.",
+                table_name
+            );
+            return Err("Not deleting table that is still opened".to_owned());
+        }
+
+        if utils::is_nodejs() {
+            Err("unimplemented".to_owned())
+        } else if utils::is_browser() {
+            let out = match Database::delete(table_name.clone()).await {
+                Ok(_) => true,
+                Err(_) => false,
+            };
+            //.map_err(|e| format!("failed to delete tabledb at: {} ({})", table_name, e))?;
+            trace!("TableStore::deleted {}", table_name);
+            Ok(out)
+        } else {
+            Err("unimplemented".to_owned())
+        }
+    }
+}

veilid-core/src/intf/wasm/utils/channel.rs

@@ -0,0 +1,140 @@
+use crate::xx::*;
+use alloc::collections::VecDeque;
+
+#[derive(Debug)]
+pub struct Channel<T> {
+    items: VecDeque<T>,
+    cap: usize,
+    eventual: Eventual,
+}
+
+#[derive(Debug)]
+pub struct Sender<T> {
+    imp: Arc<Mutex<Channel<T>>>,
+}
+
+impl<T> Clone for Sender<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Receiver<T> {
+    imp: Arc<Mutex<Channel<T>>>,
+}
+
+impl<T> Clone for Receiver<T> {
+    fn clone(&self) -> Self {
+        Self {
+            imp: self.imp.clone(),
+        }
+    }
+}
+
+pub fn channel<T>(cap: usize) -> (Sender<T>, Receiver<T>) {
+    let imp = Channel {
+        items: VecDeque::with_capacity(cap),
+        cap: cap,
+        eventual: Eventual::new(),
+    };
+
+    let imparc = Arc::new(Mutex::new(imp));
+    (
+        Sender {
+            imp: imparc.clone(),
+        },
+        Receiver {
+            imp: imparc.clone(),
+        },
+    )
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum TrySendError<T> {
+    Full(T),
+    Disconnected(T),
+}
+
+impl<T> Sender<T> {
+    // NOTE: This needs a timeout or you could block a very long time
+    // pub async fn send(&self, msg: T) -> Result<(), SendError<T>> {
+    //     xxx
+    // }
+
+    pub async fn try_send(&self, msg: T) -> Result<(), TrySendError<T>> {
+        let eventual = {
+            let mut inner = self.imp.lock();
+            if inner.items.len() == inner.cap {
+                return Err(TrySendError::Full(msg));
+            }
+            let empty = inner.items.is_empty();
+            inner.items.push_back(msg);
+            if empty {
+                Some(inner.eventual.clone())
+            } else {
+                None
+            }
+        };
+        if let Some(e) = eventual {
+            e.resolve().await;
+        }
+        Ok(())
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.lock().cap
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.lock().items.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        let inner = self.imp.lock();
+        inner.items.len() == inner.cap
+    }
+    pub fn len(&self) -> usize {
+        self.imp.lock().items.len()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct RecvError;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum TryRecvError {
+    Empty,
+    Disconnected,
+}
+
+impl<T> Receiver<T> {
+    pub async fn recv(&self) -> Result<T, RecvError> {
+        let eventual = {
+            let inner = self.imp.lock();
+            inner.eventual.clone()
+        };
+        while self.is_empty() {
+            eventual.instance_clone(true).await;
+        }
+        Ok(self.imp.lock().items.pop_front().unwrap())
+    }
+    pub async fn try_recv(&self) -> Result<T, TryRecvError> {
+        if self.is_empty() {
+            return Err(TryRecvError::Empty);
+        }
+        Ok(self.imp.lock().items.pop_front().unwrap())
+    }
+    pub fn capacity(&self) -> usize {
+        self.imp.lock().cap
+    }
+    pub fn is_empty(&self) -> bool {
+        self.imp.lock().items.is_empty()
+    }
+    pub fn is_full(&self) -> bool {
+        let inner = self.imp.lock();
+        inner.items.len() == inner.cap
+    }
+    pub fn len(&self) -> usize {
+        self.imp.lock().items.len()
+    }
+}

veilid-core/src/intf/wasm/utils/mod.rs

@@ -0,0 +1,91 @@
+#![cfg(target_arch = "wasm32")]
+
+pub mod channel;
+
+use crate::xx::*;
+use core::sync::atomic::{AtomicI8, Ordering};
+use js_sys::{global, Reflect};
+
+cfg_if! {
+    if #[cfg(feature = "wee_alloc")] {
+        // When the `wee_alloc` feature is enabled, use `wee_alloc` as the global
+        // allocator.
+        extern crate wee_alloc;
+        #[global_allocator]
+        static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
+    }
+}
+
+#[wasm_bindgen]
+extern "C" {
+    // Use `js_namespace` here to bind `console.log(..)` instead of just
+    // `log(..)`
+    #[wasm_bindgen(js_namespace = console, js_name = log)]
+    pub fn console_log(s: &str);
+
+    #[wasm_bindgen]
+    pub fn alert(s: &str);
+}
+
+pub fn is_nodejs() -> bool {
+    static CACHE: AtomicI8 = AtomicI8::new(-1);
+    let cache = CACHE.load(Ordering::Relaxed);
+    if cache != -1 {
+        return cache != 0;
+    }
+
+    let res = js_sys::eval("process.release.name === 'node'")
+        .map(|res| res.is_truthy())
+        .unwrap_or_default();
+
+    CACHE.store(res as i8, Ordering::Relaxed);
+    res
+}
+
+pub fn is_browser() -> bool {
+    static CACHE: AtomicI8 = AtomicI8::new(-1);
+    let cache = CACHE.load(Ordering::Relaxed);
+    if cache != -1 {
+        return cache != 0;
+    }
+
+    let res = Reflect::has(&global().as_ref(), &"window".into()).unwrap_or_default();
+
+    CACHE.store(res as i8, Ordering::Relaxed);
+
+    res
+}
+
+pub fn is_browser_https() -> bool {
+    static CACHE: AtomicI8 = AtomicI8::new(-1);
+    let cache = CACHE.load(Ordering::Relaxed);
+    if cache != -1 {
+        return cache != 0;
+    }
+
+    let res = js_sys::eval("window.location.protocol === 'https'")
+        .map(|res| res.is_truthy())
+        .unwrap_or_default();
+
+    CACHE.store(res as i8, Ordering::Relaxed);
+
+    res
+}
+
+pub fn node_require(module: &str) -> JsValue {
+    if !is_nodejs() {
+        return JsValue::UNDEFINED;
+    }
+
+    let mut home = env!("CARGO_MANIFEST_DIR");
+    if home.len() == 0 {
+        home = ".";
+    }
+
+    match js_sys::eval(format!("require(\"{}/{}\")", home, module).as_str()) {
+        Ok(v) => v,
+        Err(e) => {
+            panic!("node_require failed: {:?}", e);
+        }
+    }
+}

veilid-core/src/lease_manager.rs

@@ -0,0 +1,188 @@
+use crate::*;
+use network_manager::*;
+use xx::*;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum LeaseKind {
+    Signal,
+    Relay,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum RelayMode {
+    Disabled,
+    Inbound,
+    Full,
+}
+
+pub struct LeaseDetails {}
+
+pub struct LeaseManagerInner {
+    network_manager: NetworkManager,
+    max_server_signal_leases: usize,
+    max_server_relay_leases: usize,
+    max_client_signal_leases: usize,
+    max_client_relay_leases: usize,
+    // server_signal_leases: BTreeMap< //xxx :how will these be accounted for?
+    client_relay_mode: RelayMode,
+}
+
+#[derive(Clone)]
+pub struct LeaseManager {
+    inner: Arc<Mutex<LeaseManagerInner>>,
+}
+
+impl LeaseManager {
+    fn new_inner(network_manager: NetworkManager) -> LeaseManagerInner {
+        LeaseManagerInner {
+            network_manager: network_manager,
+            max_server_signal_leases: 1,
+            max_server_relay_leases: 1,
+            max_client_signal_leases: 1,
+            max_client_relay_leases: 1,
+            client_relay_mode: RelayMode::Disabled,
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
+        }
+    }
+
+    pub fn network_manager(&self) -> NetworkManager {
+        self.inner.lock().network_manager.clone()
+    }
+    pub async fn startup(&self) -> Result<(), String> {
+        // Retrieve config
+        {
+            let mut inner = self.inner.lock();
+            let config = inner.network_manager.config();
+            let c = config.get();
+            inner.max_server_signal_leases = c.network.leases.max_server_signal_leases as usize;
+            inner.max_server_relay_leases = c.network.leases.max_server_relay_leases as usize;
+            inner.max_client_signal_leases = c.network.leases.max_client_signal_leases as usize;
+            inner.max_client_relay_leases = c.network.leases.max_client_relay_leases as usize;
+        }
+
+        Ok(())
+    }
+    pub async fn tick(&self) -> Result<(), String> {
+        //
+        Ok(())
+    }
+    pub async fn shutdown(&self) {
+        let network_manager = self.network_manager();
+        *self.inner.lock() = Self::new_inner(network_manager);
+    }
+
+    ////////////////////////////////
+    // Client-side
+
+    // xxx: this should automatically get set when a lease is obtained and reset when it is released or lost or expires
+    // pub fn client_set_relay_mode(&self, relay_mode: RelayMode) {
+    //     self.inner.lock().client_relay_mode = relay_mode;
+    // }
+
+    pub fn client_get_relay_mode(&self) -> RelayMode {
+        self.inner.lock().client_relay_mode
+    }
+
+    pub fn client_is_relay_peer_addr(&self, peer_addr: PeerAddress) -> bool {
+        error!("unimplemented");
+        false
+    }
+
+    pub async fn client_request_lease(&self) -> Result<(), String> {
+        Ok(())
+    }
+
+    ////////////////////////////////
+    // Server-side
+
+    // Signal leases
+    pub fn server_has_valid_signal_lease(&self, recipient_id: &DHTKey) -> bool {
+        error!("unimplemented");
+        false
+    }
+    pub fn server_can_provide_signal_lease(&self) -> bool {
+        let inner = self.inner.lock();
+        if inner.max_server_signal_leases == 0 {
+            return false;
+        }
+        let network_class = inner.network_manager.get_network_class();
+        let out = match network_class {
+            NetworkClass::Server => true,
+            NetworkClass::Mapped => true,
+            NetworkClass::FullNAT => true,
+            NetworkClass::AddressRestrictedNAT => false,
+            NetworkClass::PortRestrictedNAT => false,
+            NetworkClass::OutboundOnly => false,
+            NetworkClass::WebApp => false,
+            NetworkClass::TorWebApp => false,
+            NetworkClass::Invalid => false,
+        };
+        return out;
+    }
+    pub fn server_will_provide_signal_lease(&self) -> bool {
+        if !self.server_can_provide_signal_lease() {
+            return false;
+        }
+        let inner = self.inner.lock();
+        if inner.max_server_signal_leases == 0 {
+            return false;
+        }
+        // xxx: check total number of signal leases active...
+        // xxx: depends on who is asking?
+        // signaling requires inbound ability, so check to see if we have public dial info
+        let routing_table = inner.network_manager.routing_table();
+        if !routing_table.has_public_dial_info() {
+            return false;
+        }
+
+        return true;
+    }
+
+    // Relay leases
+    pub fn server_has_valid_relay_lease(&self, recipient_id: &DHTKey) -> bool {
+        error!("unimplemented");
+        false
+    }
+    pub fn server_can_provide_relay_lease(&self) -> bool {
+        let inner = self.inner.lock();
+        if inner.max_server_signal_leases == 0 {
+            return false;
+        }
+        let network_class = inner.network_manager.get_network_class();
+        let out = match network_class {
+            NetworkClass::Server => true,
+            NetworkClass::Mapped => true,
+            NetworkClass::FullNAT => true,
+            NetworkClass::AddressRestrictedNAT => false,
+            NetworkClass::PortRestrictedNAT => false,
+            NetworkClass::OutboundOnly => false,
+            NetworkClass::WebApp => false,
+            NetworkClass::TorWebApp => false,
+            NetworkClass::Invalid => false,
+        };
+        // xxx: also depends on network strength / bandwidth availability?
+        return out;
+    }
+    pub fn server_will_provide_relay_lease(&self) -> bool {
+        if !self.server_can_provide_relay_lease() {
+            return false;
+        }
+        let inner = self.inner.lock();
+        if inner.max_server_relay_leases == 0 {
+            return false;
+        }
+        // xxx: check total number of signal leases active...
+        // xxx: depends on who is asking?
+        // relaying requires inbound ability, so check to see if we have public dial info
+        let routing_table = inner.network_manager.routing_table();
+        if !routing_table.has_public_dial_info() {
+            return false;
+        }
+        return true;
+    }
+}

veilid-core/src/lib.rs

@@ -0,0 +1,33 @@
+#![cfg_attr(target_arch = "wasm32", no_std)]
+
+#[macro_use]
+extern crate alloc;
+
+mod attachment_manager;
+mod callback_state_machine;
+mod connection_table;
+mod dht;
+mod intf;
+mod lease_manager;
+mod network_manager;
+mod receipt_manager;
+mod routing_table;
+mod rpc_processor;
+mod veilid_api;
+mod veilid_config;
+mod veilid_core;
+mod veilid_rng;
+
+#[macro_use]
+pub mod xx;
+
+pub use self::attachment_manager::AttachmentState;
+pub use self::veilid_api::*;
+pub use self::veilid_config::*;
+pub use self::veilid_core::{VeilidCore, VeilidCoreSetup, VeilidState, VeilidStateChange};
+
+pub mod veilid_capnp {
+    include!(concat!(env!("OUT_DIR"), "/proto/veilid_capnp.rs"));
+}
+
+pub mod tests;

veilid-core/src/network_manager.rs

@@ -0,0 +1,650 @@
+use crate::*;
+use connection_table::*;
+use dht::*;
+use futures_util::future::{select, Either};
+use futures_util::stream::{FuturesUnordered, StreamExt};
+use intf::*;
+use lease_manager::*;
+use receipt_manager::*;
+use routing_table::*;
+use rpc_processor::RPCProcessor;
+use xx::*;
+
+////////////////////////////////////////////////////////////////////////////////////////
+
+const BANDWIDTH_TABLE_SIZE: usize = 10usize;
+const CONNECTION_PROCESSOR_CHANNEL_SIZE: usize = 128usize;
+
+pub const MAX_MESSAGE_SIZE: usize = MAX_ENVELOPE_SIZE;
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub enum NetworkClass {
+    Server = 0,               // S = Device with public IP and no UDP firewall
+    Mapped = 1,               // M = Device with portmap behind any NAT
+    FullNAT = 2,              // F = Device without portmap behind full-cone NAT
+    AddressRestrictedNAT = 3, // R1 = Device without portmap behind address-only restricted NAT
+    PortRestrictedNAT = 4,    // R2 = Device without portmap behind address-and-port restricted NAT
+    OutboundOnly = 5,         // O = Outbound only
+    WebApp = 6,               // W = PWA in normal web browser
+    TorWebApp = 7,            // T = PWA in Tor browser
+    Invalid = 8,              // I = Invalid network class, unreachable or can not send packets
+}
+
+impl NetworkClass {
+    pub fn inbound_capable(&self) -> bool {
+        match self {
+            Self::Server
+            | Self::Mapped
+            | Self::FullNAT
+            | Self::AddressRestrictedNAT
+            | Self::PortRestrictedNAT => true,
+            _ => false,
+        }
+    }
+}
+
+// Things we get when we start up and go away when we shut down
+// Routing table is not in here because we want it to survive a network shutdown/startup restart
+#[derive(Clone)]
+struct NetworkComponents {
+    net: Network,
+    connection_table: ConnectionTable,
+    rpc_processor: RPCProcessor,
+    lease_manager: LeaseManager,
+    receipt_manager: ReceiptManager,
+}
+
+// The mutable state of the network manager
+pub struct NetworkManagerInner {
+    routing_table: Option<RoutingTable>,
+    components: Option<NetworkComponents>,
+    network_class: Option<NetworkClass>,
+    incoming_avg_bandwidth: f32,
+    incoming_max_bandwidth: f32,
+    incoming_bandwidth_table: Vec<f32>,
+    outgoing_avg_bandwidth: f32,
+    outgoing_max_bandwidth: f32,
+    outgoing_bandwidth_table: Vec<f32>,
+    connection_processor_jh: Option<JoinHandle<()>>,
+    connection_add_channel_tx: Option<utils::channel::Sender<SystemPinBoxFuture<()>>>,
+}
+
+#[derive(Clone)]
+pub struct NetworkManager {
+    config: VeilidConfig,
+    table_store: TableStore,
+    crypto: Crypto,
+    inner: Arc<Mutex<NetworkManagerInner>>,
+}
+
+impl NetworkManager {
+    fn new_inner() -> NetworkManagerInner {
+        let mut inner = NetworkManagerInner {
+            routing_table: None,
+            components: None,
+            network_class: None,
+            incoming_avg_bandwidth: 0.0f32,
+            incoming_max_bandwidth: 0.0f32,
+            incoming_bandwidth_table: Vec::new(),
+            outgoing_avg_bandwidth: 0.0f32,
+            outgoing_max_bandwidth: 0.0f32,
+            outgoing_bandwidth_table: Vec::new(),
+            connection_processor_jh: None,
+            connection_add_channel_tx: None,
+        };
+        inner
+            .incoming_bandwidth_table
+            .resize(BANDWIDTH_TABLE_SIZE, 0.0f32);
+        inner
+            .outgoing_bandwidth_table
+            .resize(BANDWIDTH_TABLE_SIZE, 0.0f32);
+        inner
+    }
+
+    pub fn new(config: VeilidConfig, table_store: TableStore, crypto: Crypto) -> Self {
+        Self {
+            config: config.clone(),
+            table_store: table_store,
+            crypto: crypto,
+            inner: Arc::new(Mutex::new(Self::new_inner())),
+        }
+    }
+    pub fn config(&self) -> VeilidConfig {
+        self.config.clone()
+    }
+    pub fn table_store(&self) -> TableStore {
+        self.table_store.clone()
+    }
+    pub fn crypto(&self) -> Crypto {
+        self.crypto.clone()
+    }
+    pub fn routing_table(&self) -> RoutingTable {
+        self.inner.lock().routing_table.as_ref().unwrap().clone()
+    }
+    pub fn net(&self) -> Network {
+        self.inner.lock().components.as_ref().unwrap().net.clone()
+    }
+    pub fn rpc_processor(&self) -> RPCProcessor {
+        self.inner
+            .lock()
+            .components
+            .as_ref()
+            .unwrap()
+            .rpc_processor
+            .clone()
+    }
+    pub fn lease_manager(&self) -> LeaseManager {
+        self.inner
+            .lock()
+            .components
+            .as_ref()
+            .unwrap()
+            .lease_manager
+            .clone()
+    }
+    pub fn receipt_manager(&self) -> ReceiptManager {
+        self.inner
+            .lock()
+            .components
+            .as_ref()
+            .unwrap()
+            .receipt_manager
+            .clone()
+    }
+    pub fn connection_table(&self) -> ConnectionTable {
+        self.inner
+            .lock()
+            .components
+            .as_ref()
+            .unwrap()
+            .connection_table
+            .clone()
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        let routing_table = RoutingTable::new(self.clone());
+        routing_table.init().await?;
+        self.inner.lock().routing_table = Some(routing_table.clone());
+        Ok(())
+    }
+    pub async fn terminate(&self) {
+        let mut inner = self.inner.lock();
+        if let Some(routing_table) = &inner.routing_table {
+            routing_table.terminate().await;
+            inner.routing_table = None;
+        }
+    }
+
+    pub async fn internal_startup(&self) -> Result<(), String> {
+        trace!("NetworkManager::internal_startup begin");
+        if self.inner.lock().components.is_some() {
+            debug!("NetworkManager::internal_startup already started");
+            return Ok(());
+        }
+
+        // Create network components
+        let net = Network::new(self.clone());
+        let connection_table = ConnectionTable::new();
+        let rpc_processor = RPCProcessor::new(self.clone());
+        let lease_manager = LeaseManager::new(self.clone());
+        let receipt_manager = ReceiptManager::new(self.clone());
+        self.inner.lock().components = Some(NetworkComponents {
+            net: net.clone(),
+            connection_table: connection_table.clone(),
+            rpc_processor: rpc_processor.clone(),
+            lease_manager: lease_manager.clone(),
+            receipt_manager: receipt_manager.clone(),
+        });
+
+        // Start network components
+        rpc_processor.startup().await?;
+        lease_manager.startup().await?;
+        receipt_manager.startup().await?;
+        net.startup().await?;
+
+        // Run connection processing task
+        let cac = utils::channel::channel(CONNECTION_PROCESSOR_CHANNEL_SIZE); // xxx move to config
+        self.inner.lock().connection_add_channel_tx = Some(cac.0);
+        let rx = cac.1.clone();
+        let this = self.clone();
+        self.inner.lock().connection_processor_jh = Some(spawn(this.connection_processor(rx)));
+
+        trace!("NetworkManager::internal_startup end");
+
+        Ok(())
+    }
+
+    pub async fn startup(&self) -> Result<(), String> {
+        if let Err(e) = self.internal_startup().await {
+            self.shutdown().await;
+            return Err(e);
+        }
+        Ok(())
+    }
+
+    pub async fn shutdown(&self) {
+        trace!("NetworkManager::shutdown begin");
+        let components = {
+            let mut inner = self.inner.lock();
+            // Drop/cancel the connection processing task first
+            inner.connection_processor_jh = None;
+            inner.connection_add_channel_tx = None;
+            inner.components.clone()
+        };
+
+        // Shutdown network components if they started up
+        if let Some(components) = components {
+            components.net.shutdown().await;
+            components.receipt_manager.shutdown().await;
+            components.lease_manager.shutdown().await;
+            components.rpc_processor.shutdown().await;
+        }
+
+        // reset the state
+        *self.inner.lock() = Self::new_inner();
+
+        trace!("NetworkManager::shutdown end");
+    }
+
+    pub async fn tick(&self) -> Result<(), String> {
+        let (net, lease_manager, receipt_manager) = {
+            let inner = self.inner.lock();
+            let components = inner.components.as_ref().unwrap();
+            (
+                components.net.clone(),
+                components.lease_manager.clone(),
+                components.receipt_manager.clone(),
+            )
+        };
+
+        // If the network needs to be reset, do it
+        // if things can't restart, then we fail out of the attachment manager
+        if net.needs_restart() {
+            net.shutdown().await;
+            net.startup().await?;
+        }
+
+        // Run the low level network tick
+        net.tick().await?;
+
+        // Run the lease manager tick
+        lease_manager.tick().await?;
+
+        // Run the receipt manager tick
+        receipt_manager.tick().await?;
+
+        Ok(())
+    }
+
+    // Called by low-level protocol handlers when any connection-oriented protocol connection appears
+    // either from incoming or outgoing connections. Registers connection in the connection table for later access
+    // and spawns a message processing loop for the connection
+    pub async fn on_new_connection(
+        &self,
+        descriptor: ConnectionDescriptor,
+        conn: NetworkConnection,
+    ) -> Result<(), ()> {
+        let tx = self
+            .inner
+            .lock()
+            .connection_add_channel_tx
+            .as_ref()
+            .ok_or(())?
+            .clone();
+        let this = self.clone();
+        let receiver_loop_future = Self::process_connection(this, descriptor, conn);
+        Ok(tx.try_send(receiver_loop_future).await.map_err(drop)?)
+    }
+
+    // Connection receiver loop
+    fn process_connection(
+        this: NetworkManager,
+        descriptor: ConnectionDescriptor,
+        conn: NetworkConnection,
+    ) -> SystemPinBoxFuture<()> {
+        Box::pin(async move {
+            // Add new connections to the table
+            let entry = match this
+                .connection_table()
+                .add_connection(descriptor.clone(), conn.clone())
+            {
+                Ok(e) => e,
+                Err(_) => return,
+            };
+
+            //
+            let exit_value: Result<Vec<u8>, ()> = Err(());
+
+            loop {
+                let res = match select(
+                    entry.stopper.clone().instance_clone(exit_value.clone()),
+                    conn.clone().recv(),
+                )
+                .await
+                {
+                    Either::Left((_x, _b)) => break,
+                    Either::Right((y, _a)) => y,
+                };
+                let message = match res {
+                    Ok(v) => v,
+                    Err(_) => break,
+                };
+                match this.on_recv_envelope(message.as_slice(), &descriptor).await {
+                    Ok(_) => (),
+                    Err(_) => break,
+                };
+            }
+
+            let _ = this.connection_table().remove_connection(&descriptor);
+        })
+    }
+
+    // Process connection oriented sockets in the background
+    // This never terminates and must have its task cancelled once started
+    async fn connection_processor(self, rx: utils::channel::Receiver<SystemPinBoxFuture<()>>) {
+        let mut connection_futures: FuturesUnordered<SystemPinBoxFuture<()>> =
+            FuturesUnordered::new();
+        loop {
+            // Either process an existing connection, or receive a new one to add to our list
+            match select(connection_futures.next(), Box::pin(rx.recv())).await {
+                Either::Left((x, _)) => {
+                    // Processed some connection to completion, or there are none left
+                    match x {
+                        Some(()) => {
+                            // Processed some connection to completion
+                        }
+                        None => {
+                            // No connections to process, wait for one
+                            match rx.recv().await {
+                                Ok(v) => {
+                                    connection_futures.push(v);
+                                }
+                                Err(e) => {
+                                    error!("connection processor error: {:?}", e);
+                                    // xxx: do something here??
+                                }
+                            };
+                        }
+                    }
+                }
+                Either::Right((x, _)) => {
+                    // Got a new connection future
+                    match x {
+                        Ok(v) => {
+                            connection_futures.push(v);
+                        }
+                        Err(e) => {
+                            error!("connection processor error: {:?}", e);
+                            // xxx: do something here??
+                        }
+                    };
+                }
+            }
+        }
+    }
+
+    // Return what network class we are in
+    pub fn get_network_class(&self) -> NetworkClass {
+        if let Some(components) = &self.inner.lock().components {
+            components.net.get_network_class()
+        } else {
+            NetworkClass::Invalid
+        }
+    }
+
+    // Generates an out-of-band receipt
+    pub fn generate_receipt<D: AsRef<[u8]>>(
+        &self,
+        expiration_us: u64,
+        expected_returns: u32,
+        extra_data: D,
+        callback: impl ReceiptCallback,
+    ) -> Result<Vec<u8>, String> {
+        let receipt_manager = self.receipt_manager();
+        let routing_table = self.routing_table();
+
+        // Generate receipt and serialized form to return
+        let nonce = Crypto::get_random_nonce();
+        let receipt = Receipt::try_new(0, nonce, routing_table.node_id(), extra_data)?;
+        let out = receipt
+            .to_signed_data(&routing_table.node_id_secret())
+            .map_err(|_| "failed to generate signed receipt".to_owned())?;
+
+        // Record the receipt for later
+        let exp_ts = intf::get_timestamp() + expiration_us;
+        receipt_manager.record_receipt(receipt, exp_ts, expected_returns, callback);
+
+        Ok(out)
+    }
+
+    pub fn generate_single_shot_receipt<D: AsRef<[u8]>>(
+        &self,
+        expiration_us: u64,
+        extra_data: D,
+    ) -> Result<(Vec<u8>, EventualValueCloneFuture<ReceiptEvent>), String> {
+        let receipt_manager = self.receipt_manager();
+        let routing_table = self.routing_table();
+
+        // Generate receipt and serialized form to return
+        let nonce = Crypto::get_random_nonce();
+        let receipt = Receipt::try_new(0, nonce, routing_table.node_id(), extra_data)?;
+        let out = receipt
+            .to_signed_data(&routing_table.node_id_secret())
+            .map_err(|_| "failed to generate signed receipt".to_owned())?;
+
+        // Record the receipt for later
+        let exp_ts = intf::get_timestamp() + expiration_us;
+        let eventual = SingleShotEventual::new(ReceiptEvent::CANCELLED);
+        let instance = eventual.instance();
+        receipt_manager.record_single_shot_receipt(receipt, exp_ts, eventual);
+
+        Ok((out, instance))
+    }
+
+    // Process a received out-of-band receipt
+    pub async fn process_receipt<R: AsRef<[u8]>>(&self, receipt_data: R) -> Result<(), String> {
+        let receipt_manager = self.receipt_manager();
+        let receipt = Receipt::from_signed_data(receipt_data.as_ref())
+            .map_err(|_| "failed to parse signed receipt".to_owned())?;
+        receipt_manager.handle_receipt(receipt).await
+    }
+
+    // Builds an envelope for sending over the network
+    fn build_envelope<B: AsRef<[u8]>>(
+        &self,
+        dest_node_id: key::DHTKey,
+        version: u8,
+        body: B,
+    ) -> Result<Vec<u8>, String> {
+        // DH to get encryption key
+        let routing_table = self.routing_table();
+        let node_id = routing_table.node_id();
+        let node_id_secret = routing_table.node_id_secret();
+
+        // Get timestamp, nonce
+        let ts = intf::get_timestamp();
+        let nonce = Crypto::get_random_nonce();
+
+        // Encode envelope
+        let envelope = Envelope::new(version, ts, nonce, node_id, dest_node_id);
+        envelope
+            .to_encrypted_data(self.crypto.clone(), body.as_ref(), &node_id_secret)
+            .map_err(|_| "envelope failed to encode".to_owned())
+    }
+
+    // Called by the RPC handler when we want to issue an RPC request or response
+    pub async fn send_envelope<B: AsRef<[u8]>>(
+        &self,
+        node_ref: NodeRef,
+        body: B,
+    ) -> Result<(), String> {
+        // Get node's min/max version and see if we can send to it
+        // and if so, get the max version we can use
+        let version = if let Some((node_min, node_max)) = node_ref.operate(|e| e.min_max_version())
+        {
+            if node_min > MAX_VERSION || node_max < MIN_VERSION {
+                return Err(format!(
+                    "can't talk to this node {} because version is unsupported: ({},{})",
+                    node_ref.node_id(),
+                    node_min,
+                    node_max
+                ));
+            }
+            cmp::min(node_max, MAX_VERSION)
+        } else {
+            MAX_VERSION
+        };
+
+        // Build the envelope to send
+        let out = self.build_envelope(node_ref.node_id(), version, body)?;
+
+        // Send via relay if we have to
+        self.net().send_data(node_ref, out).await
+    }
+
+    // Called by the RPC handler when we want to issue an direct receipt
+    pub async fn send_direct_receipt<B: AsRef<[u8]>>(
+        &self,
+        dial_info: DialInfo,
+        rcpt_data: B,
+        alternate_port: bool,
+    ) -> Result<(), String> {
+        // Validate receipt before we send it, otherwise this may be arbitrary data!
+        let _ = Receipt::from_signed_data(rcpt_data.as_ref())
+            .map_err(|_| "failed to validate direct receipt".to_owned())?;
+
+        // Send receipt directly
+        if alternate_port {
+            self.net()
+                .send_data_unbound_to_dial_info(&dial_info, rcpt_data.as_ref().to_vec())
+                .await
+        } else {
+            self.net()
+                .send_data_to_dial_info(&dial_info, rcpt_data.as_ref().to_vec())
+                .await
+        }
+    }
+
+    // Called when a packet potentially containing an RPC envelope is received by a low-level
+    // network protocol handler. Processes the envelope, authenticates and decrypts the RPC message
+    // and passes it to the RPC handler
+    pub async fn on_recv_envelope(
+        &self,
+        data: &[u8],
+        descriptor: &ConnectionDescriptor,
+    ) -> Result<bool, ()> {
+        // Is this an out-of-band receipt instead of an envelope?
+        if data[0..4] == *RECEIPT_MAGIC {
+            self.process_receipt(data).await.map_err(|s| {
+                trace!("receipt failed to process: {}", s);
+            })?;
+            return Ok(true);
+        }
+
+        // Decode envelope header
+        let envelope = Envelope::from_data(data).map_err(|_| {
+            trace!("envelope failed to decode");
+        })?;
+
+        // Get routing table and rpc processor
+        let (routing_table, lease_manager, rpc) = {
+            let inner = self.inner.lock();
+            (
+                inner.routing_table.as_ref().unwrap().clone(),
+                inner.components.as_ref().unwrap().lease_manager.clone(),
+                inner.components.as_ref().unwrap().rpc_processor.clone(),
+            )
+        };
+
+        // Peek at header and see if we need to send this to a relay lease
+        // If the recipient id is not our node id, then it needs relaying
+        let sender_id = envelope.get_sender_id();
+        let recipient_id = envelope.get_recipient_id();
+        if recipient_id != routing_table.node_id() {
+            // Ensure a lease exists for this node before we relay it
+            if !lease_manager.server_has_valid_relay_lease(&recipient_id)
+                && !lease_manager.server_has_valid_relay_lease(&sender_id)
+            {
+                trace!("received envelope not intended for this node");
+                return Err(());
+            }
+
+            // Resolve the node to send this to
+            let relay_nr = rpc.resolve_node(recipient_id).await.map_err(|_| {
+                trace!("failed to resolve recipient node for relay, dropping packet...");
+            })?;
+
+            // Re-send the packet to the leased node
+            self.net()
+                .send_data(relay_nr, data.to_vec())
+                .await
+                .map_err(|_| {
+                    trace!("failed to forward envelope");
+                })?;
+            // Inform caller that we dealt with the envelope, but did not process it locally
+            return Ok(false);
+        }
+
+        // DH to get decryption key (cached)
+        let node_id_secret = routing_table.node_id_secret();
+
+        // Decrypt the envelope body
+        // xxx: punish nodes that send messages that fail to decrypt eventually
+        let body = envelope.decrypt_body(self.crypto(), data, &node_id_secret)?;
+
+        // Get timestamp range
+        let (tsbehind, tsahead) = {
+            let c = self.config.get();
+            (
+                c.network.rpc.max_timestamp_behind,
+                c.network.rpc.max_timestamp_ahead,
+            )
+        };
+
+        // Validate timestamp isn't too old
+        let ts = intf::get_timestamp();
+        let ets = envelope.get_timestamp();
+        if let Some(tsbehind) = tsbehind {
+            if tsbehind > 0 && (ts > ets && ts - ets > tsbehind) {
+                trace!(
+                    "envelope time was too far in the past: {}ms ",
+                    timestamp_to_secs(ts - ets) * 1000f64
+                );
+                return Err(());
+            }
+        }
+        if let Some(tsahead) = tsahead {
+            if tsahead > 0 && (ts < ets && ets - ts > tsahead) {
+                trace!(
+                    "envelope time was too far in the future: {}ms",
+                    timestamp_to_secs(ets - ts) * 1000f64
+                );
+                return Err(());
+            }
+        }
+
+        // Cache the envelope information in the routing table
+        let source_noderef = routing_table
+            .register_node_with_existing_connection(
+                envelope.get_sender_id(),
+                descriptor.clone(),
+                ts,
+            )
+            .map_err(|_| {
+                trace!("node id registration failed");
+            })?;
+        source_noderef.operate(|e| e.set_min_max_version(envelope.get_min_max_version()));
+
+        // xxx: deal with spoofing and flooding here?
+
+        // Pass message to RPC system
+        rpc.enqueue_message(envelope, body, source_noderef)
+            .await
+            .map_err(|_| {
+                trace!("enqueing rpc message failed");
+            })?;
+
+        // Inform caller that we dealt with the envelope locally
+        Ok(true)
+    }
+}

veilid-core/src/receipt_manager.rs

@@ -0,0 +1,418 @@
+use crate::*;
+use core::fmt;
+use dht::receipt::*;
+use futures_util::stream::{FuturesUnordered, StreamExt};
+use network_manager::*;
+use xx::*;
+
+#[derive(Clone, Debug, Copy, PartialEq, Eq)]
+pub enum ReceiptEvent {
+    RETURNED,
+    EXPIRED,
+    CANCELLED,
+}
+
+cfg_if! {
+    if #[cfg(target_arch = "wasm32")] {
+        pub trait ReceiptCallback: 'static {
+            fn call(
+                &self,
+                event: ReceiptEvent,
+                nonce: ReceiptNonce,
+                returns_so_far: u32,
+                expected_returns: u32,
+            ) -> SystemPinBoxFuture<()>;
+        }
+        impl<T, F> ReceiptCallback for T
+        where
+            T: Fn(ReceiptEvent, ReceiptNonce, u32, u32) -> F + 'static,
+            F: Future<Output = ()> + 'static,
+        {
+            fn call(
+                &self,
+                event: ReceiptEvent,
+                nonce: ReceiptNonce,
+                returns_so_far: u32,
+                expected_returns: u32,
+            ) -> SystemPinBoxFuture<()> {
+                Box::pin(self(event, nonce, returns_so_far, expected_returns))
+            }
+        }
+    } else {
+        pub trait ReceiptCallback: Send + 'static {
+            fn call(
+                &self,
+                event: ReceiptEvent,
+                nonce: ReceiptNonce,
+                returns_so_far: u32,
+                expected_returns: u32,
+            ) -> SystemPinBoxFuture<()>;
+        }
+        impl<F, T> ReceiptCallback for T
+        where
+            T: Fn(ReceiptEvent, ReceiptNonce, u32, u32) -> F + Send + 'static,
+            F: Future<Output = ()> + Send + 'static
+        {
+            fn call(
+                &self,
+                event: ReceiptEvent,
+                nonce: ReceiptNonce,
+                returns_so_far: u32,
+                expected_returns: u32,
+            ) -> SystemPinBoxFuture<()> {
+                Box::pin(self(event, nonce, returns_so_far, expected_returns))
+            }
+        }
+    }
+}
+
+type ReceiptCallbackType = Box<dyn ReceiptCallback>;
+type ReceiptSingleShotType = SingleShotEventual<ReceiptEvent>;
+
+enum ReceiptRecordCallbackType {
+    Normal(ReceiptCallbackType),
+    SingleShot(Option<ReceiptSingleShotType>),
+}
+impl fmt::Debug for ReceiptRecordCallbackType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "ReceiptRecordCallbackType::{}",
+            match self {
+                Self::Normal(_) => "Normal".to_owned(),
+                Self::SingleShot(_) => "SingleShot".to_owned(),
+            }
+        )
+    }
+}
+
+pub struct ReceiptRecord {
+    expiration_ts: u64,
+    nonce: ReceiptNonce,
+    expected_returns: u32,
+    returns_so_far: u32,
+    receipt_callback: ReceiptRecordCallbackType,
+}
+
+impl fmt::Debug for ReceiptRecord {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ReceiptRecord")
+            .field("expiration_ts", &self.expiration_ts)
+            .field("nonce", &self.nonce)
+            .field("expected_returns", &self.expected_returns)
+            .field("returns_so_far", &self.returns_so_far)
+            .field("receipt_callback", &self.receipt_callback)
+            .finish()
+    }
+}
+
+impl ReceiptRecord {
+    pub fn from_receipt(
+        receipt: &Receipt,
+        expiration_ts: u64,
+        expected_returns: u32,
+        receipt_callback: impl ReceiptCallback,
+    ) -> Self {
+        Self {
+            expiration_ts: expiration_ts,
+            nonce: receipt.get_nonce(),
+            expected_returns: expected_returns,
+            returns_so_far: 0u32,
+            receipt_callback: ReceiptRecordCallbackType::Normal(Box::new(receipt_callback)),
+        }
+    }
+
+    pub fn from_single_shot_receipt(
+        receipt: &Receipt,
+        expiration_ts: u64,
+        eventual: ReceiptSingleShotType,
+    ) -> Self {
+        Self {
+            expiration_ts: expiration_ts,
+            nonce: receipt.get_nonce(),
+            returns_so_far: 0u32,
+            expected_returns: 1u32,
+            receipt_callback: ReceiptRecordCallbackType::SingleShot(Some(eventual)),
+        }
+    }
+}
+
+/* XXX: may be useful for O(1) timestamp expiration
+#[derive(Clone, Debug)]
+struct ReceiptRecordTimestampSort {
+    expiration_ts: u64,
+    record: Arc<Mutex<ReceiptRecord>>,
+}
+
+impl PartialEq for ReceiptRecordTimestampSort {
+    fn eq(&self, other: &ReceiptRecordTimestampSort) -> bool {
+        self.expiration_ts == other.expiration_ts
+    }
+}
+impl Eq for ReceiptRecordTimestampSort {}
+impl Ord for ReceiptRecordTimestampSort {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.expiration_ts.cmp(&other.expiration_ts).reverse()
+    }
+}
+impl PartialOrd for ReceiptRecordTimestampSort {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(&other))
+    }
+}
+*/
+
+///////////////////////////////////
+
+pub struct ReceiptManagerInner {
+    network_manager: NetworkManager,
+    receipts_by_nonce: BTreeMap<ReceiptNonce, Arc<Mutex<ReceiptRecord>>>,
+    next_oldest_ts: Option<u64>,
+    timeout_task: SingleFuture<()>,
+}
+
+#[derive(Clone)]
+pub struct ReceiptManager {
+    inner: Arc<Mutex<ReceiptManagerInner>>,
+}
+
+impl ReceiptManager {
+    fn new_inner(network_manager: NetworkManager) -> ReceiptManagerInner {
+        ReceiptManagerInner {
+            network_manager: network_manager,
+            receipts_by_nonce: BTreeMap::new(),
+            next_oldest_ts: None,
+            timeout_task: SingleFuture::new(),
+        }
+    }
+
+    pub fn new(network_manager: NetworkManager) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
+        }
+    }
+
+    pub fn network_manager(&self) -> NetworkManager {
+        self.inner.lock().network_manager.clone()
+    }
+
+    pub async fn startup(&self) -> Result<(), String> {
+        // Retrieve config
+        /*
+                {
+                    let config = self.core().config();
+                    let c = config.get();
+                    let mut inner = self.inner.lock();
+                    inner.max_server_signal_leases = c.network.leases.max_server_signal_leases as usize;
+                    inner.max_server_relay_leases = c.network.leases.max_server_relay_leases as usize;
+                    inner.max_client_signal_leases = c.network.leases.max_client_signal_leases as usize;
+                    inner.max_client_relay_leases = c.network.leases.max_client_relay_leases as usize;
+                }
+        */
+        Ok(())
+    }
+
+    fn perform_callback(
+        evt: ReceiptEvent,
+        record_mut: &mut ReceiptRecord,
+    ) -> Option<SystemPinBoxFuture<()>> {
+        match &mut record_mut.receipt_callback {
+            ReceiptRecordCallbackType::Normal(callback) => Some(callback.call(
+                evt,
+                record_mut.nonce,
+                record_mut.returns_so_far,
+                record_mut.expected_returns,
+            )),
+            ReceiptRecordCallbackType::SingleShot(eventual) => {
+                // resolve this eventual with the receiptevent
+                // don't need to wait for the instance to receive it
+                // because this can only happen once
+                if let Some(eventual) = eventual.take() {
+                    eventual.resolve(evt);
+                }
+                None
+            }
+        }
+    }
+
+    pub async fn timeout_task_routine(self, now: u64) {
+        // Go through all receipts and build a list of expired nonces
+        let mut new_next_oldest_ts: Option<u64> = None;
+        let mut expired_records = Vec::new();
+        {
+            let mut inner = self.inner.lock();
+            let mut expired_nonces = Vec::new();
+            for (k, v) in &inner.receipts_by_nonce {
+                let receipt_inner = v.lock();
+                if receipt_inner.expiration_ts <= now {
+                    // Expire this receipt
+                    expired_nonces.push(k.clone());
+                } else if new_next_oldest_ts.is_none()
+                    || receipt_inner.expiration_ts < new_next_oldest_ts.unwrap()
+                {
+                    // Mark the next oldest timestamp we would need to take action on as we go through everything
+                    new_next_oldest_ts = Some(receipt_inner.expiration_ts);
+                }
+            }
+            if expired_nonces.len() == 0 {
+                return;
+            }
+            // Now remove the expired receipts
+            for e in expired_nonces {
+                let expired_record = inner
+                    .receipts_by_nonce
+                    .remove(&e)
+                    .expect("key should exist");
+                expired_records.push(expired_record);
+            }
+            // Update the next oldest timestamp
+            inner.next_oldest_ts = new_next_oldest_ts;
+        }
+        let mut callbacks = FuturesUnordered::new();
+        for expired_record in expired_records {
+            let mut expired_record_mut = expired_record.lock();
+            if let Some(callback) =
+                Self::perform_callback(ReceiptEvent::EXPIRED, &mut expired_record_mut)
+            {
+                callbacks.push(callback)
+            }
+        }
+
+        // Wait on all the multi-call callbacks
+        while callbacks.next().await.is_some() {}
+    }
+
+    pub async fn tick(&self) -> Result<(), String> {
+        let (next_oldest_ts, timeout_task) = {
+            let inner = self.inner.lock();
+            (inner.next_oldest_ts, inner.timeout_task.clone())
+        };
+        let now = intf::get_timestamp();
+        // If we have at least one timestamp to expire, lets do it
+        if let Some(next_oldest_ts) = next_oldest_ts {
+            if now >= next_oldest_ts {
+                // Single-spawn the timeout task routine
+                let _ = timeout_task
+                    .single_spawn(self.clone().timeout_task_routine(now))
+                    .await;
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn shutdown(&self) {
+        let network_manager = self.network_manager();
+        *self.inner.lock() = Self::new_inner(network_manager);
+    }
+
+    pub fn record_receipt(
+        &self,
+        receipt: Receipt,
+        expiration: u64,
+        expected_returns: u32,
+        callback: impl ReceiptCallback,
+    ) {
+        let record = Arc::new(Mutex::new(ReceiptRecord::from_receipt(
+            &receipt,
+            expiration,
+            expected_returns,
+            callback,
+        )));
+        let mut inner = self.inner.lock();
+        inner
+            .receipts_by_nonce
+            .insert(receipt.get_nonce(), record.clone());
+    }
+
+    pub fn record_single_shot_receipt(
+        &self,
+        receipt: Receipt,
+        expiration: u64,
+        eventual: ReceiptSingleShotType,
+    ) {
+        let record = Arc::new(Mutex::new(ReceiptRecord::from_single_shot_receipt(
+            &receipt, expiration, eventual,
+        )));
+        let mut inner = self.inner.lock();
+        inner
+            .receipts_by_nonce
+            .insert(receipt.get_nonce(), record.clone());
+    }
+
+    fn update_next_oldest_timestamp(inner: &mut ReceiptManagerInner) {
+        // Update the next oldest timestamp
+        let mut new_next_oldest_ts: Option<u64> = None;
+        for (_, v) in &inner.receipts_by_nonce {
+            let receipt_inner = v.lock();
+            if new_next_oldest_ts.is_none()
+                || receipt_inner.expiration_ts < new_next_oldest_ts.unwrap()
+            {
+                // Mark the next oldest timestamp we would need to take action on as we go through everything
+                new_next_oldest_ts = Some(receipt_inner.expiration_ts);
+            }
+        }
+
+        inner.next_oldest_ts = new_next_oldest_ts;
+    }
+
+    pub async fn cancel_receipt(&self, nonce: &ReceiptNonce) -> Result<(), String> {
+        // Remove the record
+        let record = {
+            let mut inner = self.inner.lock();
+            let record = match inner.receipts_by_nonce.remove(nonce) {
+                Some(r) => r,
+                None => {
+                    return Err("receipt not recorded".to_owned());
+                }
+            };
+            Self::update_next_oldest_timestamp(&mut *inner);
+            record
+        };
+
+        // Generate a cancelled callback
+        let callback_future = {
+            let mut record_mut = record.lock();
+            Self::perform_callback(ReceiptEvent::CANCELLED, &mut record_mut)
+        };
+
+        // Issue the callback
+        if let Some(callback_future) = callback_future {
+            callback_future.await;
+        }
+
+        Ok(())
+    }
+
+    pub async fn handle_receipt(&self, receipt: Receipt) -> Result<(), String> {
+        // Increment return count
+        let callback_future = {
+            // Look up the receipt record from the nonce
+            let mut inner = self.inner.lock();
+            let record = match inner.receipts_by_nonce.get(&receipt.get_nonce()) {
+                Some(r) => r.clone(),
+                None => {
+                    return Err("receipt not recorded".to_owned());
+                }
+            };
+            // Generate the callback future
+            let mut record_mut = record.lock();
+            record_mut.returns_so_far += 1;
+            let callback_future = Self::perform_callback(ReceiptEvent::RETURNED, &mut record_mut);
+
+            // Remove the record if we're done
+            if record_mut.returns_so_far == record_mut.expected_returns {
+                inner.receipts_by_nonce.remove(&receipt.get_nonce());
+
+                Self::update_next_oldest_timestamp(&mut *inner);
+            }
+            callback_future
+        };
+
+        // Issue the callback
+        if let Some(callback_future) = callback_future {
+            callback_future.await;
+        }
+
+        Ok(())
+    }
+}

veilid-core/src/routing_table/bucket.rs

@@ -0,0 +1,142 @@
+use super::*;
+
+#[derive(Clone)]
+pub struct Bucket {
+    routing_table: RoutingTable,
+    entries: BTreeMap<DHTKey, BucketEntry>,
+    newest_entry: Option<DHTKey>,
+}
+pub(super) type EntriesIterMut<'a> =
+    alloc::collections::btree_map::IterMut<'a, DHTKey, BucketEntry>;
+pub(super) type EntriesIter<'a> = alloc::collections::btree_map::Iter<'a, DHTKey, BucketEntry>;
+
+fn state_ordering(state: BucketEntryState) -> usize {
+    match state {
+        BucketEntryState::Dead => 0,
+        BucketEntryState::Unreliable => 1,
+        BucketEntryState::Reliable => 2,
+    }
+}
+
+impl Bucket {
+    pub fn new(routing_table: RoutingTable) -> Self {
+        Self {
+            routing_table: routing_table,
+            entries: BTreeMap::new(),
+            newest_entry: None,
+        }
+    }
+
+    pub(super) fn add_entry(&mut self, node_id: DHTKey) -> NodeRef {
+        info!("Node added: {}", node_id.encode());
+
+        // Add new entry
+        self.entries.insert(node_id, BucketEntry::new());
+
+        // This is now the newest bucket entry
+        self.newest_entry = Some(node_id);
+
+        // Get a node ref to return
+        let entry_ref = self.entries.get_mut(&node_id).unwrap();
+        NodeRef::new(self.routing_table.clone(), node_id, entry_ref)
+    }
+
+    pub(super) fn remove_entry(&mut self, node_id: &DHTKey) {
+        info!("Node removed: {}", node_id);
+
+        // Remove the entry
+        self.entries.remove(node_id);
+
+        // newest_entry is updated by kick_bucket()
+    }
+
+    pub(super) fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) {
+        // Called every ROLLING_TRANSFERS_INTERVAL_SECS
+        for entry in &mut self.entries {
+            entry.1.roll_transfers(last_ts, cur_ts);
+        }
+    }
+
+    pub(super) fn entry_mut(&mut self, key: &DHTKey) -> Option<&mut BucketEntry> {
+        self.entries.get_mut(key)
+    }
+
+    pub(super) fn entries(&self) -> EntriesIter {
+        self.entries.iter()
+    }
+
+    pub(super) fn entries_mut(&mut self) -> EntriesIterMut {
+        self.entries.iter_mut()
+    }
+
+    pub(super) fn kick(&mut self, bucket_depth: usize) -> Option<BTreeSet<DHTKey>> {
+        // Get number of entries to attempt to purge from bucket
+        let bucket_len = self.entries.len();
+        if bucket_len <= bucket_depth {
+            return None;
+        }
+        // Try to purge the newest entries that overflow the bucket
+        let mut dead_node_ids: BTreeSet<DHTKey> = BTreeSet::new();
+        let mut extra_entries = bucket_len - bucket_depth;
+
+        // Get the sorted list of entries by their kick order
+        let mut sorted_entries: Vec<(&_, &_)> = self.entries.iter().collect();
+        let cur_ts = get_timestamp();
+        sorted_entries.sort_by(
+            |a: &(&DHTKey, &BucketEntry), b: &(&DHTKey, &BucketEntry)| -> core::cmp::Ordering {
+                let ea = a.1;
+                let eb = b.1;
+                let astate = state_ordering(ea.state(cur_ts));
+                let bstate = state_ordering(eb.state(cur_ts));
+                // first kick dead nodes, then unreliable nodes
+                if astate < bstate {
+                    return core::cmp::Ordering::Less;
+                }
+                if astate > bstate {
+                    return core::cmp::Ordering::Greater;
+                }
+                // then kick by time added, most recent nodes are kicked first
+                let ata = ea.peer_stats().time_added;
+                let bta = eb.peer_stats().time_added;
+                bta.cmp(&ata)
+            },
+        );
+
+        self.newest_entry = None;
+        for i in 0..sorted_entries.len() {
+            // If we're not evicting more entries, exit, noting this may be the newest entry
+            if extra_entries == 0 {
+                // The first 'live' entry we find is our newest entry
+                if self.newest_entry.is_none() {
+                    self.newest_entry = Some(sorted_entries[i].0.clone());
+                }
+                break;
+            }
+            extra_entries -= 1;
+
+            // if this entry has references we can't drop it yet
+            if sorted_entries[i].1.ref_count > 0 {
+                // The first 'live' entry we fine is our newest entry
+                if self.newest_entry.is_none() {
+                    self.newest_entry = Some(sorted_entries[i].0.clone());
+                }
+                continue;
+            }
+
+            // if no references, lets evict it
+            dead_node_ids.insert(sorted_entries[i].0.clone());
+        }
+
+        // Now purge the dead node ids
+        for id in &dead_node_ids {
+            // Remove the entry
+            self.remove_entry(id);
+        }
+
+        if dead_node_ids.len() > 0 {
+            Some(dead_node_ids)
+        } else {
+            None
+        }
+    }
+}

veilid-core/src/routing_table/bucket_entry.rs

@@ -0,0 +1,446 @@
+use super::*;
+
+// Latency entry is per round-trip packet (ping or data)
+// - Size is number of entries
+const ROLLING_LATENCIES_SIZE: usize = 10;
+
+// Transfers entries are in bytes total for the interval
+// - Size is number of entries
+// - Interval is number of seconds in each entry
+const ROLLING_TRANSFERS_SIZE: usize = 10;
+pub const ROLLING_TRANSFERS_INTERVAL_SECS: u32 = 10;
+
+// Reliable pings are done with increased spacing between pings
+// - Start secs is the number of seconds between the first two pings
+// - Max secs is the maximum number of seconds between consecutive pings
+// - Multiplier changes the number of seconds between pings over time
+//   making it longer as the node becomes more reliable
+const RELIABLE_PING_INTERVAL_START_SECS: u32 = 10;
+const RELIABLE_PING_INTERVAL_MAX_SECS: u32 = 10 * 60;
+const RELIABLE_PING_INTERVAL_MULTIPLIER: f64 = 2.0;
+
+// Unreliable pings are done for a fixed amount of time while the
+// node is given a chance to come back online before it is made dead
+// If a node misses a single ping, it is marked unreliable and must
+// return reliable pings for the duration of the span before being
+// marked reliable again
+// - Span is the number of seconds total to attempt to validate the node
+// - Interval is the number of seconds between each ping
+const UNRELIABLE_PING_SPAN_SECS: u32 = 60;
+const UNRELIABLE_PING_INTERVAL_SECS: u32 = 5;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum BucketEntryState {
+    Reliable,
+    Unreliable,
+    Dead,
+}
+
+#[derive(Debug, Clone)]
+pub struct BucketEntry {
+    pub(super) ref_count: u32,
+    min_max_version: Option<(u8, u8)>,
+    last_connection: Option<(ConnectionDescriptor, u64)>,
+    dial_info_entries: VecDeque<DialInfoEntry>,
+    rolling_latencies: VecDeque<u64>,
+    rolling_transfers: VecDeque<(u64, u64)>,
+    current_transfer: (u64, u64),
+    peer_stats: PeerStats,
+}
+
+impl BucketEntry {
+    pub(super) fn new() -> Self {
+        Self {
+            ref_count: 0,
+            min_max_version: None,
+            last_connection: None,
+            dial_info_entries: VecDeque::new(),
+            rolling_latencies: VecDeque::new(),
+            rolling_transfers: VecDeque::new(),
+            current_transfer: (0, 0),
+            peer_stats: PeerStats {
+                time_added: get_timestamp(),
+                last_seen: None,
+                ping_stats: PingStats::default(),
+                latency: None,
+                transfer: (TransferStats::default(), TransferStats::default()),
+                node_info: None,
+            },
+        }
+    }
+
+    pub fn add_dial_info(&mut self, dial_info: DialInfo) -> Result<(), String> {
+        let mut idx: Option<usize> = None;
+        for i in 0..self.dial_info_entries.len() {
+            if self.dial_info_entries[i].dial_info() == &dial_info {
+                idx = Some(i);
+                break;
+            }
+        }
+        match idx {
+            None => {
+                self.dial_info_entries
+                    .push_front(DialInfoEntry::try_new(dial_info)?);
+            }
+            Some(idx) => {
+                let die = self.dial_info_entries.remove(idx).unwrap();
+                self.dial_info_entries.push_front(die);
+            }
+        }
+        Ok(())
+    }
+
+    pub fn best_dial_info(&self) -> Option<DialInfo> {
+        self.dial_info_entries
+            .front()
+            .map(|die| die.dial_info().clone())
+    }
+
+    pub fn filtered_dial_info<F>(&self, filter: F) -> Option<DialInfo>
+    where
+        F: Fn(&DialInfoEntry) -> bool,
+    {
+        for die in &self.dial_info_entries {
+            if filter(die) {
+                return Some(die.dial_info().clone());
+            }
+        }
+        None
+    }
+
+    pub fn dial_info_entries_as_ref(&self) -> &VecDeque<DialInfoEntry> {
+        &self.dial_info_entries
+    }
+
+    pub fn dial_info(&self) -> Vec<DialInfo> {
+        self.dial_info_entries
+            .iter()
+            .map(|e| e.dial_info().clone())
+            .collect()
+    }
+
+    pub fn public_dial_info(&self) -> Vec<DialInfo> {
+        self.dial_info_entries
+            .iter()
+            .filter_map(|e| {
+                if e.is_public() {
+                    Some(e.dial_info().clone())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn public_dial_info_for_protocol(&self, protocol_type: ProtocolType) -> Vec<DialInfo> {
+        self.dial_info_entries
+            .iter()
+            .filter_map(|e| {
+                if e.dial_info().protocol_type() != protocol_type {
+                    None
+                } else if e.is_public() {
+                    Some(e.dial_info().clone())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn private_dial_info(&self) -> Vec<DialInfo> {
+        self.dial_info_entries
+            .iter()
+            .filter_map(|e| {
+                if e.is_private() {
+                    Some(e.dial_info().clone())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn private_dial_info_for_protocol(&mut self, protocol_type: ProtocolType) -> Vec<DialInfo> {
+        self.dial_info_entries
+            .iter_mut()
+            .filter_map(|e| {
+                if e.dial_info().protocol_type() != protocol_type {
+                    None
+                } else if e.is_private() {
+                    Some(e.dial_info().clone())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn get_peer_info(&self, key: DHTKey, scope: PeerScope) -> PeerInfo {
+        PeerInfo {
+            node_id: NodeId::new(key),
+            dial_infos: match scope {
+                PeerScope::All => self.dial_info(),
+                PeerScope::Public => self.public_dial_info(),
+                PeerScope::Private => self.private_dial_info(),
+            },
+        }
+    }
+
+    pub fn set_last_connection(&mut self, last_connection: ConnectionDescriptor, timestamp: u64) {
+        self.last_connection = Some((last_connection, timestamp));
+
+        // sort the dialinfoentries by the last peer address if we have a match
+        // if one particular peer address is being used and matches a dialinfoentry
+        // then we should prefer it
+        for i in 0..self.dial_info_entries.len() {
+            let die = &mut self.dial_info_entries[i];
+
+            // see if we have a matching address
+            if RoutingTable::dial_info_peer_address_match(
+                die.dial_info(),
+                &self.last_connection.as_ref().unwrap().0.remote,
+            ) {
+                drop(die);
+
+                // push the most recent dialinfo to the front
+                let dies = &mut self.dial_info_entries;
+                let die = dies.remove(i).unwrap();
+                dies.push_front(die);
+
+                break;
+            }
+        }
+    }
+
+    pub fn last_connection(&self) -> Option<ConnectionDescriptor> {
+        match self.last_connection.as_ref() {
+            Some(x) => Some(x.0.clone()),
+            None => None,
+        }
+    }
+
+    pub fn set_min_max_version(&mut self, min_max_version: (u8, u8)) {
+        self.min_max_version = Some(min_max_version);
+    }
+
+    pub fn min_max_version(&self) -> Option<(u8, u8)> {
+        self.min_max_version
+    }
+
+    pub fn state(&self, cur_ts: u64) -> BucketEntryState {
+        if self.check_reliable(cur_ts) {
+            BucketEntryState::Reliable
+        } else if self.check_dead(cur_ts) {
+            BucketEntryState::Dead
+        } else {
+            BucketEntryState::Unreliable
+        }
+    }
+
+    pub fn peer_stats(&self) -> &PeerStats {
+        &self.peer_stats
+    }
+
+    pub fn update_node_info(&mut self, node_info: NodeInfo) {
+        self.peer_stats.node_info = Some(node_info);
+    }
+
+    ///// stats methods
+    // called every ROLLING_TRANSFERS_INTERVAL_SECS seconds
+    pub(super) fn roll_transfers(&mut self, last_ts: u64, cur_ts: u64) {
+        let dur_ms = (cur_ts - last_ts) / 1000u64;
+        while self.rolling_transfers.len() >= ROLLING_TRANSFERS_SIZE {
+            self.rolling_transfers.pop_front();
+        }
+        self.rolling_transfers.push_back(self.current_transfer);
+        self.current_transfer = (0, 0);
+
+        let xd = &mut self.peer_stats.transfer.0;
+        let xu = &mut self.peer_stats.transfer.1;
+
+        xd.maximum = 0;
+        xu.maximum = 0;
+        xd.minimum = u64::MAX;
+        xu.minimum = u64::MAX;
+        xd.average = 0;
+        xu.average = 0;
+        for (rtd, rtu) in &self.rolling_transfers {
+            let bpsd = rtd * 1000u64 / dur_ms;
+            let bpsu = rtu * 1000u64 / dur_ms;
+            if bpsd > xd.maximum {
+                xd.maximum = bpsd;
+            }
+            if bpsu > xu.maximum {
+                xu.maximum = bpsu;
+            }
+            if bpsd < xd.minimum {
+                xd.minimum = bpsd;
+            }
+            if bpsu < xu.minimum {
+                xu.minimum = bpsu;
+            }
+            xd.average += bpsd;
+            xu.average += bpsu;
+        }
+        let len = self.rolling_transfers.len() as u64;
+        xd.average /= len;
+        xu.average /= len;
+        // total remains unchanged
+    }
+
+    // Called for every round trip packet we receive
+    fn record_latency(&mut self, latency: u64) {
+        while self.rolling_latencies.len() >= ROLLING_LATENCIES_SIZE {
+            self.rolling_latencies.pop_front();
+        }
+        self.rolling_latencies.push_back(latency);
+
+        let mut ls = LatencyStats {
+            fastest: 0,
+            average: 0,
+            slowest: 0,
+        };
+        for rl in &self.rolling_latencies {
+            if *rl < ls.fastest {
+                ls.fastest = *rl;
+            }
+            if *rl > ls.slowest {
+                ls.slowest = *rl;
+            }
+            ls.average += *rl;
+        }
+        let len = self.rolling_latencies.len() as u64;
+        ls.average /= len;
+
+        self.peer_stats.latency = Some(ls);
+    }
+
+    ///// state machine handling
+    pub(super) fn check_reliable(&self, cur_ts: u64) -> bool {
+        // if we have had consecutive ping replies for longer that UNRELIABLE_PING_SPAN_SECS
+        match self.peer_stats.ping_stats.first_consecutive_pong_time {
+            None => false,
+            Some(ts) => (cur_ts - ts) >= (UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64),
+        }
+    }
+    pub(super) fn check_dead(&self, cur_ts: u64) -> bool {
+        // if we have not heard from the node at all for the duration of the unreliable ping span
+        match self.peer_stats.last_seen {
+            None => true,
+            Some(ts) => (cur_ts - ts) >= (UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64),
+        }
+    }
+
+    pub(super) fn needs_ping(&self, cur_ts: u64) -> bool {
+        // if we need a ping right now to validate us
+        match self.state(cur_ts) {
+            BucketEntryState::Reliable => {
+                // If we are in a reliable state, we need a ping on an exponential scale
+                match self.peer_stats.ping_stats.last_pinged {
+                    None => true,
+                    Some(last_pinged) => {
+                        let first_consecutive_pong_time = self
+                            .peer_stats
+                            .ping_stats
+                            .first_consecutive_pong_time
+                            .unwrap();
+                        let start_of_reliable_time = first_consecutive_pong_time
+                            + (UNRELIABLE_PING_SPAN_SECS as u64 * 1000000u64);
+                        let reliable_cur = cur_ts - start_of_reliable_time;
+                        let reliable_last = last_pinged - start_of_reliable_time;
+
+                        retry_falloff_log(
+                            reliable_last,
+                            reliable_cur,
+                            RELIABLE_PING_INTERVAL_START_SECS as u64 * 1000000u64,
+                            RELIABLE_PING_INTERVAL_MAX_SECS as u64 * 1000000u64,
+                            RELIABLE_PING_INTERVAL_MULTIPLIER,
+                        )
+                    }
+                }
+            }
+            BucketEntryState::Unreliable => {
+                // If we are in an unreliable state, we need a ping every UNRELIABLE_PING_INTERVAL_SECS seconds
+                match self.peer_stats.ping_stats.last_pinged {
+                    None => true,
+                    Some(last_pinged) => {
+                        (cur_ts - last_pinged)
+                            >= (UNRELIABLE_PING_INTERVAL_SECS as u64 * 1000000u64)
+                    }
+                }
+            }
+            BucketEntryState::Dead => false,
+        }
+    }
+
+    pub(super) fn touch_last_seen(&mut self, ts: u64) {
+        // If we've heard from the node at all, we can always restart our lost ping count
+        self.peer_stats.ping_stats.recent_lost_pings = 0;
+        // Mark the node as seen
+        self.peer_stats.last_seen = Some(ts);
+    }
+
+    ////////////////////////////////////////////////////////////////
+    /// Called by RPC processor as events happen
+
+    pub fn ping_sent(&mut self, ts: u64, bytes: u64) {
+        self.peer_stats.ping_stats.total_sent += 1;
+        self.current_transfer.1 += bytes;
+        self.peer_stats.ping_stats.in_flight += 1;
+        self.peer_stats.ping_stats.last_pinged = Some(ts);
+    }
+    pub fn ping_rcvd(&mut self, ts: u64, bytes: u64) {
+        self.current_transfer.0 += bytes;
+        self.touch_last_seen(ts);
+    }
+    pub fn pong_sent(&mut self, _ts: u64, bytes: u64) {
+        self.current_transfer.1 += bytes;
+    }
+    pub fn pong_rcvd(&mut self, send_ts: u64, recv_ts: u64, bytes: u64) {
+        self.current_transfer.0 += bytes;
+        self.peer_stats.ping_stats.in_flight -= 1;
+        self.peer_stats.ping_stats.total_returned += 1;
+        self.peer_stats.ping_stats.consecutive_pongs += 1;
+        if self
+            .peer_stats
+            .ping_stats
+            .first_consecutive_pong_time
+            .is_none()
+        {
+            self.peer_stats.ping_stats.first_consecutive_pong_time = Some(recv_ts);
+        }
+        self.record_latency(recv_ts - send_ts);
+        self.touch_last_seen(recv_ts);
+    }
+    pub fn ping_lost(&mut self, _ts: u64) {
+        self.peer_stats.ping_stats.in_flight -= 1;
+        self.peer_stats.ping_stats.recent_lost_pings += 1;
+        self.peer_stats.ping_stats.consecutive_pongs = 0;
+        self.peer_stats.ping_stats.first_consecutive_pong_time = None;
+    }
+    pub fn question_sent(&mut self, _ts: u64, bytes: u64) {
+        self.current_transfer.1 += bytes;
+    }
+    pub fn question_rcvd(&mut self, ts: u64, bytes: u64) {
+        self.current_transfer.0 += bytes;
+        self.touch_last_seen(ts);
+    }
+    pub fn answer_sent(&mut self, _ts: u64, bytes: u64) {
+        self.current_transfer.1 += bytes;
+    }
+    pub fn answer_rcvd(&mut self, send_ts: u64, recv_ts: u64, bytes: u64) {
+        self.current_transfer.0 += bytes;
+        self.record_latency(recv_ts - send_ts);
+        self.touch_last_seen(recv_ts);
+    }
+    pub fn question_lost(&mut self, _ts: u64) {
+        self.peer_stats.ping_stats.consecutive_pongs = 0;
+        self.peer_stats.ping_stats.first_consecutive_pong_time = None;
+    }
+}
+
+impl Drop for BucketEntry {
+    fn drop(&mut self) {
+        assert_eq!(self.ref_count, 0);
+    }
+}

veilid-core/src/routing_table/dial_info_entry.rs

@@ -0,0 +1,61 @@
+use super::*;
+
+#[derive(Debug, Clone)]
+pub struct DialInfoEntry {
+    dial_info: DialInfo,
+    resolved_address: IpAddr,
+}
+
+impl DialInfoEntry {
+    pub fn try_new(dial_info: DialInfo) -> Result<Self, String> {
+        let addr = match dial_info.resolve() {
+            Ok(a) => a,
+            Err(_) => return Err("failed to resolve address".to_owned()),
+        };
+        Ok(Self {
+            dial_info: dial_info,
+            resolved_address: addr,
+        })
+    }
+
+    pub fn dial_info(&self) -> &DialInfo {
+        &self.dial_info
+    }
+
+    pub fn address(&self) -> IpAddr {
+        self.resolved_address
+    }
+
+    pub fn resolve(&mut self) -> Result<IpAddr, String> {
+        let addr = match self.dial_info.resolve() {
+            Ok(a) => a,
+            Err(_) => return Err("failed to resolve address".to_owned()),
+        };
+        self.resolved_address = addr;
+        Ok(addr)
+    }
+
+    pub fn matches_peer_scope(&self, scope: PeerScope) -> bool {
+        match scope {
+            PeerScope::All => true,
+            PeerScope::Public => self.is_public(),
+            PeerScope::Private => self.is_private(),
+        }
+    }
+
+    pub fn is_public(&self) -> bool {
+        ipaddr_is_global(&self.resolved_address)
+    }
+    pub fn is_private(&self) -> bool {
+        match self.resolved_address {
+            IpAddr::V4(a) => ipv4addr_is_private(&a),
+            IpAddr::V6(a) => ipv6addr_is_unicast_site_local(&a),
+        }
+    }
+    pub fn is_valid(&self) -> bool {
+        self.is_public() || self.is_private()
+    }
+    pub fn is_loopback(&self) -> bool {
+        ipaddr_is_loopback(&self.resolved_address)
+    }
+}

veilid-core/src/routing_table/find_nodes.rs

@@ -0,0 +1,280 @@
+use super::*;
+
+use crate::dht::*;
+use crate::intf::*;
+use crate::xx::*;
+use crate::*;
+
+impl RoutingTable {
+    // Retrieve the fastest nodes in the routing table with a particular kind of protocol address type
+    // Returns noderefs are are scoped to that address type only
+    pub fn get_fast_nodes_of_type(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Vec<NodeRef> {
+        self.find_fastest_nodes(
+            // filter
+            Some(Box::new(
+                move |params: &(&DHTKey, Option<&mut BucketEntry>)| {
+                    // Only interested in nodes with node info
+                    if let Some(node_info) = &params.1.as_ref().unwrap().peer_stats().node_info {
+                        // Will the node validate dial info?
+                        // and does it have a UDPv4, public scope, dial info?
+                        if node_info.will_validate_dial_info
+                            && params
+                                .1
+                                .as_ref()
+                                .unwrap()
+                                .dial_info_entries_as_ref()
+                                .iter()
+                                .find_map(|die| {
+                                    if die.matches_peer_scope(PeerScope::Public)
+                                        && die.dial_info().protocol_address_type()
+                                            == protocol_address_type
+                                    {
+                                        Some(())
+                                    } else {
+                                        None
+                                    }
+                                })
+                                .is_some()
+                        {
+                            // If so return true and include this node
+                            return true;
+                        }
+                    }
+                    false
+                },
+            )),
+            // transform
+            |e| {
+                NodeRef::new_filtered(
+                    self.clone(),
+                    *e.0,
+                    e.1.as_mut().unwrap(),
+                    protocol_address_type,
+                )
+            },
+        )
+    }
+
+    pub fn get_own_peer_info(&self, scope: PeerScope) -> PeerInfo {
+        let dial_infos = match scope {
+            PeerScope::All => {
+                let mut divec = self.public_dial_info();
+                divec.append(&mut self.local_dial_info());
+                divec.dedup();
+                divec
+            }
+            PeerScope::Public => self.public_dial_info(),
+            PeerScope::Private => self.local_dial_info(),
+        };
+
+        PeerInfo {
+            node_id: NodeId::new(self.node_id()),
+            dial_infos: dial_infos.iter().map(|x| x.dial_info.clone()).collect(),
+        }
+    }
+
+    pub fn transform_to_peer_info(
+        kv: &mut (&DHTKey, Option<&mut BucketEntry>),
+        scope: PeerScope,
+        own_peer_info: &PeerInfo,
+    ) -> PeerInfo {
+        match &kv.1 {
+            None => own_peer_info.clone(),
+            Some(entry) => entry.get_peer_info(*kv.0, scope),
+        }
+    }
+
+    pub fn find_peers_with_sort_and_filter<F, C, T, O>(
+        &self,
+        node_count: usize,
+        cur_ts: u64,
+        filter: F,
+        compare: C,
+        transform: T,
+    ) -> Vec<O>
+    where
+        F: Fn(&(&DHTKey, Option<&mut BucketEntry>)) -> bool,
+        C: Fn(
+            &(&DHTKey, Option<&mut BucketEntry>),
+            &(&DHTKey, Option<&mut BucketEntry>),
+        ) -> core::cmp::Ordering,
+        T: Fn(&mut (&DHTKey, Option<&mut BucketEntry>)) -> O,
+    {
+        let mut inner = self.inner.lock();
+
+        // collect all the nodes for sorting
+        let mut nodes =
+            Vec::<(&DHTKey, Option<&mut BucketEntry>)>::with_capacity(inner.bucket_entry_count + 1);
+        // add our own node (only one of there with the None entry)
+        let self_node_id = inner.node_id.clone();
+        let selfkv = (&self_node_id, None);
+        if filter(&selfkv) {
+            nodes.push(selfkv);
+        }
+        // add all nodes from buckets
+        for b in &mut inner.buckets {
+            for (k, v) in b.entries_mut() {
+                // Don't bother with dead nodes
+                if !v.check_dead(cur_ts) {
+                    // Apply filter
+                    let kv = (k, Some(v));
+                    if filter(&kv) {
+                        nodes.push(kv);
+                    }
+                }
+            }
+        }
+
+        // sort by preference for returning nodes
+        nodes.sort_by(compare);
+
+        // return transformed vector for filtered+sorted nodes
+        let cnt = usize::min(node_count, nodes.len());
+        let mut out = Vec::<O>::with_capacity(cnt);
+        for i in 0..cnt {
+            let val = transform(&mut nodes[i]);
+            out.push(val);
+        }
+
+        out
+    }
+
+    pub fn find_fastest_nodes<T, O>(
+        &self,
+        filter: Option<Box<dyn Fn(&(&DHTKey, Option<&mut BucketEntry>)) -> bool>>,
+        transform: T,
+    ) -> Vec<O>
+    where
+        T: Fn(&mut (&DHTKey, Option<&mut BucketEntry>)) -> O,
+    {
+        let cur_ts = get_timestamp();
+        let node_count = {
+            let c = self.config.get();
+            c.network.dht.max_find_node_count as usize
+        };
+        let out = self.find_peers_with_sort_and_filter(
+            node_count,
+            cur_ts,
+            // filter
+            |kv| {
+                if kv.1.is_none() {
+                    // filter out self peer, as it is irrelevant to the 'fastest nodes' search
+                    false
+                } else if filter.is_some() && !filter.as_ref().unwrap()(kv) {
+                    false
+                } else {
+                    true
+                }
+            },
+            // sort
+            |(a_key, a_entry), (b_key, b_entry)| {
+                // same nodes are always the same
+                if a_key == b_key {
+                    return core::cmp::Ordering::Equal;
+                }
+                // our own node always comes last (should not happen, here for completeness)
+                if a_entry.is_none() {
+                    return core::cmp::Ordering::Greater;
+                }
+                if b_entry.is_none() {
+                    return core::cmp::Ordering::Less;
+                }
+                // reliable nodes come first
+                let ae = a_entry.as_ref().unwrap();
+                let be = b_entry.as_ref().unwrap();
+
+                let ra = ae.check_reliable(cur_ts);
+                let rb = be.check_reliable(cur_ts);
+                if ra != rb {
+                    if ra {
+                        return core::cmp::Ordering::Less;
+                    } else {
+                        return core::cmp::Ordering::Greater;
+                    }
+                }
+
+                // latency is the next metric, closer nodes first
+                let a_latency = match ae.peer_stats().latency.as_ref() {
+                    None => {
+                        // treat unknown latency as slow
+                        return core::cmp::Ordering::Greater;
+                    }
+                    Some(l) => l,
+                };
+                let b_latency = match be.peer_stats().latency.as_ref() {
+                    None => {
+                        // treat unknown latency as slow
+                        return core::cmp::Ordering::Less;
+                    }
+                    Some(l) => l,
+                };
+                // Sort by average latency
+                a_latency.average.cmp(&b_latency.average)
+            },
+            // transform,
+            transform,
+        );
+        trace!(">> find_fastest_nodes: node count = {}", out.len());
+        out
+    }
+
+    pub fn find_closest_nodes<T, O>(
+        &self,
+        node_id: DHTKey,
+        filter: Option<Box<dyn Fn(&(&DHTKey, Option<&mut BucketEntry>)) -> bool>>,
+        transform: T,
+    ) -> Vec<O>
+    where
+        T: Fn(&mut (&DHTKey, Option<&mut BucketEntry>)) -> O,
+    {
+        let cur_ts = get_timestamp();
+        let node_count = {
+            let c = self.config.get();
+            c.network.dht.max_find_node_count as usize
+        };
+        let out = self.find_peers_with_sort_and_filter(
+            node_count,
+            cur_ts,
+            // filter
+            |kv| {
+                if kv.1.is_none() {
+                    // include self peer, as it is relevant to the 'closest nodes' search
+                    true
+                } else if filter.is_some() && !filter.as_ref().unwrap()(kv) {
+                    false
+                } else {
+                    true
+                }
+            },
+            // sort
+            |(a_key, a_entry), (b_key, b_entry)| {
+                // same nodes are always the same
+                if a_key == b_key {
+                    return core::cmp::Ordering::Equal;
+                }
+                // reliable nodes come first, pessimistically treating our own node as unreliable
+                let ra = a_entry.as_ref().map_or(false, |x| x.check_reliable(cur_ts));
+                let rb = b_entry.as_ref().map_or(false, |x| x.check_reliable(cur_ts));
+                if ra != rb {
+                    if ra {
+                        return core::cmp::Ordering::Less;
+                    } else {
+                        return core::cmp::Ordering::Greater;
+                    }
+                }
+
+                // distance is the next metric, closer nodes first
+                let da = distance(a_key, &node_id);
+                let db = distance(b_key, &node_id);
+                da.cmp(&db)
+            },
+            // transform,
+            transform,
+        );
+        trace!(">> find_closest_nodes: node count = {}", out.len());
+        out
+    }
+}

veilid-core/src/routing_table/mod.rs

@@ -0,0 +1,731 @@
+mod bucket;
+mod bucket_entry;
+mod dial_info_entry;
+mod find_nodes;
+mod node_ref;
+
+use bucket::*;
+pub use bucket_entry::*;
+pub use dial_info_entry::*;
+pub use find_nodes::*;
+pub use node_ref::*;
+
+use crate::dht::*;
+use crate::intf::*;
+use crate::network_manager::*;
+use crate::rpc_processor::*;
+use crate::xx::*;
+use crate::*;
+use alloc::collections::VecDeque;
+use alloc::str::FromStr;
+use futures_util::stream::{FuturesUnordered, StreamExt};
+
+//////////////////////////////////////////////////////////////////////////
+
+#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Ord, Eq)]
+pub enum DialInfoOrigin {
+    Static,
+    Discovered,
+    Mapped,
+}
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq)]
+pub struct DialInfoDetail {
+    pub dial_info: DialInfo,
+    pub origin: DialInfoOrigin,
+    pub network_class: Option<NetworkClass>,
+    pub timestamp: u64,
+}
+
+struct RoutingTableInner {
+    network_manager: NetworkManager,
+    node_id: DHTKey,
+    node_id_secret: DHTKeySecret,
+    buckets: Vec<Bucket>,
+    //recent_nodes: VecDeque<DHTKey>,
+    //closest_reliable_nodes: Vec<DHTKey>,
+    //fastest_reliable_nodes: Vec<DHTKey>,
+    //closest_nodes: Vec<DHTKey>,
+    //fastest_nodes: Vec<DHTKey>,
+    local_dial_info: Vec<DialInfoDetail>,
+    public_dial_info: Vec<DialInfoDetail>,
+    bucket_entry_count: usize,
+    // Waiters
+    eventual_changed_dial_info: Eventual,
+}
+
+struct RoutingTableUnlockedInner {
+    // Background processes
+    rolling_transfers_task: TickTask,
+    bootstrap_task: TickTask,
+    peer_minimum_refresh_task: TickTask,
+    ping_validator_task: TickTask,
+}
+
+#[derive(Clone)]
+pub struct RoutingTable {
+    config: VeilidConfig,
+    inner: Arc<Mutex<RoutingTableInner>>,
+    unlocked_inner: Arc<RoutingTableUnlockedInner>,
+}
+
+impl RoutingTable {
+    fn new_inner(network_manager: NetworkManager) -> RoutingTableInner {
+        RoutingTableInner {
+            network_manager: network_manager,
+            node_id: DHTKey::default(),
+            node_id_secret: DHTKeySecret::default(),
+            buckets: Vec::new(),
+            //recent_nodes: VecDeque::new(),
+            //closest_reliable_nodes: Vec::new(),
+            //fastest_reliable_nodes: Vec::new(),
+            //closest_nodes: Vec::new(),
+            //fastest_nodes: Vec::new(),
+            local_dial_info: Vec::new(),
+            public_dial_info: Vec::new(),
+            bucket_entry_count: 0,
+            eventual_changed_dial_info: Eventual::new(),
+        }
+    }
+    fn new_unlocked_inner(config: VeilidConfig) -> RoutingTableUnlockedInner {
+        let c = config.get();
+        RoutingTableUnlockedInner {
+            rolling_transfers_task: TickTask::new(bucket_entry::ROLLING_TRANSFERS_INTERVAL_SECS),
+            bootstrap_task: TickTask::new(1),
+            peer_minimum_refresh_task: TickTask::new_us(c.network.dht.min_peer_refresh_time),
+            ping_validator_task: TickTask::new(1),
+        }
+    }
+    pub fn new(network_manager: NetworkManager) -> Self {
+        let config = network_manager.config();
+        let this = Self {
+            config: config.clone(),
+            inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
+            unlocked_inner: Arc::new(Self::new_unlocked_inner(config)),
+        };
+        // Set rolling transfers tick task
+        {
+            let this2 = this.clone();
+            this.unlocked_inner
+                .rolling_transfers_task
+                .set_routine(move |l, t| {
+                    Box::pin(this2.clone().rolling_transfers_task_routine(l, t))
+                });
+        }
+        // Set bootstrap tick task
+        {
+            let this2 = this.clone();
+            this.unlocked_inner
+                .bootstrap_task
+                .set_routine(move |_l, _t| Box::pin(this2.clone().bootstrap_task_routine()));
+        }
+        // Set peer minimum refresh tick task
+        {
+            let this2 = this.clone();
+            this.unlocked_inner
+                .peer_minimum_refresh_task
+                .set_routine(move |_l, _t| {
+                    Box::pin(this2.clone().peer_minimum_refresh_task_routine())
+                });
+        }
+        // Set ping validator tick task
+        {
+            let this2 = this.clone();
+            this.unlocked_inner
+                .ping_validator_task
+                .set_routine(move |l, t| Box::pin(this2.clone().ping_validator_task_routine(l, t)));
+        }
+        this
+    }
+
+    pub fn network_manager(&self) -> NetworkManager {
+        self.inner.lock().network_manager.clone()
+    }
+    pub fn rpc_processor(&self) -> RPCProcessor {
+        self.network_manager().rpc_processor()
+    }
+
+    pub fn node_id(&self) -> DHTKey {
+        self.inner.lock().node_id
+    }
+
+    pub fn node_id_secret(&self) -> DHTKeySecret {
+        self.inner.lock().node_id_secret
+    }
+
+    pub fn has_local_dial_info(&self) -> bool {
+        let inner = self.inner.lock();
+        inner.local_dial_info.len() > 0
+    }
+
+    pub fn local_dial_info(&self) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner.local_dial_info.clone()
+    }
+
+    pub fn local_dial_info_for_protocol(&self, protocol_type: ProtocolType) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner
+            .local_dial_info
+            .iter()
+            .filter_map(|di| {
+                if di.dial_info.protocol_type() != protocol_type {
+                    None
+                } else {
+                    Some(di.clone())
+                }
+            })
+            .collect()
+    }
+
+    pub fn local_dial_info_for_protocol_address_type(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner
+            .local_dial_info
+            .iter()
+            .filter_map(|di| {
+                if di.dial_info.protocol_address_type() != protocol_address_type {
+                    None
+                } else {
+                    Some(di.clone())
+                }
+            })
+            .collect()
+    }
+
+    pub fn register_local_dial_info(&self, dial_info: DialInfo, origin: DialInfoOrigin) {
+        let ts = get_timestamp();
+        let mut inner = self.inner.lock();
+
+        inner.local_dial_info.push(DialInfoDetail {
+            dial_info: dial_info.clone(),
+            origin: origin,
+            network_class: None,
+            timestamp: ts,
+        });
+
+        info!(
+            "Local Dial Info: {} ({:?})",
+            NodeDialInfoSingle {
+                node_id: NodeId::new(inner.node_id),
+                dial_info: dial_info.clone()
+            }
+            .to_string(),
+            origin,
+        );
+    }
+
+    pub fn clear_local_dial_info(&self) {
+        self.inner.lock().local_dial_info.clear();
+    }
+
+    pub fn has_public_dial_info(&self) -> bool {
+        let inner = self.inner.lock();
+        inner.public_dial_info.len() > 0
+    }
+
+    pub fn public_dial_info(&self) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner.public_dial_info.clone()
+    }
+
+    pub fn public_dial_info_for_protocol(
+        &self,
+        protocol_type: ProtocolType,
+    ) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner
+            .public_dial_info
+            .iter()
+            .filter_map(|di| {
+                if di.dial_info.protocol_type() != protocol_type {
+                    None
+                } else {
+                    Some(di.clone())
+                }
+            })
+            .collect()
+    }
+    pub fn public_dial_info_for_protocol_address_type(
+        &self,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Vec<DialInfoDetail> {
+        let inner = self.inner.lock();
+        inner
+            .public_dial_info
+            .iter()
+            .filter_map(|di| {
+                if di.dial_info.protocol_address_type() != protocol_address_type {
+                    None
+                } else {
+                    Some(di.clone())
+                }
+            })
+            .collect()
+    }
+
+    pub fn register_public_dial_info(
+        &self,
+        dial_info: DialInfo,
+        network_class: Option<NetworkClass>,
+        origin: DialInfoOrigin,
+    ) {
+        let ts = get_timestamp();
+        let mut inner = self.inner.lock();
+
+        inner.public_dial_info.push(DialInfoDetail {
+            dial_info: dial_info.clone(),
+            origin: origin,
+            network_class: network_class,
+            timestamp: ts,
+        });
+
+        info!(
+            "Public Dial Info: {} ({:?}#{:?})",
+            NodeDialInfoSingle {
+                node_id: NodeId::new(inner.node_id),
+                dial_info: dial_info.clone()
+            }
+            .to_string(),
+            origin,
+            network_class,
+        );
+    }
+
+    pub fn clear_public_dial_info(&self) {
+        self.inner.lock().public_dial_info.clear();
+    }
+
+    pub async fn wait_changed_dial_info(&self) {
+        let inst = self
+            .inner
+            .lock()
+            .eventual_changed_dial_info
+            .instance_empty();
+        inst.await;
+    }
+    pub async fn trigger_changed_dial_info(&self) {
+        let eventual = {
+            let mut inner = self.inner.lock();
+            let mut new_eventual = Eventual::new();
+            core::mem::swap(&mut inner.eventual_changed_dial_info, &mut new_eventual);
+            new_eventual
+        };
+        eventual.resolve().await;
+    }
+
+    fn bucket_depth(index: usize) -> usize {
+        match index {
+            0 => 256,
+            1 => 128,
+            2 => 64,
+            3 => 32,
+            4 => 16,
+            5 => 8,
+            6 => 4,
+            7 => 4,
+            8 => 4,
+            9 => 4,
+            _ => 4,
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), String> {
+        let mut inner = self.inner.lock();
+        // Size the buckets (one per bit)
+        inner.buckets.reserve(DHT_KEY_LENGTH * 8);
+        for _ in 0..DHT_KEY_LENGTH * 8 {
+            let bucket = Bucket::new(self.clone());
+            inner.buckets.push(bucket);
+        }
+
+        // make local copy of node id for easy access
+        let c = self.config.get();
+        inner.node_id = c.network.node_id;
+        inner.node_id_secret = c.network.node_id_secret;
+
+        Ok(())
+    }
+
+    pub async fn terminate(&self) {
+        *self.inner.lock() = Self::new_inner(self.network_manager());
+    }
+
+    // Just match address and port to help sort dialinfoentries for buckets
+    // because inbound connections will not have dialinfo associated with them
+    // but should have ip addresses if they have changed
+    fn dial_info_peer_address_match(dial_info: &DialInfo, peer_addr: &PeerAddress) -> bool {
+        match dial_info {
+            DialInfo::UDP(_) => {
+                peer_addr.protocol_type == ProtocolType::UDP
+                    && peer_addr.port == dial_info.port()
+                    && peer_addr.address.address_string() == dial_info.address_string()
+            }
+            DialInfo::TCP(_) => {
+                peer_addr.protocol_type == ProtocolType::TCP
+                    && peer_addr.port == dial_info.port()
+                    && peer_addr.address.address_string() == dial_info.address_string()
+            }
+            DialInfo::WS(_) => {
+                peer_addr.protocol_type == ProtocolType::WS
+                    && peer_addr.port == dial_info.port()
+                    && peer_addr.address.address_string() == dial_info.address_string()
+            }
+            DialInfo::WSS(_) => {
+                peer_addr.protocol_type == ProtocolType::WSS
+                    && peer_addr.port == dial_info.port()
+                    && peer_addr.address.address_string() == dial_info.address_string()
+            }
+        }
+    }
+
+    // Attempt to settle buckets and remove entries down to the desired number
+    // which may not be possible due extant NodeRefs
+    fn kick_bucket(inner: &mut RoutingTableInner, idx: usize) {
+        let bucket = &mut inner.buckets[idx];
+        let bucket_depth = Self::bucket_depth(idx);
+
+        if let Some(dead_node_ids) = bucket.kick(bucket_depth) {
+            // Remove counts
+            inner.bucket_entry_count -= dead_node_ids.len();
+            debug!("Routing table now has {} nodes", inner.bucket_entry_count);
+
+            // Now purge the routing table inner vectors
+            //let filter = |k: &DHTKey| dead_node_ids.contains(k);
+            //inner.closest_reliable_nodes.retain(filter);
+            //inner.fastest_reliable_nodes.retain(filter);
+            //inner.closest_nodes.retain(filter);
+            //inner.fastest_nodes.retain(filter);
+        }
+    }
+
+    fn find_bucket_index(inner: &RoutingTableInner, node_id: DHTKey) -> usize {
+        distance(&node_id, &inner.node_id)
+            .first_nonzero_bit()
+            .unwrap()
+    }
+
+    fn drop_node_ref(&self, node_id: DHTKey) {
+        // Reduce ref count on entry
+        let mut inner = self.inner.lock();
+        let idx = Self::find_bucket_index(&*inner, node_id);
+        let new_ref_count = {
+            let bucket = &mut inner.buckets[idx];
+            let entry = bucket.entry_mut(&node_id).unwrap();
+            entry.ref_count -= 1;
+            entry.ref_count
+        };
+
+        // If this entry could possibly go away, kick the bucket
+        if new_ref_count == 0 {
+            // it important to do this in the same inner lock as the ref count decrease
+            Self::kick_bucket(&mut *inner, idx);
+        }
+    }
+
+    pub fn create_node_ref(&self, node_id: DHTKey) -> Result<NodeRef, String> {
+        // Ensure someone isn't trying register this node itself
+        if node_id == self.node_id() {
+            return Err("can't register own node".to_owned());
+        }
+
+        // Insert into bucket, possibly evicting the newest bucket member
+        let noderef = match self.lookup_node_ref(node_id) {
+            None => {
+                // Make new entry
+                let mut inner = self.inner.lock();
+                let idx = Self::find_bucket_index(&*inner, node_id);
+                let nr = {
+                    // Get the bucket for the entry
+                    let bucket = &mut inner.buckets[idx];
+                    // Add new entry
+                    let nr = bucket.add_entry(node_id);
+
+                    // Update count
+                    inner.bucket_entry_count += 1;
+                    debug!("Routing table now has {} nodes", inner.bucket_entry_count);
+                    nr
+                };
+
+                // Kick the bucket
+                // It is important to do this in the same inner lock as the add_entry
+                Self::kick_bucket(&mut *inner, idx);
+
+                nr
+            }
+            Some(nr) => nr,
+        };
+
+        Ok(noderef)
+    }
+
+    pub fn lookup_node_ref(&self, node_id: DHTKey) -> Option<NodeRef> {
+        let mut inner = self.inner.lock();
+        let idx = Self::find_bucket_index(&*inner, node_id);
+        let bucket = &mut inner.buckets[idx];
+        match bucket.entry_mut(&node_id) {
+            None => None,
+            Some(e) => Some(NodeRef::new(self.clone(), node_id, e)),
+        }
+    }
+
+    // Shortcut function to add a node to our routing table if it doesn't exist
+    // and add the dial info we have for it, since that's pretty common
+    pub fn register_node_with_dial_info(
+        &self,
+        node_id: DHTKey,
+        dial_infos: &[DialInfo],
+    ) -> Result<NodeRef, String> {
+        let nr = match self.create_node_ref(node_id) {
+            Err(e) => {
+                return Err(format!("Couldn't create node reference: {}", e));
+            }
+            Ok(v) => v,
+        };
+
+        nr.operate(move |e| -> Result<(), String> {
+            for di in dial_infos {
+                e.add_dial_info(di.clone())?;
+            }
+            Ok(())
+        })?;
+
+        Ok(nr)
+    }
+
+    // Shortcut function to add a node to our routing table if it doesn't exist
+    // and add the last peer address we have for it, since that's pretty common
+    pub fn register_node_with_existing_connection(
+        &self,
+        node_id: DHTKey,
+        descriptor: ConnectionDescriptor,
+        timestamp: u64,
+    ) -> Result<NodeRef, String> {
+        let nr = match self.create_node_ref(node_id) {
+            Err(e) => {
+                return Err(format!("Couldn't create node reference: {}", e));
+            }
+            Ok(v) => v,
+        };
+
+        nr.operate(move |e| {
+            // set the most recent node address for connection finding and udp replies
+            e.set_last_connection(descriptor, timestamp);
+        });
+
+        Ok(nr)
+    }
+
+    fn operate_on_bucket_entry<T, F>(&self, node_id: DHTKey, f: F) -> T
+    where
+        F: FnOnce(&mut BucketEntry) -> T,
+    {
+        let mut inner = self.inner.lock();
+        let idx = Self::find_bucket_index(&*inner, node_id);
+        let bucket = &mut inner.buckets[idx];
+        let entry = bucket.entry_mut(&node_id).unwrap();
+        f(entry)
+    }
+
+    pub async fn find_self(&self, node_ref: NodeRef) -> Result<Vec<NodeRef>, String> {
+        let node_id = self.node_id();
+        let rpc_processor = self.rpc_processor();
+
+        let res = match rpc_processor
+            .rpc_call_find_node(
+                Destination::Direct(node_ref.clone()),
+                node_id,
+                None,
+                RespondTo::Sender,
+            )
+            .await
+        {
+            Ok(v) => v,
+            Err(e) => {
+                return Err(format!("couldn't contact node at {:?}: {}", &node_ref, e));
+            }
+        };
+        trace!(
+            "find_self for at {:?} answered in {}ms",
+            &node_ref,
+            timestamp_to_secs(res.latency) * 1000.0f64
+        );
+
+        // register nodes we'd found
+        let mut out = Vec::<NodeRef>::with_capacity(res.peers.len());
+        for p in res.peers {
+            // if our own node if is in the list then ignore it, as we don't add ourselves to our own routing table
+            if p.node_id.key == node_id {
+                // however, it is useful to note when
+                continue;
+            }
+
+            // register the node if it's new
+            let nr = match self.register_node_with_dial_info(p.node_id.key, &p.dial_infos) {
+                Ok(v) => v,
+                Err(e) => {
+                    return Err(format!(
+                        "couldn't register node {} at {:?}: {}",
+                        p.node_id.key, &p.dial_infos, e
+                    ));
+                }
+            };
+            out.push(nr);
+        }
+        Ok(out)
+    }
+
+    pub async fn reverse_find_node(&self, node_ref: NodeRef, wide: bool) {
+        // Ask bootstrap node to 'find' our own node so we can get some more nodes near ourselves
+        // and then contact those nodes to inform -them- that we exist
+
+        // Ask bootstrap server for nodes closest to our own node
+        let closest_nodes = match self.find_self(node_ref.clone()).await {
+            Err(e) => {
+                error!(
+                    "reverse_find_node: find_self failed for {:?}: {}",
+                    &node_ref, e
+                );
+                return;
+            }
+            Ok(v) => v,
+        };
+
+        // Ask each node near us to find us as well
+        if wide {
+            for closest_nr in closest_nodes {
+                match self.find_self(closest_nr.clone()).await {
+                    Err(e) => {
+                        error!(
+                            "reverse_find_node: closest node find_self failed for {:?}: {}",
+                            &closest_nr, e
+                        );
+                        return;
+                    }
+                    Ok(v) => v,
+                };
+            }
+        }
+    }
+
+    async fn bootstrap_task_routine(self) -> Result<(), String> {
+        let bootstrap = {
+            let c = self.config.get();
+            c.network.bootstrap.clone()
+        };
+
+        // Map all bootstrap entries to a single key with multiple dialinfo
+        let mut bsmap: BTreeMap<DHTKey, Vec<DialInfo>> = BTreeMap::new();
+        for b in bootstrap {
+            let ndis = match NodeDialInfoSingle::from_str(b.as_str()) {
+                Err(_) => {
+                    return Err(format!("Invalid dial info in bootstrap entry: {}", b));
+                }
+                Ok(v) => v,
+            };
+            let node_id = ndis.node_id.key;
+            bsmap
+                .entry(node_id)
+                .or_insert(Vec::new())
+                .push(ndis.dial_info);
+        }
+
+        // Run all bootstrap operations concurrently
+        let mut unord = FuturesUnordered::new();
+        for (k, v) in bsmap {
+            let nr = match self.register_node_with_dial_info(k, &v) {
+                Ok(nr) => nr,
+                Err(e) => {
+                    return Err(format!("Couldn't add bootstrap node: {}", e));
+                }
+            };
+
+            info!("Bootstrapping {} with {:?}", k.encode(), &v);
+            unord.push(self.reverse_find_node(nr, true));
+        }
+        while unord.next().await.is_some() {}
+        Ok(())
+    }
+
+    ///////////////////////////////////////////////////////////
+    /// Peer ping validation
+
+    // Ask our remaining peers to give us more peers before we go
+    // back to the bootstrap servers to keep us from bothering them too much
+    async fn peer_minimum_refresh_task_routine(self) -> Result<(), String> {
+        // get list of all peers we know about, even the unreliable ones, and ask them to bootstrap too
+        let noderefs = {
+            let mut inner = self.inner.lock();
+            let mut noderefs = Vec::<NodeRef>::with_capacity(inner.bucket_entry_count);
+            for b in &mut inner.buckets {
+                for (k, entry) in b.entries_mut() {
+                    noderefs.push(NodeRef::new(self.clone(), *k, entry))
+                }
+            }
+            noderefs
+        };
+
+        // do peer minimum search concurrently
+        let mut unord = FuturesUnordered::new();
+        for nr in noderefs {
+            debug!("Peer minimum search with {:?}", nr);
+            unord.push(self.reverse_find_node(nr, false));
+        }
+        while unord.next().await.is_some() {}
+
+        Ok(())
+    }
+
+    // Ping each node in the routing table if they need to be pinged
+    // to determine their reliability
+    async fn ping_validator_task_routine(self, _last_ts: u64, cur_ts: u64) -> Result<(), String> {
+        let rpc = self.rpc_processor();
+        let mut inner = self.inner.lock();
+        for b in &mut inner.buckets {
+            for (k, entry) in b.entries_mut() {
+                if entry.needs_ping(cur_ts) {
+                    let nr = NodeRef::new(self.clone(), *k, entry);
+                    intf::spawn_local(rpc.clone().rpc_call_info(nr)).detach();
+                }
+            }
+        }
+        Ok(())
+    }
+
+    // Compute transfer statistics to determine how 'fast' a node is
+    async fn rolling_transfers_task_routine(self, last_ts: u64, cur_ts: u64) -> Result<(), String> {
+        let mut inner = self.inner.lock();
+        for b in &mut inner.buckets {
+            b.roll_transfers(last_ts, cur_ts);
+        }
+        Ok(())
+    }
+
+    // Ticks about once per second
+    // to run tick tasks which may run at slower tick rates as configured
+    pub async fn tick(&self) -> Result<(), String> {
+        // Do rolling transfers every ROLLING_TRANSFERS_INTERVAL_SECS secs
+        self.unlocked_inner.rolling_transfers_task.tick().await?;
+
+        // If routing table is empty, then add the bootstrap nodes to it
+        if self.inner.lock().bucket_entry_count == 0 {
+            self.unlocked_inner.bootstrap_task.tick().await?;
+        }
+
+        // If we still don't have enough peers, find nodes until we do
+        let min_peer_count = {
+            let c = self.config.get();
+            c.network.dht.min_peer_count as usize
+        };
+        if self.inner.lock().bucket_entry_count < min_peer_count {
+            self.unlocked_inner.peer_minimum_refresh_task.tick().await?;
+        }
+        // Ping validate some nodes to groom the table
+        self.unlocked_inner.ping_validator_task.tick().await?;
+
+        Ok(())
+    }
+}

veilid-core/src/routing_table/node_ref.rs

@@ -0,0 +1,108 @@
+use super::*;
+use crate::dht::*;
+use alloc::fmt;
+
+pub struct NodeRef {
+    routing_table: RoutingTable,
+    node_id: DHTKey,
+    protocol_address_type: Option<ProtocolAddressType>,
+}
+
+impl NodeRef {
+    pub fn new(routing_table: RoutingTable, key: DHTKey, entry: &mut BucketEntry) -> Self {
+        entry.ref_count += 1;
+        Self {
+            routing_table: routing_table,
+            node_id: key,
+            protocol_address_type: None,
+        }
+    }
+    pub fn new_filtered(
+        routing_table: RoutingTable,
+        key: DHTKey,
+        entry: &mut BucketEntry,
+        protocol_address_type: ProtocolAddressType,
+    ) -> Self {
+        entry.ref_count += 1;
+        Self {
+            routing_table: routing_table,
+            node_id: key,
+            protocol_address_type: Some(protocol_address_type),
+        }
+    }
+
+    pub fn node_id(&self) -> DHTKey {
+        self.node_id
+    }
+
+    pub fn protocol_address_type(&self) -> Option<ProtocolAddressType> {
+        self.protocol_address_type
+    }
+
+    pub fn set_protocol_address_type(
+        &mut self,
+        protocol_address_type: Option<ProtocolAddressType>,
+    ) {
+        self.protocol_address_type = protocol_address_type;
+    }
+
+    pub fn operate<T, F>(&self, f: F) -> T
+    where
+        F: FnOnce(&mut BucketEntry) -> T,
+    {
+        self.routing_table.operate_on_bucket_entry(self.node_id, f)
+    }
+
+    pub fn dial_info(&self) -> Option<DialInfo> {
+        match self.protocol_address_type {
+            None => self.operate(|e| e.best_dial_info()),
+            Some(pat) => self.operate(|e| {
+                e.filtered_dial_info(|die| die.dial_info().protocol_address_type() == pat)
+            }),
+        }
+    }
+    pub fn last_connection(&self) -> Option<ConnectionDescriptor> {
+        match self.operate(|e| e.last_connection()) {
+            None => None,
+            Some(c) => {
+                if let Some(protocol_address_type) = self.protocol_address_type {
+                    if c.remote.protocol_address_type() == protocol_address_type {
+                        Some(c)
+                    } else {
+                        None
+                    }
+                } else {
+                    Some(c)
+                }
+            }
+        }
+    }
+}
+
+impl Clone for NodeRef {
+    fn clone(&self) -> Self {
+        self.operate(move |e| {
+            e.ref_count += 1;
+        });
+        Self {
+            routing_table: self.routing_table.clone(),
+            node_id: self.node_id.clone(),
+            protocol_address_type: self.protocol_address_type,
+        }
+    }
+}
+
+impl fmt::Debug for NodeRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.protocol_address_type {
+            None => write!(f, "{}", self.node_id.encode()),
+            Some(pat) => write!(f, "{}#{:?}", self.node_id.encode(), pat),
+        }
+    }
+}
+
+impl Drop for NodeRef {
+    fn drop(&mut self) {
+        self.routing_table.drop_node_ref(self.node_id);
+    }
+}

veilid-core/src/rpc_processor/coders/address.rs

@@ -0,0 +1,69 @@
+use crate::xx::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+pub fn encode_address(
+    address: &Address,
+    builder: &mut veilid_capnp::address::Builder,
+) -> Result<(), RPCError> {
+    match address {
+        Address::IPV4(v4) => {
+            let mut v4b = builder.reborrow().init_ipv4();
+            v4b.set_addr(u32::from_be_bytes(v4.octets()));
+        }
+        Address::IPV6(v6) => {
+            let mut v6b = builder.reborrow().init_ipv6();
+            v6b.set_addr0(u32::from_be_bytes(
+                v6.octets()[0..4]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr1(u32::from_be_bytes(
+                v6.octets()[4..8]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr2(u32::from_be_bytes(
+                v6.octets()[8..12]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr3(u32::from_be_bytes(
+                v6.octets()[12..16]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+        }
+        Address::Hostname(h) => {
+            let mut tb = builder.reborrow().init_hostname(
+                h.len()
+                    .try_into()
+                    .map_err(map_error_internal!("hostname too long"))?,
+            );
+            tb.push_str(h.as_str());
+        }
+    };
+    Ok(())
+}
+
+pub fn decode_address(reader: &veilid_capnp::address::Reader) -> Result<Address, RPCError> {
+    match reader.reborrow().which() {
+        Ok(veilid_capnp::address::Which::Ipv4(Ok(v4))) => {
+            let v4b = v4.get_addr().to_be_bytes();
+            Ok(Address::IPV4(Ipv4Addr::new(v4b[0], v4b[1], v4b[2], v4b[3])))
+        }
+        Ok(veilid_capnp::address::Which::Ipv6(Ok(v6))) => {
+            let v6b0 = v6.get_addr0().to_be_bytes();
+            let v6b1 = v6.get_addr1().to_be_bytes();
+            let v6b2 = v6.get_addr2().to_be_bytes();
+            let v6b3 = v6.get_addr3().to_be_bytes();
+            Ok(Address::IPV6(Ipv6Addr::from([
+                v6b0[0], v6b0[1], v6b0[2], v6b0[3], v6b1[0], v6b1[1], v6b1[2], v6b1[3], v6b2[0],
+                v6b2[1], v6b2[2], v6b2[3], v6b3[0], v6b3[1], v6b3[2], v6b3[3],
+            ])))
+        }
+        Ok(veilid_capnp::address::Which::Hostname(Ok(tr))) => Ok(Address::Hostname(tr.to_owned())),
+        _ => Err(rpc_error_internal("invalid address type")),
+    }
+}

veilid-core/src/rpc_processor/coders/dial_info.rs

@@ -0,0 +1,101 @@
+use crate::xx::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+pub fn decode_dial_info(reader: &veilid_capnp::dial_info::Reader) -> Result<DialInfo, RPCError> {
+    match reader.reborrow().which() {
+        Ok(veilid_capnp::dial_info::Which::Udp(Ok(udp))) => {
+            let address_reader = udp
+                .get_address()
+                .map_err(map_error_internal!("missing udp address"))?;
+            let address = decode_address(&address_reader)?;
+            let port = udp.get_port();
+            Ok(DialInfo::udp(address, port))
+        }
+        Ok(veilid_capnp::dial_info::Which::Tcp(Ok(tcp))) => {
+            let address_reader = tcp
+                .get_address()
+                .map_err(map_error_internal!("missing tcp address"))?;
+            let address = decode_address(&address_reader)?;
+            let port = tcp.get_port();
+            Ok(DialInfo::tcp(address, port))
+        }
+        Ok(veilid_capnp::dial_info::Which::Ws(Ok(ws))) => {
+            let fqdn = ws
+                .get_fqdn()
+                .map_err(map_error_internal!("missing ws fqdn"))?;
+            let port = ws.get_port();
+            let path = ws
+                .get_path()
+                .map_err(map_error_internal!("missing ws path"))?;
+            Ok(DialInfo::ws(fqdn.to_owned(), port, path.to_owned()))
+        }
+        Ok(veilid_capnp::dial_info::Which::Wss(Ok(wss))) => {
+            let fqdn = wss
+                .get_fqdn()
+                .map_err(map_error_internal!("missing wss fqdn"))?;
+            let port = wss.get_port();
+            let path = wss
+                .get_path()
+                .map_err(map_error_internal!("missing wss path"))?;
+            Ok(DialInfo::wss(fqdn.to_owned(), port, path.to_owned()))
+        }
+        _ => Err(rpc_error_internal("invalid dial info type")),
+    }
+}
+
+pub fn encode_dial_info(
+    dial_info: &DialInfo,
+    builder: &mut veilid_capnp::dial_info::Builder,
+) -> Result<(), RPCError> {
+    match dial_info {
+        DialInfo::UDP(udp) => {
+            let mut di_udp_builder = builder.reborrow().init_udp();
+            encode_address(&udp.address, &mut di_udp_builder.reborrow().init_address())?;
+            di_udp_builder.set_port(udp.port);
+        }
+        DialInfo::TCP(tcp) => {
+            let mut di_tcp_builder = builder.reborrow().init_tcp();
+            encode_address(&tcp.address, &mut di_tcp_builder.reborrow().init_address())?;
+            di_tcp_builder.set_port(tcp.port);
+        }
+        DialInfo::WS(ws) => {
+            let mut di_ws_builder = builder.reborrow().init_ws();
+            let mut fqdnb = di_ws_builder.reborrow().init_fqdn(
+                ws.fqdn
+                    .len()
+                    .try_into()
+                    .map_err(map_error_internal!("fqdn too long"))?,
+            );
+            fqdnb.push_str(ws.fqdn.as_str());
+            di_ws_builder.set_port(ws.port);
+            let mut pathb = di_ws_builder.init_path(
+                ws.path
+                    .len()
+                    .try_into()
+                    .map_err(map_error_internal!("path too long"))?,
+            );
+            pathb.push_str(ws.path.as_str());
+        }
+        DialInfo::WSS(wss) => {
+            let mut di_wss_builder = builder.reborrow().init_wss();
+            let mut fqdnb = di_wss_builder.reborrow().init_fqdn(
+                wss.fqdn
+                    .len()
+                    .try_into()
+                    .map_err(map_error_internal!("fqdn too long"))?,
+            );
+            fqdnb.push_str(wss.fqdn.as_str());
+            di_wss_builder.set_port(wss.port);
+            let mut pathb = di_wss_builder.init_path(
+                wss.path
+                    .len()
+                    .try_into()
+                    .map_err(map_error_internal!("path too long"))?,
+            );
+            pathb.push_str(wss.path.as_str());
+        }
+    };
+    Ok(())
+}

veilid-core/src/rpc_processor/coders/mod.rs

@@ -0,0 +1,21 @@
+mod address;
+mod dial_info;
+mod node_dial_info_single;
+mod node_info;
+mod nonce;
+mod peer_info;
+mod private_safety_route;
+mod public_key;
+mod sender_info;
+mod socket_address;
+
+pub use address::*;
+pub use dial_info::*;
+pub use node_dial_info_single::*;
+pub use node_info::*;
+pub use nonce::*;
+pub use peer_info::*;
+pub use private_safety_route::*;
+pub use public_key::*;
+pub use sender_info::*;
+pub use socket_address::*;

veilid-core/src/rpc_processor/coders/node_dial_info_single.rs

@@ -0,0 +1,29 @@
+use crate::*;
+use rpc_processor::*;
+
+pub fn encode_node_dial_info_single(
+    ndis: &NodeDialInfoSingle,
+    builder: &mut veilid_capnp::node_dial_info_single::Builder,
+) -> Result<(), RPCError> {
+    let mut ni_builder = builder.reborrow().init_node_id();
+    encode_public_key(&ndis.node_id.key, &mut ni_builder)?;
+    let mut di_builder = builder.reborrow().init_dial_info();
+    encode_dial_info(&ndis.dial_info, &mut di_builder)?;
+    Ok(())
+}
+
+pub fn decode_node_dial_info_single(
+    reader: &veilid_capnp::node_dial_info_single::Reader,
+) -> Result<NodeDialInfoSingle, RPCError> {
+    let node_id = decode_public_key(&reader.get_node_id().map_err(map_error_internal!(
+        "invalid public key in node_dial_info_single"
+    ))?);
+    let dial_info = decode_dial_info(&reader.get_dial_info().map_err(map_error_internal!(
+        "invalid dial_info in node_dial_info_single"
+    ))?)?;
+
+    Ok(NodeDialInfoSingle {
+        node_id: NodeId::new(node_id),
+        dial_info: dial_info,
+    })
+}

veilid-core/src/rpc_processor/coders/node_info.rs

@@ -0,0 +1,39 @@
+use crate::*;
+use rpc_processor::*;
+
+pub fn encode_node_info(
+    node_info: &NodeInfo,
+    builder: &mut veilid_capnp::node_info::Builder,
+) -> Result<(), RPCError> {
+    builder.set_can_route(node_info.can_route);
+    builder.set_will_route(node_info.will_route);
+
+    builder.set_can_tunnel(node_info.can_tunnel);
+    builder.set_will_tunnel(node_info.will_tunnel);
+
+    builder.set_can_signal_lease(node_info.can_signal_lease);
+    builder.set_will_signal_lease(node_info.will_signal_lease);
+
+    builder.set_can_relay_lease(node_info.can_relay_lease);
+    builder.set_will_relay_lease(node_info.will_relay_lease);
+
+    builder.set_can_validate_dial_info(node_info.can_validate_dial_info);
+    builder.set_will_validate_dial_info(node_info.will_validate_dial_info);
+
+    Ok(())
+}
+
+pub fn decode_node_info(reader: &veilid_capnp::node_info::Reader) -> Result<NodeInfo, RPCError> {
+    Ok(NodeInfo {
+        can_route: reader.reborrow().get_can_route(),
+        will_route: reader.reborrow().get_will_route(),
+        can_tunnel: reader.reborrow().get_can_tunnel(),
+        will_tunnel: reader.reborrow().get_will_tunnel(),
+        can_signal_lease: reader.reborrow().get_can_signal_lease(),
+        will_signal_lease: reader.reborrow().get_will_signal_lease(),
+        can_relay_lease: reader.reborrow().get_can_relay_lease(),
+        will_relay_lease: reader.reborrow().get_will_relay_lease(),
+        can_validate_dial_info: reader.reborrow().get_can_validate_dial_info(),
+        will_validate_dial_info: reader.reborrow().get_will_validate_dial_info(),
+    })
+}

veilid-core/src/rpc_processor/coders/nonce.rs

@@ -0,0 +1,33 @@
+use crate::*;
+use rpc_processor::*;
+
+pub fn encode_nonce(
+    nonce: &Nonce,
+    builder: &mut veilid_capnp::x_cha_cha20_poly1305_nonce::Builder,
+) {
+    builder.set_u0(u64::from_be_bytes(
+        nonce[0..8].try_into().expect("slice with incorrect length"),
+    ));
+    builder.set_u1(u64::from_be_bytes(
+        nonce[8..16]
+            .try_into()
+            .expect("slice with incorrect length"),
+    ));
+    builder.set_u2(u64::from_be_bytes(
+        nonce[16..24]
+            .try_into()
+            .expect("slice with incorrect length"),
+    ));
+}
+
+pub fn decode_nonce(reader: &veilid_capnp::x_cha_cha20_poly1305_nonce::Reader) -> Nonce {
+    let u0 = reader.get_u0().to_be_bytes();
+    let u1 = reader.get_u1().to_be_bytes();
+    let u2 = reader.get_u2().to_be_bytes();
+
+    [
+        u0[0], u0[1], u0[2], u0[3], u0[4], u0[5], u0[6], u0[7], // u0
+        u1[0], u1[1], u1[2], u1[3], u1[4], u1[5], u1[6], u1[7], // u1
+        u2[0], u2[1], u2[2], u2[3], u2[4], u2[5], u2[6], u2[7], // u2
+    ]
+}

veilid-core/src/rpc_processor/coders/peer_info.rs

@@ -0,0 +1,50 @@
+use crate::xx::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+pub fn encode_peer_info(
+    peer_info: &PeerInfo,
+    builder: &mut veilid_capnp::peer_info::Builder,
+) -> Result<(), RPCError> {
+    //
+    let mut nid_builder = builder.reborrow().init_node_id();
+    encode_public_key(&peer_info.node_id.key, &mut nid_builder)?;
+    let mut dil_builder = builder.reborrow().init_dial_info_list(
+        peer_info
+            .dial_infos
+            .len()
+            .try_into()
+            .map_err(map_error_internal!("too many dial infos in peer info"))?,
+    );
+
+    for idx in 0..peer_info.dial_infos.len() {
+        let mut di_builder = dil_builder.reborrow().get(idx as u32);
+        encode_dial_info(&peer_info.dial_infos[idx], &mut di_builder)?;
+    }
+    Ok(())
+}
+
+pub fn decode_peer_info(reader: &veilid_capnp::peer_info::Reader) -> Result<PeerInfo, RPCError> {
+    let nid_reader = reader
+        .reborrow()
+        .get_node_id()
+        .map_err(map_error_capnp_error!())?;
+    let dil_reader = reader
+        .reborrow()
+        .get_dial_info_list()
+        .map_err(map_error_capnp_error!())?;
+    let mut dial_infos = Vec::<DialInfo>::with_capacity(
+        dil_reader
+            .len()
+            .try_into()
+            .map_err(map_error_internal!("too many dial infos"))?,
+    );
+    for di in dil_reader.iter() {
+        dial_infos.push(decode_dial_info(&di)?)
+    }
+    Ok(PeerInfo {
+        node_id: NodeId::new(decode_public_key(&nid_reader)),
+        dial_infos: dial_infos,
+    })
+}

veilid-core/src/rpc_processor/coders/private_safety_route.rs

@@ -0,0 +1,212 @@
+use crate::xx::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#[derive(Clone, Debug)]
+pub struct RouteHopData {
+    pub nonce: Nonce,
+    pub blob: Vec<u8>,
+}
+
+#[derive(Clone, Debug)]
+pub struct RouteHop {
+    pub dial_info: NodeDialInfoSingle,
+    pub next_hop: Option<RouteHopData>,
+}
+
+#[derive(Clone, Debug)]
+pub struct PrivateRoute {
+    pub public_key: DHTKey,
+    pub hop_count: u8,
+    pub hops: Option<RouteHop>,
+}
+
+#[derive(Clone, Debug)]
+pub enum SafetyRouteHops {
+    Data(RouteHopData),
+    Private(PrivateRoute),
+}
+
+#[derive(Clone, Debug)]
+pub struct SafetyRoute {
+    pub public_key: DHTKey,
+    pub hop_count: u8,
+    pub hops: SafetyRouteHops,
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+pub fn encode_route_hop_data(
+    route_hop_data: &RouteHopData,
+    builder: &mut veilid_capnp::route_hop_data::Builder,
+) -> Result<(), RPCError> {
+    //
+    let mut nonce_builder = builder.reborrow().init_nonce();
+    encode_nonce(&route_hop_data.nonce, &mut nonce_builder);
+    let blob_builder = builder.reborrow().init_blob(
+        route_hop_data
+            .blob
+            .len()
+            .try_into()
+            .map_err(map_error_internal!("invalid blob length in route hop data"))?,
+    );
+    blob_builder.copy_from_slice(route_hop_data.blob.as_slice());
+    Ok(())
+}
+
+pub fn encode_route_hop(
+    route_hop: &RouteHop,
+    builder: &mut veilid_capnp::route_hop::Builder,
+) -> Result<(), RPCError> {
+    encode_node_dial_info_single(
+        &route_hop.dial_info,
+        &mut builder.reborrow().init_dial_info(),
+    )?;
+    if let Some(rhd) = &route_hop.next_hop {
+        let mut rhd_builder = builder.reborrow().init_next_hop();
+        encode_route_hop_data(&rhd, &mut rhd_builder)?;
+    }
+    Ok(())
+}
+
+pub fn encode_private_route(
+    private_route: &PrivateRoute,
+    builder: &mut veilid_capnp::private_route::Builder,
+) -> Result<(), RPCError> {
+    encode_public_key(
+        &private_route.public_key,
+        &mut builder.reborrow().init_public_key(),
+    )?;
+    builder.set_hop_count(private_route.hop_count);
+    if let Some(rh) = &private_route.hops {
+        let mut rh_builder = builder.reborrow().init_first_hop();
+        encode_route_hop(&rh, &mut rh_builder)?;
+    };
+
+    Ok(())
+}
+
+pub fn encode_safety_route(
+    safety_route: &SafetyRoute,
+    builder: &mut veilid_capnp::safety_route::Builder,
+) -> Result<(), RPCError> {
+    encode_public_key(
+        &safety_route.public_key,
+        &mut builder.reborrow().init_public_key(),
+    )?;
+    builder.set_hop_count(safety_route.hop_count);
+    let h_builder = builder.reborrow().init_hops();
+    match &safety_route.hops {
+        SafetyRouteHops::Data(rhd) => {
+            let mut rhd_builder = h_builder.init_data();
+            encode_route_hop_data(&rhd, &mut rhd_builder)?;
+        }
+        SafetyRouteHops::Private(pr) => {
+            let mut pr_builder = h_builder.init_private();
+            encode_private_route(&pr, &mut pr_builder)?;
+        }
+    };
+
+    Ok(())
+}
+
+pub fn decode_route_hop_data(
+    reader: &veilid_capnp::route_hop_data::Reader,
+) -> Result<RouteHopData, RPCError> {
+    let nonce = decode_nonce(
+        &reader
+            .reborrow()
+            .get_nonce()
+            .map_err(map_error_internal!("invalid nonce in route hop data"))?,
+    );
+
+    let blob = reader
+        .reborrow()
+        .get_blob()
+        .map_err(map_error_internal!("invalid blob in route hop data"))?
+        .to_vec();
+
+    Ok(RouteHopData {
+        nonce: nonce,
+        blob: blob,
+    })
+}
+
+pub fn decode_route_hop(reader: &veilid_capnp::route_hop::Reader) -> Result<RouteHop, RPCError> {
+    let dial_info = decode_node_dial_info_single(
+        &reader
+            .reborrow()
+            .get_dial_info()
+            .map_err(map_error_internal!("invalid dial info in route hop"))?,
+    )?;
+
+    let next_hop = if reader.has_next_hop() {
+        let rhd_reader = reader
+            .get_next_hop()
+            .map_err(map_error_internal!("invalid next hop in route hop"))?;
+        Some(decode_route_hop_data(&rhd_reader)?)
+    } else {
+        None
+    };
+
+    Ok(RouteHop {
+        dial_info: dial_info,
+        next_hop: next_hop,
+    })
+}
+
+pub fn decode_private_route(
+    reader: &veilid_capnp::private_route::Reader,
+) -> Result<PrivateRoute, RPCError> {
+    let public_key = decode_public_key(
+        &reader
+            .get_public_key()
+            .map_err(map_error_internal!("invalid public key in private route"))?,
+    );
+    let hop_count = reader.get_hop_count();
+    let hops = if reader.has_first_hop() {
+        let rh_reader = reader
+            .get_first_hop()
+            .map_err(map_error_internal!("invalid first hop in private route"))?;
+        Some(decode_route_hop(&rh_reader)?)
+    } else {
+        None
+    };
+
+    Ok(PrivateRoute {
+        public_key: public_key,
+        hop_count: hop_count,
+        hops: hops,
+    })
+}
+
+pub fn decode_safety_route(
+    reader: &veilid_capnp::safety_route::Reader,
+) -> Result<SafetyRoute, RPCError> {
+    let public_key = decode_public_key(
+        &reader
+            .get_public_key()
+            .map_err(map_error_internal!("invalid public key in safety route"))?,
+    );
+    let hop_count = reader.get_hop_count();
+    let hops = match reader.get_hops().which() {
+        Ok(veilid_capnp::safety_route::hops::Which::Data(Ok(rhd_reader))) => {
+            SafetyRouteHops::Data(decode_route_hop_data(&rhd_reader)?)
+        }
+        Ok(veilid_capnp::safety_route::hops::Which::Private(Ok(pr_reader))) => {
+            SafetyRouteHops::Private(decode_private_route(&pr_reader)?)
+        }
+        _ => {
+            return Err(rpc_error_internal("invalid hops in safety route"));
+        }
+    };
+
+    Ok(SafetyRoute {
+        public_key: public_key,
+        hop_count: hop_count,
+        hops: hops,
+    })
+}

veilid-core/src/rpc_processor/coders/public_key.rs

@@ -0,0 +1,49 @@
+use crate::dht::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+pub fn decode_public_key(public_key: &veilid_capnp::curve25519_public_key::Reader) -> key::DHTKey {
+    let u0 = public_key.get_u0().to_be_bytes();
+    let u1 = public_key.get_u1().to_be_bytes();
+    let u2 = public_key.get_u2().to_be_bytes();
+    let u3 = public_key.get_u3().to_be_bytes();
+
+    let mut x: [u8; 32] = Default::default();
+    x[0..8].copy_from_slice(&u0);
+    x[8..16].copy_from_slice(&u1);
+    x[16..24].copy_from_slice(&u2);
+    x[24..32].copy_from_slice(&u3);
+
+    key::DHTKey::new(x)
+}
+
+pub fn encode_public_key(
+    key: &key::DHTKey,
+    builder: &mut veilid_capnp::curve25519_public_key::Builder,
+) -> Result<(), RPCError> {
+    if !key.valid {
+        return Err(rpc_error_internal("invalid key"));
+    }
+    builder.set_u0(u64::from_be_bytes(
+        key.bytes[0..8]
+            .try_into()
+            .map_err(map_error_internal!("slice with incorrect length"))?,
+    ));
+    builder.set_u1(u64::from_be_bytes(
+        key.bytes[8..16]
+            .try_into()
+            .map_err(map_error_internal!("slice with incorrect length"))?,
+    ));
+    builder.set_u2(u64::from_be_bytes(
+        key.bytes[16..24]
+            .try_into()
+            .map_err(map_error_internal!("slice with incorrect length"))?,
+    ));
+    builder.set_u3(u64::from_be_bytes(
+        key.bytes[24..32]
+            .try_into()
+            .map_err(map_error_internal!("slice with incorrect length"))?,
+    ));
+    Ok(())
+}

veilid-core/src/rpc_processor/coders/sender_info.rs

@@ -0,0 +1,34 @@
+use crate::*;
+use rpc_processor::*;
+
+pub fn encode_sender_info(
+    sender_info: &SenderInfo,
+    builder: &mut veilid_capnp::sender_info::Builder,
+) -> Result<(), RPCError> {
+    if let Some(socket_address) = &sender_info.socket_address {
+        let mut sab = builder.reborrow().init_socket_address();
+        encode_socket_address(socket_address, &mut sab)?;
+    }
+    Ok(())
+}
+
+pub fn decode_sender_info(
+    reader: &veilid_capnp::sender_info::Reader,
+) -> Result<SenderInfo, RPCError> {
+    if !reader.has_socket_address() {
+        return Err(rpc_error_internal("invalid socket address type"));
+    }
+    let socket_address = if reader.has_socket_address() {
+        Some(decode_socket_address(
+            &reader
+                .reborrow()
+                .get_socket_address()
+                .map_err(map_error_internal!("invalid socket address in sender_info"))?,
+        )?)
+    } else {
+        None
+    };
+    Ok(SenderInfo {
+        socket_address: socket_address,
+    })
+}

veilid-core/src/rpc_processor/coders/socket_address.rs

@@ -0,0 +1,72 @@
+use crate::xx::*;
+use crate::*;
+use core::convert::TryInto;
+use rpc_processor::*;
+
+pub fn encode_socket_address(
+    address: &SocketAddr,
+    builder: &mut veilid_capnp::socket_address::Builder,
+) -> Result<(), RPCError> {
+    match address {
+        SocketAddr::V4(v4) => {
+            let mut v4b = builder.reborrow().init_ipv4();
+            v4b.set_addr(u32::from_be_bytes(v4.ip().octets()));
+            builder.set_port(v4.port());
+        }
+        SocketAddr::V6(v6) => {
+            let mut v6b = builder.reborrow().init_ipv6();
+            v6b.set_addr0(u32::from_be_bytes(
+                v6.ip().octets()[0..4]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr1(u32::from_be_bytes(
+                v6.ip().octets()[4..8]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr2(u32::from_be_bytes(
+                v6.ip().octets()[8..12]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            v6b.set_addr3(u32::from_be_bytes(
+                v6.ip().octets()[12..16]
+                    .try_into()
+                    .expect("slice with incorrect length"),
+            ));
+            builder.set_port(v6.port());
+        }
+    };
+    Ok(())
+}
+
+pub fn decode_socket_address(
+    reader: &veilid_capnp::socket_address::Reader,
+) -> Result<SocketAddr, RPCError> {
+    match reader.reborrow().which() {
+        Ok(veilid_capnp::socket_address::Which::Ipv4(Ok(v4))) => {
+            let v4b = v4.get_addr().to_be_bytes();
+            Ok(SocketAddr::V4(SocketAddrV4::new(
+                Ipv4Addr::new(v4b[0], v4b[1], v4b[2], v4b[3]),
+                reader.get_port(),
+            )))
+        }
+        Ok(veilid_capnp::socket_address::Which::Ipv6(Ok(v6))) => {
+            let v6b0 = v6.get_addr0().to_be_bytes();
+            let v6b1 = v6.get_addr1().to_be_bytes();
+            let v6b2 = v6.get_addr2().to_be_bytes();
+            let v6b3 = v6.get_addr3().to_be_bytes();
+            Ok(SocketAddr::V6(SocketAddrV6::new(
+                Ipv6Addr::from([
+                    v6b0[0], v6b0[1], v6b0[2], v6b0[3], v6b1[0], v6b1[1], v6b1[2], v6b1[3],
+                    v6b2[0], v6b2[1], v6b2[2], v6b2[3], v6b3[0], v6b3[1], v6b3[2], v6b3[3],
+                ]),
+                reader.get_port(),
+                0,
+                0,
+            )))
+        }
+        _ => Err(rpc_error_internal("invalid socket address type")),
+    }
+}

veilid-core/src/tests/.gitignore

@@ -0,0 +1,4 @@
+# exclude everything
+tmp/*
+# exception to the rule
+!tmp/.gitkeep 

veilid-core/src/tests/android/.gitignore

@@ -0,0 +1,16 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+/.idea/deploymentTargetDropDown.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

veilid-core/src/tests/android/.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

veilid-core/src/tests/android/.idea/.name

@@ -0,0 +1 @@
+VeilidCore Tests