Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-10-01 01:26:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'veilidchat-work' into 'main'

Browse Source 
Network Fixes (Breaking)

See merge request veilid/veilid!73
This commit is contained in:
Christien Rioux 2023-07-16 21:26:33 +00:00
commit 7d40735eec
151 changed files with 16055 additions and 5168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.bumpversion.cfg Normal file
View File

@ -0,0 +1,34 @@
[bumpversion]
current_version = 0.1.1
[bumpversion:file:veilid-server/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-flutter/rust/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-wasm/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-core/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-cli/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-tools/Cargo.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-python/pyproject.toml]
search = version = "{current_version}"
replace = version = "{new_version}"
[bumpversion:file:veilid-flutter/pubspec.yaml]
search = version: {current_version}
replace = version: {new_version}

1529
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

3
Cargo.toml
View File

@ -7,8 +7,7 @@ members = [
"veilid-flutter/rust",
"veilid-wasm",
]
exclude = [ "./external/keyring-manager", "./external/cursive", "./external/hashlink" ]
exclude = [ "./external" ]
[patch.crates-io]
cursive = { path = "./external/cursive/cursive" }

10
RELEASING.md
View File

@ -68,3 +68,13 @@ Occasionally a release will happen that needs to be reverted. This is done manua
> * Standalone RedHat/CentOS RPM file as a 'release file' on the `veilid` GitLab repository
> * Pushed to Yum repository at https://packages.veilid.com
### Version Numbering:
All versions of Veilid Rust crates as well as `veilid-python` and `veilid-flutter` packages are versioned using Semver. Versions can differ per crate and package, and it is important for the Semver rules to be followed (https://semver.org/):
* MAJOR version when you make incompatible API changes
* MINOR version when you add functionality in a backward compatible manner
* PATCH version when you make backward compatible bug fixes
The `version_bump.sh` script should be run on every release to stable. All of the Rust crates are versioned together and should have the same version, as well as the `veilid-python` Python package and `veilid-flutter` Flutter plugin.

2
external/cursive_buffered_backend vendored

@ -1 +1 @@
Subproject commit c4a7301b865d5af525fad30e76c2c5d121189943
Subproject commit 89d84d57e1880f2b0dbc37b44a0289d8c76858bd

11
setup_linux.sh
View File

@ -85,12 +85,23 @@ else
exit 1
fi
# ensure pip3 is installed
if command -v pip3 &> /dev/null; then
echo '[X] pip3 is available in the path'
else
echo 'pip3 is not available in the path'
exit 1
fi
# install targets
rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android wasm32-unknown-unknown
# install cargo packages
cargo install wasm-bindgen-cli wasm-pack
# install pip packages
pip3 install --upgrade bumpversion
# Install capnproto using the same mechanism as our earthly build
$SCRIPTDIR/scripts/earthly/install_capnproto.sh
# Install protoc using the same mechanism as our earthly build

13
setup_macos.sh
View File

@ -96,10 +96,18 @@ else
exit 1
fi
# ensure pip3 is installed
if command -v pip3 &> /dev/null; then
echo '[X] pip3 is available in the path'
else
echo 'pip3 is not available in the path'
exit 1
fi
# ensure we have command line tools
xcode-select --install 2> /dev/null || true
until [ -d /Library/Developer/CommandLineTools/usr/bin ]; do
sleep 5;
sleep 5;
done
# ensure packages are installed
@ -126,4 +134,7 @@ rustup target add aarch64-apple-darwin aarch64-apple-ios x86_64-apple-darwin x86
# install cargo packages
cargo install wasm-bindgen-cli wasm-pack
# install pip packages
pip3 install --upgrade bumpversion
sudo gem install cocoapods

2
veilid-cli/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "veilid-cli"
version = "0.1.0"
version = "0.1.1"
authors = ["John Smith <jsmith@example.com>"]
edition = "2021"
license = "LGPL-2.0-or-later OR MPL-2.0 OR (MIT AND BSD-3-Clause)"

147
veilid-core/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "veilid-core"
version = "0.1.0"
version = "0.1.1"
authors = ["John Smith <nobody@example.com>"]
edition = "2021"
build = "build.rs"
@ -31,100 +31,127 @@ veilid_core_android_tests = ["dep:paranoid-android"]
veilid_core_ios_tests = ["dep:tracing-oslog"]
network-result-extra = ["veilid-tools/network-result-extra"]
### DEPENDENCIES
[dependencies]
# Tools
veilid-tools = { path = "../veilid-tools", features = [ "tracing" ] }
paste = "1.0.12"
once_cell = "^1"
owning_ref = "^0"
backtrace = { version = "^0" }
num-traits = "0.2.15"
shell-words = "1.1.0"
static_assertions = "^1"
cfg-if = "^1"
hex = "^0"
lazy_static = "^1"
directories = "^4"
# Logging
tracing = { version = "^0", features = ["log", "attributes"] }
tracing-subscriber = "^0"
tracing-error = "^0"
eyre = "^0"
capnp = { version = "^0", default_features = false }
static_assertions = "^1"
cfg-if = "^1"
thiserror = "^1"
hex = "^0"
generic-array = "^0"
secrecy = "^0"
chacha20poly1305 = "^0"
chacha20 = "^0"
hashlink = { path = "../external/hashlink", features = ["serde_impl"] }
serde = { version = "^1", features = ["derive" ] }
serde_json = { version = "^1" }
serde-big-array = "^0"
futures-util = { version = "^0", default_features = false, features = ["alloc"] }
parking_lot = "^0"
lazy_static = "^1"
directories = "^4"
once_cell = "^1"
json = "^0"
owning_ref = "^0"
flume = { version = "^0", features = ["async"] }
enumset = { version= "^1", features = ["serde"] }
backtrace = { version = "^0" }
stop-token = { version = "^0", default-features = false }
num-traits = "0.2.15"
shell-words = "1.1.0"
# Data structures
enumset = { version= "^1", features = ["serde"] }
keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" }
range-set-blaze = "0.1.5"
weak-table = "0.3.2"
generic-array = "^0"
hashlink = { path = "../external/hashlink", features = ["serde_impl"] }
# System
futures-util = { version = "^0", default_features = false, features = ["alloc"] }
flume = { version = "^0", features = ["async"] }
parking_lot = "^0"
stop-token = { version = "^0", default-features = false }
# Crypto
ed25519-dalek = { version = "^1", default_features = false, features = ["alloc", "u64_backend"] }
x25519-dalek = { version = "^1", default_features = false, features = ["u64_backend"] }
curve25519-dalek = { version = "^3", default_features = false, features = ["alloc", "u64_backend"] }
blake3 = { version = "^1" }
chacha20poly1305 = "^0"
chacha20 = "^0"
argon2 = "0.5.0"
# Network
async-std-resolver = { version = "^0", optional = true }
trust-dns-resolver = { version = "^0", optional = true }
enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0
keyvaluedb = { path = "../external/keyvaluedb/keyvaluedb" }
rkyv = { version = "^0", default_features = false, features = ["std", "alloc", "strict", "size_32", "validation"] }
# Serialization
capnp = { version = "^0", default_features = false }
serde = { version = "^1", features = ["derive" ] }
serde_json = { version = "^1" }
serde-big-array = "^0"
json = "^0"
data-encoding = { version = "^2" }
weak-table = "0.3.2"
range-set-blaze = "0.1.5"
argon2 = "0.5.0"
paste = "1.0.12"
schemars = "0.8.12"
lz4_flex = { version = "0.11.1", default-features = false, features = ["safe-encode", "safe-decode"] }
# Dependencies for native builds only
# Linux, Windows, Mac, iOS, Android
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
async-std = { version = "^1", features = ["unstable"], optional = true}
tokio = { version = "^1", features = ["full"], optional = true}
tokio-util = { version = "^0", features = ["compat"], optional = true}
tokio-stream = { version = "^0", features = ["net"], optional = true}
async-io = { version = "^1" }
async-tungstenite = { version = "^0", features = ["async-tls"] }
maplit = "^1"
# Tools
config = { version = "^0", features = ["yaml"] }
keyring-manager = { path = "../external/keyring-manager" }
async-tls = "^0.11"
igd = { path = "../external/rust-igd" }
webpki = "^0"
webpki-roots = "^0"
rustls = "^0.19"
rustls-pemfile = "^0.2"
futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
socket2 = { version = "^0", features = ["all"] }
bugsalot = { git = "https://github.com/crioux/bugsalot.git" }
chrono = "^0"
libc = "^0"
nix = "^0"
# System
async-std = { version = "^1", features = ["unstable"], optional = true}
tokio = { version = "^1", features = ["full"], optional = true}
tokio-util = { version = "^0", features = ["compat"], optional = true}
tokio-stream = { version = "^0", features = ["net"], optional = true}
async-io = { version = "^1" }
futures-util = { version = "^0", default-features = false, features = ["async-await", "sink", "std", "io"] }
# Data structures
keyring-manager = { path = "../external/keyring-manager" }
keyvaluedb-sqlite = { path = "../external/keyvaluedb/keyvaluedb-sqlite" }
# Network
async-tungstenite = { version = "^0", features = ["async-tls"] }
igd = { path = "../external/rust-igd" }
async-tls = "^0.11"
webpki = "^0"
webpki-roots = "^0"
rustls = "^0.19"
rustls-pemfile = "^0.2"
socket2 = { version = "^0", features = ["all"] }
# Dependencies for WASM builds only
[target.'cfg(target_arch = "wasm32")'.dependencies]
# Tools
getrandom = { version = "^0", features = ["js"] }
# System
async_executors = { version = "^0", default-features = false, features = [ "bindgen", "timer" ]}
async-lock = "^2"
wasm-bindgen = "^0"
js-sys = "^0"
wasm-bindgen-futures = "^0"
keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
getrandom = { version = "^0", features = ["js"] }
ws_stream_wasm = "^0"
async_executors = { version = "^0", default-features = false, features = [ "bindgen", "timer" ]}
async-lock = "^2"
send_wrapper = { version = "^0", features = ["futures"] }
# Network
ws_stream_wasm = "^0"
# Logging
wasm-logger = "^0"
tracing-wasm = "^0"
# Configuration for WASM32 'web-sys' crate
# Data Structures
keyvaluedb-web = { path = "../external/keyvaluedb/keyvaluedb-web" }
### Configuration for WASM32 'web-sys' crate
[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
version = "^0"
features = [
@ -154,9 +181,9 @@ ifstructs = "^0"
# Dependencies for Linux or Android
[target.'cfg(any(target_os = "android", target_os = "linux"))'.dependencies]
rtnetlink = { version = "^0", default-features = false}
netlink-sys = { version = "0.8" }
netlink-packet-route = { version = "0.15" }
rtnetlink = { version = "=0.13.0", default-features = false}
netlink-sys = { version = "=0.8.5" }
netlink-packet-route = { version = "=0.17.0" }
# Dependencies for Windows
[target.'cfg(target_os = "windows")'.dependencies]

23
veilid-core/src/crypto/envelope.rs
View File

@ -213,6 +213,10 @@ impl Envelope {
&dh_secret,
);
// Decompress body
let body = decompress_size_prepended(&body)
.map_err(|e| VeilidAPIError::parse_error("failed to decompress", e))?;
Ok(body)
}
@ -223,10 +227,25 @@ impl Envelope {
node_id_secret: &SecretKey,
network_key: &Option<SharedSecret>,
) -> VeilidAPIResult<Vec<u8>> {
// Ensure body isn't too long
let uncompressed_body_size: usize = body.len() + MIN_ENVELOPE_SIZE;
if uncompressed_body_size > MAX_ENVELOPE_SIZE {
apibail_parse_error!(
"envelope size before compression is too large",
uncompressed_body_size
);
}
// Compress body
let body = compress_prepend_size(&body);
// Ensure body isn't too long
let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE;
if envelope_size > MAX_ENVELOPE_SIZE {
apibail_parse_error!("envelope size is too large", envelope_size);
apibail_parse_error!(
"envelope size after compression is too large",
envelope_size
);
}
// Generate dh secret
let vcrypto = crypto
@ -271,7 +290,7 @@ impl Envelope {
}
// Encrypt and authenticate message
let encrypted_body = vcrypto.crypt_no_auth_unaligned(body, &self.nonce.bytes, &dh_secret);
let encrypted_body = vcrypto.crypt_no_auth_unaligned(&body, &self.nonce.bytes, &dh_secret);
// Write body
if !encrypted_body.is_empty() {

3
veilid-core/src/crypto/types/byte_array_types.rs
View File

@ -77,8 +77,7 @@ where
macro_rules! byte_array_type {
($name:ident, $size:expr, $encoded_size:expr) => {
#[derive(Clone, Copy, Hash, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, Eq, PartialEq, PartialOrd, Ord))]
#[derive(Clone, Copy, Hash)]
pub struct $name {
pub bytes: [u8; $size],
}

19
veilid-core/src/crypto/types/crypto_typed.rs
View File

@ -1,7 +1,6 @@
use super::*;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CryptoTyped<K>
where
K: Clone
@ -14,9 +13,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub kind: CryptoKind,
pub value: K,
@ -34,9 +31,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub fn new(kind: CryptoKind, value: K) -> Self {
Self { kind, value }
@ -54,9 +49,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
Some(self.cmp(other))
@ -75,9 +68,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn cmp(&self, other: &Self) -> cmp::Ordering {
let x = compare_crypto_kind(&self.kind, &other.kind);
@ -100,9 +91,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
write!(f, "{}:{}", self.kind, self.value)
@ -120,9 +109,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Err = VeilidAPIError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
@ -152,9 +139,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
@ -176,9 +161,7 @@ where
+ Ord
+ PartialOrd
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where

36
veilid-core/src/crypto/types/crypto_typed_group.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Debug,
Serialize,
Deserialize,
PartialOrd,
Ord,
PartialEq,
Eq,
Hash,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
Default,
)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, Hash, Default)]
#[serde(from = "Vec<CryptoTyped<K>>", into = "Vec<CryptoTyped<K>>")]
pub struct CryptoTypedGroup<K = PublicKey>
where
@ -29,10 +14,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
<Vec<CryptoTyped<K>> as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
items: Vec<CryptoTyped<K>>,
}
@ -49,9 +31,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
pub fn new() -> Self {
Self { items: Vec::new() }
@ -163,9 +143,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Target = [CryptoTyped<K>];
@ -187,9 +165,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
write!(f, "[")?;
@ -217,9 +193,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
type Err = VeilidAPIError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
@ -250,9 +224,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: CryptoTyped<K>) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(1);
@ -272,9 +244,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: Vec<CryptoTyped<K>>) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(x.len());
@ -294,9 +264,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn from(x: &[CryptoTyped<K>]) -> Self {
let mut tks = CryptoTypedGroup::<K>::with_capacity(x.len());
@ -316,9 +284,7 @@ where
+ PartialOrd
+ Ord
+ Hash
+ RkyvArchive
+ Encodable,
<K as RkyvArchive>::Archived: Hash + PartialEq + Eq,
{
fn into(self) -> Vec<CryptoTyped<K>> {
self.items

15
veilid-core/src/crypto/types/keypair.rs
View File

@ -1,19 +1,6 @@
use super::*;
#[derive(
Clone,
Copy,
Default,
PartialOrd,
Ord,
PartialEq,
Eq,
Hash,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes, Hash, PartialEq, Eq))]
#[derive(Clone, Copy, Default, PartialOrd, Ord, PartialEq, Eq, Hash)]
pub struct KeyPair {
pub key: PublicKey,
pub secret: SecretKey,

31
veilid-core/src/intf/native/protected_store.rs
View File

@ -141,16 +141,6 @@ impl ProtectedStore {
}
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_json<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
@ -161,27 +151,6 @@ impl ProtectedStore {
self.save_user_secret(&key, &v).await
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_rkyv<K, T>(&self, key: K) -> EyreResult<Option<T>>
where
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = from_rkyv(b)?;
Ok(Some(obj))
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_json<K, T>(&self, key: K) -> EyreResult<Option<T>>
where

35
veilid-core/src/intf/wasm/protected_store.rs
View File

@ -1,9 +1,5 @@
use super::*;
use data_encoding::BASE64URL_NOPAD;
use rkyv::{
bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize,
Serialize as RkyvSerialize,
};
use web_sys::*;
@ -126,16 +122,6 @@ impl ProtectedStore {
}
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self, value))]
pub async fn save_user_secret_json<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
@ -146,27 +132,6 @@ impl ProtectedStore {
self.save_user_secret(key, &v).await
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_rkyv<K, T>(&self, key: K) -> EyreResult<Option<T>>
where
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = from_rkyv(b)?;
Ok(Some(obj))
}
#[instrument(level = "trace", skip(self))]
pub async fn load_user_secret_json<K, T>(&self, key: K) -> EyreResult<Option<T>>
where

12
veilid-core/src/lib.rs
View File

@ -88,16 +88,12 @@ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [
use cfg_if::*;
use enumset::*;
use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr};
use parking_lot::*;
use rkyv::{
bytecheck, bytecheck::CheckBytes, de::deserializers::SharedDeserializeMap, with::Skip,
Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize,
};
use tracing::*;
use veilid_tools::*;
type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>;
use futures_util::stream::FuturesUnordered;
use lz4_flex::block::{compress_prepend_size, decompress_size_prepended};
use parking_lot::*;
use schemars::{schema_for, JsonSchema};
use serde::*;
use stop_token::*;
use thiserror::Error as ThisError;
use tracing::*;
use veilid_tools::*;

92
veilid-core/src/network_manager/address_filter.rs
View File

@ -3,6 +3,7 @@ use alloc::collections::btree_map::Entry;
// XXX: Move to config eventually?
const PUNISHMENT_DURATION_MIN: usize = 60;
const MAX_PUNISHMENTS_BY_NODE_ID: usize = 65536;
#[derive(ThisError, Debug, Clone, Copy, PartialEq, Eq)]
pub enum AddressFilterError {
@ -26,15 +27,37 @@ struct AddressFilterInner {
conn_timestamps_by_ip6_prefix: BTreeMap<Ipv6Addr, Vec<Timestamp>>,
punishments_by_ip4: BTreeMap<Ipv4Addr, Timestamp>,
punishments_by_ip6_prefix: BTreeMap<Ipv6Addr, Timestamp>,
punishments_by_node_id: BTreeMap<TypedKey, Timestamp>,
}
#[derive(Debug)]
struct AddressFilterUnlockedInner {
max_connections_per_ip4: usize,
max_connections_per_ip6_prefix: usize,
max_connections_per_ip6_prefix_size: usize,
max_connection_frequency_per_min: usize,
punishment_duration_min: usize,
routing_table: RoutingTable,
}
impl fmt::Debug for AddressFilterUnlockedInner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AddressFilterUnlockedInner")
.field("max_connections_per_ip4", &self.max_connections_per_ip4)
.field(
"max_connections_per_ip6_prefix",
&self.max_connections_per_ip6_prefix,
)
.field(
"max_connections_per_ip6_prefix_size",
&self.max_connections_per_ip6_prefix_size,
)
.field(
"max_connection_frequency_per_min",
&self.max_connection_frequency_per_min,
)
.field("punishment_duration_min", &self.punishment_duration_min)
.finish()
}
}
#[derive(Clone, Debug)]
@ -44,7 +67,7 @@ pub struct AddressFilter {
}
impl AddressFilter {
pub fn new(config: VeilidConfig) -> Self {
pub fn new(config: VeilidConfig, routing_table: RoutingTable) -> Self {
let c = config.get();
Self {
unlocked_inner: Arc::new(AddressFilterUnlockedInner {
@ -55,6 +78,7 @@ impl AddressFilter {
max_connection_frequency_per_min: c.network.max_connection_frequency_per_min
as usize,
punishment_duration_min: PUNISHMENT_DURATION_MIN,
routing_table,
}),
inner: Arc::new(Mutex::new(AddressFilterInner {
conn_count_by_ip4: BTreeMap::new(),
@ -63,6 +87,7 @@ impl AddressFilter {
conn_timestamps_by_ip6_prefix: BTreeMap::new(),
punishments_by_ip4: BTreeMap::new(),
punishments_by_ip6_prefix: BTreeMap::new(),
punishments_by_node_id: BTreeMap::new(),
})),
}
}
@ -135,9 +160,29 @@ impl AddressFilter {
inner.punishments_by_ip6_prefix.remove(&key);
}
}
// node id
{
let mut dead_keys = Vec::<TypedKey>::new();
for (key, value) in &mut inner.punishments_by_node_id {
// Drop punishments older than the punishment duration
if cur_ts.as_u64().saturating_sub(value.as_u64())
> self.unlocked_inner.punishment_duration_min as u64 * 60_000_000u64
{
dead_keys.push(*key);
}
}
for key in dead_keys {
log_net!(debug ">>> FORGIVING: {}", key);
inner.punishments_by_node_id.remove(&key);
// make the entry alive again if it's still here
if let Ok(Some(nr)) = self.unlocked_inner.routing_table.lookup_node_ref(key) {
nr.operate_mut(|_rti, e| e.set_punished(false));
}
}
}
}
fn is_punished_inner(&self, inner: &AddressFilterInner, ipblock: IpAddr) -> bool {
fn is_ip_addr_punished_inner(&self, inner: &AddressFilterInner, ipblock: IpAddr) -> bool {
match ipblock {
IpAddr::V4(v4) => {
if inner.punishments_by_ip4.contains_key(&v4) {
@ -153,16 +198,16 @@ impl AddressFilter {
false
}
pub fn is_punished(&self, addr: IpAddr) -> bool {
pub fn is_ip_addr_punished(&self, addr: IpAddr) -> bool {
let inner = self.inner.lock();
let ipblock = ip_to_ipblock(
self.unlocked_inner.max_connections_per_ip6_prefix_size,
addr,
);
self.is_punished_inner(&*inner, ipblock)
self.is_ip_addr_punished_inner(&*inner, ipblock)
}
pub fn punish(&self, addr: IpAddr) {
pub fn punish_ip_addr(&self, addr: IpAddr) {
log_net!(debug ">>> PUNISHED: {}", addr);
let ts = get_aligned_timestamp();
@ -186,6 +231,39 @@ impl AddressFilter {
};
}
fn is_node_id_punished_inner(&self, inner: &AddressFilterInner, node_id: TypedKey) -> bool {
if inner.punishments_by_node_id.contains_key(&node_id) {
return true;
}
false
}
pub fn is_node_id_punished(&self, node_id: TypedKey) -> bool {
let inner = self.inner.lock();
self.is_node_id_punished_inner(&*inner, node_id)
}
pub fn punish_node_id(&self, node_id: TypedKey) {
if let Ok(Some(nr)) = self.unlocked_inner.routing_table.lookup_node_ref(node_id) {
// make the entry dead if it's punished
nr.operate_mut(|_rti, e| e.set_punished(true));
}
let ts = get_aligned_timestamp();
let mut inner = self.inner.lock();
if inner.punishments_by_node_id.len() >= MAX_PUNISHMENTS_BY_NODE_ID {
log_net!(debug ">>> PUNISHMENT TABLE FULL: {}", node_id);
return;
}
log_net!(debug ">>> PUNISHED: {}", node_id);
inner
.punishments_by_node_id
.entry(node_id)
.and_modify(|v| *v = ts)
.or_insert(ts);
}
pub async fn address_filter_task_routine(
self,
_stop_token: StopToken,
@ -207,7 +285,7 @@ impl AddressFilter {
self.unlocked_inner.max_connections_per_ip6_prefix_size,
addr,
);
if self.is_punished_inner(inner, ipblock) {
if self.is_ip_addr_punished_inner(inner, ipblock) {
return Err(AddressFilterError::Punished);
}

119
veilid-core/src/network_manager/connection_manager.rs
View File

@ -173,6 +173,7 @@ impl ConnectionManager {
Ok(Some(conn)) => {
// Connection added and a different one LRU'd out
// Send it to be terminated
log_net!(debug "== LRU kill connection due to limit: {:?}", conn);
let _ = inner.sender.send(ConnectionManagerEvent::Dead(conn));
}
Err(ConnectionTableAddError::AddressFilter(conn, e)) => {
@ -205,40 +206,6 @@ impl ConnectionManager {
.get_connection_by_descriptor(descriptor)
}
// Terminate any connections that would collide with a new connection
// using different protocols to the same remote address and port. Used to ensure
// that we can switch quickly between TCP and WS if necessary to the same node
// Returns true if we killed off colliding connections
async fn kill_off_colliding_connections(&self, dial_info: &DialInfo) -> bool {
let protocol_type = dial_info.protocol_type();
let socket_address = dial_info.socket_address();
let killed = self.arc.connection_table.drain_filter(|prior_descriptor| {
// If the protocol types aren't the same, then this is a candidate to be killed off
// If they are the same, then we would just return the exact same connection from get_or_create_connection()
if prior_descriptor.protocol_type() == protocol_type {
return false;
}
// If the prior remote is not the same address, then we're not going to collide
if *prior_descriptor.remote().socket_address() != socket_address {
return false;
}
log_net!(debug
">< Terminating connection prior_descriptor={:?}",
prior_descriptor
);
true
});
// Wait for the killed connections to end their recv loops
let did_kill = !killed.is_empty();
for mut k in killed {
k.close();
k.await;
}
did_kill
}
/// Called when we want to create a new connection or get the current one that already exists
/// This will kill off any connections that are in conflict with the new connection to be made
/// in order to make room for the new connection in the system's connection table
@ -246,45 +213,53 @@ impl ConnectionManager {
#[instrument(level = "trace", skip(self), ret, err)]
pub async fn get_or_create_connection(
&self,
local_addr: Option<SocketAddr>,
dial_info: DialInfo,
) -> EyreResult<NetworkResult<ConnectionHandle>> {
// Async lock on the remote address for atomicity per remote
let peer_address = dial_info.to_peer_address();
let remote_addr = peer_address.to_socket_addr();
let mut preferred_local_address = self
.network_manager()
.net()
.get_preferred_local_address(&dial_info);
let best_port = preferred_local_address.map(|pla| pla.port());
// Async lock on the remote address for atomicity per remote
let _lock_guard = self.arc.address_lock_table.lock_tag(remote_addr).await;
log_net!(
"== get_or_create_connection local_addr={:?} dial_info={:?}",
local_addr,
dial_info
);
// Kill off any possibly conflicting connections
let did_kill = self.kill_off_colliding_connections(&dial_info).await;
let mut retry_count = if did_kill { 2 } else { 0 };
log_net!("== get_or_create_connection dial_info={:?}", dial_info);
// If any connection to this remote exists that has the same protocol, return it
// Any connection will do, we don't have to match the local address
if let Some(conn) = self
// Any connection will do, we don't have to match the local address but if we can
// match the preferred port do it
if let Some(best_existing_conn) = self
.arc
.connection_table
.get_last_connection_by_remote(peer_address)
.get_best_connection_by_remote(best_port, peer_address)
{
log_net!(
"== Returning existing connection local_addr={:?} peer_address={:?}",
local_addr,
peer_address
"== Returning best existing connection {:?}",
best_existing_conn
);
return Ok(NetworkResult::Value(conn));
return Ok(NetworkResult::Value(best_existing_conn));
}
// If there is a low-level connection collision here, then we release the 'preferred local address'
// so we can make a second connection with an ephemeral port
if self
.arc
.connection_table
.check_for_colliding_connection(&dial_info)
{
preferred_local_address = None;
}
// Attempt new connection
let mut retry_count = 0; // Someday, if we need this
let prot_conn = network_result_try!(loop {
let result_net_res = ProtocolNetworkConnection::connect(
local_addr,
preferred_local_address,
&dial_info,
self.arc.connection_initial_timeout_ms,
self.network_manager().address_filter(),
@ -292,24 +267,28 @@ impl ConnectionManager {
.await;
match result_net_res {
Ok(net_res) => {
// If the connection 'already exists', then try one last time to return a connection from the table, in case
// an 'accept' happened at literally the same time as our connect
if net_res.is_already_exists() {
if let Some(conn) = self
.arc
.connection_table
.get_last_connection_by_remote(peer_address)
{
log_net!(
"== Returning existing connection in race local_addr={:?} peer_address={:?}",
local_addr,
peer_address
);
return Ok(NetworkResult::Value(conn));
}
}
// // If the connection 'already exists', then try one last time to return a connection from the table, in case
// // an 'accept' happened at literally the same time as our connect. A preferred local address must have been
// // specified otherwise we would have picked a different ephemeral port and this could not have happened
// if net_res.is_already_exists() && preferred_local_address.is_some() {
// // Make 'already existing' connection descriptor
// let conn_desc = ConnectionDescriptor::new(
// dial_info.to_peer_address(),
// SocketAddress::from_socket_addr(preferred_local_address.unwrap()),
// );
// // Return the connection for this if we have it
// if let Some(conn) = self
// .arc
// .connection_table
// .get_connection_by_descriptor(conn_desc)
// {
// // Should not really happen, lets make sure we see this if it does
// log_net!(warn "== Returning existing connection in race: {:?}", conn_desc);
// return Ok(NetworkResult::Value(conn));
// }
// }
if net_res.is_value() || retry_count == 0 {
// Successful new connection, return it
break net_res;
}
}

109
veilid-core/src/network_manager/connection_table.rs
View File

@ -92,6 +92,37 @@ impl ConnectionTable {
while unord.next().await.is_some() {}
}
// Return true if there is another connection in the table using a different protocol type
// to the same address and port with the same low level protocol type.
// Specifically right now this checks for a TCP connection that exists to the same
// low level TCP remote as a WS or WSS connection, since they are all low-level TCP
#[instrument(level = "trace", skip(self), ret)]
pub fn check_for_colliding_connection(&self, dial_info: &DialInfo) -> bool {
let inner = self.inner.lock();
let protocol_type = dial_info.protocol_type();
let low_level_protocol_type = protocol_type.low_level_protocol_type();
// check protocol types
let mut check_protocol_types = ProtocolTypeSet::empty();
for check_pt in ProtocolTypeSet::all().iter() {
if check_pt != protocol_type
&& check_pt.low_level_protocol_type() == low_level_protocol_type
{
check_protocol_types.insert(check_pt);
}
}
let socket_address = dial_info.socket_address();
for check_pt in check_protocol_types {
let check_pa = PeerAddress::new(socket_address, check_pt);
if inner.ids_by_remote.contains_key(&check_pa) {
return true;
}
}
false
}
#[instrument(level = "trace", skip(self), ret, err)]
pub fn add_connection(
&self,
@ -183,14 +214,42 @@ impl ConnectionTable {
Some(out.get_handle())
}
//#[instrument(level = "trace", skip(self), ret)]
pub fn get_last_connection_by_remote(&self, remote: PeerAddress) -> Option<ConnectionHandle> {
// #[instrument(level = "trace", skip(self), ret)]
pub fn get_best_connection_by_remote(
&self,
best_port: Option<u16>,
remote: PeerAddress,
) -> Option<ConnectionHandle> {
let mut inner = self.inner.lock();
let id = inner.ids_by_remote.get(&remote).map(|v| v[v.len() - 1])?;
let all_ids_by_remote = inner.ids_by_remote.get(&remote)?;
let protocol_index = Self::protocol_to_index(remote.protocol_type());
let out = inner.conn_by_id[protocol_index].get(&id).unwrap();
Some(out.get_handle())
if all_ids_by_remote.len() == 0 {
// no connections
return None;
}
if all_ids_by_remote.len() == 1 {
// only one connection
let id = all_ids_by_remote[0];
let nc = inner.conn_by_id[protocol_index].get(&id).unwrap();
return Some(nc.get_handle());
}
// multiple connections, find the one that matches the best port, or the most recent
if let Some(best_port) = best_port {
for id in all_ids_by_remote.iter().copied() {
let nc = inner.conn_by_id[protocol_index].peek(&id).unwrap();
if let Some(local_addr) = nc.connection_descriptor().local() {
if local_addr.port() == best_port {
let nc = inner.conn_by_id[protocol_index].get(&id).unwrap();
return Some(nc.get_handle());
}
}
}
}
// just return most recent network connection if a best port match can not be found
let best_id = *all_ids_by_remote.last().unwrap();
let nc = inner.conn_by_id[protocol_index].get(&best_id).unwrap();
Some(nc.get_handle())
}
//#[instrument(level = "trace", skip(self), ret)]
@ -204,26 +263,26 @@ impl ConnectionTable {
.unwrap_or_default()
}
pub fn drain_filter<F>(&self, mut filter: F) -> Vec<NetworkConnection>
where
F: FnMut(ConnectionDescriptor) -> bool,
{
let mut inner = self.inner.lock();
let mut filtered_ids = Vec::new();
for cbi in &mut inner.conn_by_id {
for (id, conn) in cbi {
if filter(conn.connection_descriptor()) {
filtered_ids.push(*id);
}
}
}
let mut filtered_connections = Vec::new();
for id in filtered_ids {
let conn = Self::remove_connection_records(&mut *inner, id);
filtered_connections.push(conn)
}
filtered_connections
}
// pub fn drain_filter<F>(&self, mut filter: F) -> Vec<NetworkConnection>
// where
// F: FnMut(ConnectionDescriptor) -> bool,
// {
// let mut inner = self.inner.lock();
// let mut filtered_ids = Vec::new();
// for cbi in &mut inner.conn_by_id {
// for (id, conn) in cbi {
// if filter(conn.connection_descriptor()) {
// filtered_ids.push(*id);
// }
// }
// }
// let mut filtered_connections = Vec::new();
// for id in filtered_ids {
// let conn = Self::remove_connection_records(&mut *inner, id);
// filtered_connections.push(conn)
// }
// filtered_connections
// }
pub fn connection_count(&self) -> usize {
let inner = self.inner.lock();

52
veilid-core/src/network_manager/mod.rs
View File

@ -36,12 +36,16 @@ use hashlink::LruCache;
use intf::*;
#[cfg(not(target_arch = "wasm32"))]
use native::*;
#[cfg(not(target_arch = "wasm32"))]
pub use native::{LOCAL_NETWORK_CAPABILITIES, MAX_CAPABILITIES, PUBLIC_INTERNET_CAPABILITIES};
use receipt_manager::*;
use routing_table::*;
use rpc_processor::*;
use storage_manager::*;
#[cfg(target_arch = "wasm32")]
use wasm::*;
#[cfg(target_arch = "wasm32")]
pub use wasm::{LOCAL_NETWORK_CAPABILITIES, MAX_CAPABILITIES, PUBLIC_INTERNET_CAPABILITIES};
////////////////////////////////////////////////////////////////////////////////////////
@ -139,9 +143,9 @@ struct NetworkManagerUnlockedInner {
#[cfg(feature = "unstable-blockstore")]
block_store: BlockStore,
crypto: Crypto,
address_filter: AddressFilter,
// Accessors
routing_table: RwLock<Option<RoutingTable>>,
address_filter: RwLock<Option<AddressFilter>>,
components: RwLock<Option<NetworkComponents>>,
update_callback: RwLock<Option<UpdateCallback>>,
// Background processes
@ -185,7 +189,7 @@ impl NetworkManager {
#[cfg(feature = "unstable-blockstore")]
block_store,
crypto,
address_filter: AddressFilter::new(config),
address_filter: RwLock::new(None),
routing_table: RwLock::new(None),
components: RwLock::new(None),
update_callback: RwLock::new(None),
@ -288,7 +292,12 @@ impl NetworkManager {
self.unlocked_inner.crypto.clone()
}
pub fn address_filter(&self) -> AddressFilter {
self.unlocked_inner.address_filter.clone()
self.unlocked_inner
.address_filter
.read()
.as_ref()
.unwrap()
.clone()
}
pub fn routing_table(&self) -> RoutingTable {
self.unlocked_inner
@ -347,7 +356,9 @@ impl NetworkManager {
pub async fn init(&self, update_callback: UpdateCallback) -> EyreResult<()> {
let routing_table = RoutingTable::new(self.clone());
routing_table.init().await?;
let address_filter = AddressFilter::new(self.config(), routing_table.clone());
*self.unlocked_inner.routing_table.write() = Some(routing_table.clone());
*self.unlocked_inner.address_filter.write() = Some(address_filter);
*self.unlocked_inner.update_callback.write() = Some(update_callback);
Ok(())
}
@ -665,7 +676,11 @@ impl NetworkManager {
// Process a received signal
#[instrument(level = "trace", skip(self), err)]
pub async fn handle_signal(&self, signal_info: SignalInfo) -> EyreResult<NetworkResult<()>> {
pub async fn handle_signal(
&self,
connection_descriptor: ConnectionDescriptor,
signal_info: SignalInfo,
) -> EyreResult<NetworkResult<()>> {
match signal_info {
SignalInfo::ReverseConnect { receipt, peer_info } => {
let routing_table = self.routing_table();
@ -686,6 +701,10 @@ impl NetworkManager {
}
};
// Restrict reverse connection to same protocol as inbound signal
let peer_nr = peer_nr
.filtered_clone(NodeRefFilter::from(connection_descriptor.protocol_type()));
// Make a reverse connection to the peer and send the receipt to it
rpc.rpc_call_return_receipt(Destination::direct(peer_nr), receipt)
.await
@ -900,7 +919,7 @@ impl NetworkManager {
// Ensure we can read the magic number
if data.len() < 4 {
log_net!(debug "short packet");
self.address_filter().punish(remote_addr);
self.address_filter().punish_ip_addr(remote_addr);
return Ok(false);
}
@ -935,7 +954,7 @@ impl NetworkManager {
Ok(v) => v,
Err(e) => {
log_net!(debug "envelope failed to decode: {}", e);
self.address_filter().punish(remote_addr);
self.address_filter().punish_ip_addr(remote_addr);
return Ok(false);
}
};
@ -987,6 +1006,10 @@ impl NetworkManager {
// Peek at header and see if we need to relay this
// If the recipient id is not our node id, then it needs relaying
let sender_id = TypedKey::new(envelope.get_crypto_kind(), envelope.get_sender_id());
if self.address_filter().is_node_id_punished(sender_id) {
return Ok(false);
}
let recipient_id = TypedKey::new(envelope.get_crypto_kind(), envelope.get_recipient_id());
if !routing_table.matches_own_node_id(&[recipient_id]) {
// See if the source node is allowed to resolve nodes
@ -1023,16 +1046,11 @@ impl NetworkManager {
};
if let Some(relay_nr) = some_relay_nr {
// Force sequencing if this came in sequenced.
// The sender did the prefer/ensure calculation when it did get_contact_method,
// so we don't need to do it here.
let relay_nr = if connection_descriptor.remote().protocol_type().is_ordered() {
let mut relay_nr = relay_nr.clone();
relay_nr.set_sequencing(Sequencing::EnsureOrdered);
relay_nr
} else {
relay_nr
};
// Ensure the protocol is forwarded exactly as is
// Address type is allowed to change if connectivity is better
let relay_nr = relay_nr.filtered_clone(
NodeRefFilter::new().with_protocol_type(connection_descriptor.protocol_type()),
);
// Relay the packet to the desired destination
log_net!("relaying {} bytes to {}", data.len(), relay_nr);
@ -1066,7 +1084,7 @@ impl NetworkManager {
Ok(v) => v,
Err(e) => {
log_net!(debug "failed to decrypt envelope body: {}",e);
self.address_filter().punish(remote_addr);
self.address_filter().punish_ip_addr(remote_addr);
return Ok(false);
}
};

54
veilid-core/src/network_manager/native/mod.rs
View File

@ -30,6 +30,43 @@ use std::path::{Path, PathBuf};
pub const PEEK_DETECT_LEN: usize = 64;
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 8;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 7;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
CAP_RELAY,
CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 4;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
/////////////////////////////////////////////////////////////////
struct NetworkInner {
@ -296,7 +333,7 @@ impl Network {
}
}
pub fn get_local_port(&self, protocol_type: ProtocolType) -> u16 {
pub fn get_local_port(&self, protocol_type: ProtocolType) -> Option<u16> {
let inner = self.inner.lock();
let local_port = match protocol_type {
ProtocolType::UDP => inner.udp_port,
@ -304,10 +341,10 @@ impl Network {
ProtocolType::WS => inner.ws_port,
ProtocolType::WSS => inner.wss_port,
};
local_port
Some(local_port)
}
fn get_preferred_local_address(&self, dial_info: &DialInfo) -> SocketAddr {
pub fn get_preferred_local_address(&self, dial_info: &DialInfo) -> Option<SocketAddr> {
let inner = self.inner.lock();
let local_port = match dial_info.protocol_type() {
@ -317,10 +354,10 @@ impl Network {
ProtocolType::WSS => inner.wss_port,
};
match dial_info.address_type() {
Some(match dial_info.address_type() {
AddressType::IPV4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), local_port),
AddressType::IPV6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), local_port),
}
})
}
pub fn is_usable_interface_address(&self, addr: IpAddr) -> bool {
@ -373,7 +410,7 @@ impl Network {
if self
.network_manager()
.address_filter()
.is_punished(dial_info.address().to_ip_addr())
.is_ip_addr_punished(dial_info.address().to_ip_addr())
{
return Ok(NetworkResult::no_connection_other("punished"));
}
@ -440,7 +477,7 @@ impl Network {
if self
.network_manager()
.address_filter()
.is_punished(dial_info.address().to_ip_addr())
.is_ip_addr_punished(dial_info.address().to_ip_addr())
{
return Ok(NetworkResult::no_connection_other("punished"));
}
@ -594,10 +631,9 @@ impl Network {
.wrap_err("failed to send data to dial info")?);
} else {
// Handle connection-oriented protocols
let local_addr = self.get_preferred_local_address(&dial_info);
let conn = network_result_try!(
self.connection_manager()
.get_or_create_connection(Some(local_addr), dial_info.clone())
.get_or_create_connection(dial_info.clone())
.await?
);

2
veilid-core/src/network_manager/native/network_class_discovery.rs
View File

@ -256,7 +256,7 @@ impl DiscoveryContext {
let at = inner.address_type.unwrap();
let external_address_1 = inner.external_1_address.unwrap();
let node_1 = inner.node_1.as_ref().unwrap().clone();
let local_port = self.net.get_local_port(pt);
let local_port = self.net.get_local_port(pt).unwrap();
(pt, llpt, at, external_address_1, node_1, local_port)
};

2
veilid-core/src/network_manager/native/network_tcp.rs
View File

@ -120,7 +120,7 @@ impl Network {
};
// Check to see if it is punished
let address_filter = self.network_manager().address_filter();
if address_filter.is_punished(peer_addr.ip()) {
if address_filter.is_ip_addr_punished(peer_addr.ip()) {
return;
}

2
veilid-core/src/network_manager/native/protocol/mod.rs
View File

@ -24,7 +24,7 @@ impl ProtocolNetworkConnection {
timeout_ms: u32,
address_filter: AddressFilter,
) -> io::Result<NetworkResult<ProtocolNetworkConnection>> {
if address_filter.is_punished(dial_info.address().to_ip_addr()) {
if address_filter.is_ip_addr_punished(dial_info.address().to_ip_addr()) {
return Ok(NetworkResult::no_connection_other("punished"));
}
match dial_info.protocol_type() {

4
veilid-core/src/network_manager/native/protocol/udp.rs
View File

@ -25,7 +25,7 @@ impl RawUdpProtocolHandler {
// Check to see if it is punished
if let Some(af) = self.address_filter.as_ref() {
if af.is_punished(remote_addr.ip()) {
if af.is_ip_addr_punished(remote_addr.ip()) {
continue;
}
}
@ -97,7 +97,7 @@ impl RawUdpProtocolHandler {
// Check to see if it is punished
if let Some(af) = self.address_filter.as_ref() {
if af.is_punished(remote_addr.ip()) {
if af.is_ip_addr_punished(remote_addr.ip()) {
return Ok(NetworkResult::no_connection_other("punished"));
}
}

7
veilid-core/src/network_manager/native/protocol/ws.rs
View File

@ -21,9 +21,10 @@ fn err_to_network_result<T>(err: async_tungstenite::tungstenite::Error) -> Netwo
match err {
async_tungstenite::tungstenite::Error::ConnectionClosed
| async_tungstenite::tungstenite::Error::AlreadyClosed
| async_tungstenite::tungstenite::Error::Io(_) => {
NetworkResult::NoConnection(to_io_error_other(err))
}
| async_tungstenite::tungstenite::Error::Io(_)
| async_tungstenite::tungstenite::Error::Protocol(
async_tungstenite::tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
) => NetworkResult::NoConnection(to_io_error_other(err)),
_ => NetworkResult::InvalidMessage(err.to_string()),
}
}

2
veilid-core/src/network_manager/network_connection.rs
View File

@ -305,7 +305,7 @@ impl NetworkConnection {
let peer_address = protocol_connection.descriptor().remote();
// Check to see if it is punished
if address_filter.is_punished(peer_address.to_socket_addr().ip()) {
if address_filter.is_ip_addr_punished(peer_address.to_socket_addr().ip()) {
return RecvLoopAction::Finish;
}

5
veilid-core/src/network_manager/send_data.rs
View File

@ -307,6 +307,11 @@ impl NetworkManager {
) -> EyreResult<NodeContactMethod> {
let routing_table = self.routing_table();
// If a node is punished, then don't try to contact it
if target_node_ref.node_ids().iter().find(|nid| self.address_filter().is_node_id_punished(**nid)).is_some() {
return Ok(NodeContactMethod::Unreachable);
}
// Figure out the best routing domain to get the contact method over
let routing_domain = match target_node_ref.best_routing_domain() {
Some(rd) => rd,

3
veilid-core/src/network_manager/tests/test_connection_table.rs
View File

@ -2,10 +2,11 @@ use super::*;
use super::connection_table::*;
use crate::tests::common::test_veilid_config::*;
use crate::tests::mock_routing_table;
pub async fn test_add_get_remove() {
let config = get_config();
let address_filter = AddressFilter::new(config.clone());
let address_filter = AddressFilter::new(config.clone(), mock_routing_table());
let table = ConnectionTable::new(config, address_filter);
let a1 = ConnectionDescriptor::new_no_local(PeerAddress::new(

17
veilid-core/src/network_manager/types/address.rs
View File

@ -2,22 +2,7 @@ use super::*;
// Ordering here matters, IPV6 is preferred to IPV4 in dial info sorts
// See issue #236 for eventual resolution of this unfortunate implementation
#[derive(
Copy,
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub enum Address {
IPV6(Ipv6Addr),
IPV4(Ipv4Addr),

19
veilid-core/src/network_manager/types/address_type.rs
View File

@ -1,22 +1,11 @@
#![allow(non_snake_case)]
use super::*;
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
EnumSetType,
)]
#[derive(Debug, PartialOrd, Ord, Hash, Serialize, Deserialize, EnumSetType)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum AddressType {
IPV4,
IPV6,
IPV6 = 0,
IPV4 = 1,
}
pub type AddressTypeSet = EnumSet<AddressType>;

17
veilid-core/src/network_manager/types/connection_descriptor.rs
View File

@ -7,22 +7,7 @@ use super::*;
/// If the medium does not allow local addresses, None should have been used or 'new_no_local'
/// If we are specifying only a port, then the socket's 'local_address()' should have been used, since an
/// established connection is always from a real address to another real address.
#[derive(
Copy,
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ConnectionDescriptor {
remote: PeerAddress,
local: Option<SocketAddress>,

16
veilid-core/src/network_manager/types/dial_info/mod.rs
View File

@ -12,21 +12,7 @@ pub use wss::*;
// Keep member order appropriate for sorting < preference
// Must match ProtocolType order
#[derive(
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
#[serde(tag = "kind")]
pub enum DialInfo {
UDP(DialInfoUDP),

17
veilid-core/src/network_manager/types/dial_info/tcp.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoTCP {
pub socket_address: SocketAddress,
}

17
veilid-core/src/network_manager/types/dial_info/udp.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoUDP {
pub socket_address: SocketAddress,
}

19
veilid-core/src/network_manager/types/dial_info/ws.rs
View File

@ -1,22 +1,7 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoWS {
pub socket_address: SocketAddress,
pub request: String,
}
}

17
veilid-core/src/network_manager/types/dial_info/wss.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Clone,
Default,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoWSS {
pub socket_address: SocketAddress,
pub request: String,

17
veilid-core/src/network_manager/types/dial_info_class.rs
View File

@ -1,22 +1,7 @@
use super::*;
// Keep member order appropriate for sorting < preference
#[derive(
Copy,
Clone,
Debug,
Eq,
PartialEq,
Ord,
PartialOrd,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum DialInfoClass {
Direct = 0, // D = Directly reachable with public IP and no firewall, with statically configured port
Mapped = 1, // M = Directly reachable with via portmap behind any NAT or firewalled with dynamically negotiated port

18
veilid-core/src/network_manager/types/dial_info_filter.rs
View File

@ -1,24 +1,8 @@
use super::*;
#[derive(
Copy,
Clone,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct DialInfoFilter {
#[with(RkyvEnumSet)]
pub protocol_type_set: ProtocolTypeSet,
#[with(RkyvEnumSet)]
pub address_type_set: AddressTypeSet,
}

20
veilid-core/src/network_manager/types/low_level_protocol_type.rs
View File

@ -1,25 +1,15 @@
#![allow(non_snake_case)]
use super::*;
// Keep member order appropriate for sorting < preference
// Must match DialInfo order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum LowLevelProtocolType {
UDP,
TCP,
UDP = 0,
TCP = 1,
}
impl LowLevelProtocolType {

17
veilid-core/src/network_manager/types/network_class.rs
View File

@ -1,21 +1,6 @@
use super::*;
#[derive(
Copy,
Clone,
Debug,
Eq,
PartialEq,
Ord,
PartialOrd,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum NetworkClass {
InboundCapable = 0, // I = Inbound capable without relay, may require signal
OutboundOnly = 1, // O = Outbound only, inbound relay required except with reverse connect signal

19
veilid-core/src/network_manager/types/peer_address.rs
View File

@ -1,24 +1,9 @@
use super::*;
#[derive(
Copy,
Clone,
Debug,
PartialEq,
PartialOrd,
Eq,
Ord,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
pub struct PeerAddress {
protocol_type: ProtocolType,
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
socket_address: SocketAddress,
}

23
veilid-core/src/network_manager/types/protocol_type.rs
View File

@ -1,27 +1,16 @@
#![allow(non_snake_case)]
use super::*;
// Keep member order appropriate for sorting < preference
// Must match DialInfo order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum ProtocolType {
UDP,
TCP,
WS,
WSS,
UDP = 0,
TCP = 1,
WS = 2,
WSS = 3,
}
impl ProtocolType {

3
veilid-core/src/network_manager/types/signal_info.rs
View File

@ -1,8 +1,7 @@
use super::*;
/// Parameter for Signal operation
#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum SignalInfo {
/// UDP Hole Punch Request
HolePunch {

16
veilid-core/src/network_manager/types/socket_address.rs
View File

@ -1,22 +1,8 @@
use super::*;
#[derive(
Copy,
Default,
Clone,
Debug,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
Copy, Default, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
pub struct SocketAddress {
address: Address,
port: u16,

73
veilid-core/src/network_manager/wasm/mod.rs
View File

@ -10,6 +10,45 @@ use std::io;
/////////////////////////////////////////////////////////////////
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 5;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 4;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
//CAP_RELAY,
//CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 2;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
//CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
/////////////////////////////////////////////////////////////////
struct NetworkInner {
network_started: bool,
network_needs_restart: bool,
@ -94,7 +133,7 @@ impl Network {
if self
.network_manager()
.address_filter()
.is_punished(dial_info.address().to_ip_addr())
.is_ip_addr_punished(dial_info.address().to_ip_addr())
{
return Ok(NetworkResult::no_connection_other("punished"));
}
@ -143,7 +182,7 @@ impl Network {
if self
.network_manager()
.address_filter()
.is_punished(dial_info.address().to_ip_addr())
.is_ip_addr_punished(dial_info.address().to_ip_addr())
{
return Ok(NetworkResult::no_connection_other("punished"));
}
@ -245,7 +284,7 @@ impl Network {
// Handle connection-oriented protocols
let conn = network_result_try!(
self.connection_manager()
.get_or_create_connection(None, dial_info.clone())
.get_or_create_connection(dial_info.clone())
.await?
);
@ -273,10 +312,10 @@ impl Network {
let inbound = ProtocolTypeSet::new();
let mut outbound = ProtocolTypeSet::new();
if c.network.protocol.ws.connect && c.capabilities.protocol_connect_ws {
if c.network.protocol.ws.connect {
outbound.insert(ProtocolType::WS);
}
if c.network.protocol.wss.connect && c.capabilities.protocol_connect_wss {
if c.network.protocol.wss.connect {
outbound.insert(ProtocolType::WSS);
}
@ -301,15 +340,26 @@ impl Network {
// set up the routing table's network config
// if we have static public dialinfo, upgrade our network class
let public_internet_capabilities = {
let c = self.config.get();
PUBLIC_INTERNET_CAPABILITIES
.iter()
.copied()
.filter(|cap| !c.capabilities.disable.contains(cap))
.collect::<Vec<Capability>>()
};
editor_public_internet.setup_network(
protocol_config.outbound,
protocol_config.inbound,
protocol_config.family_global,
public_internet_capabilities,
);
editor_public_internet.set_network_class(Some(NetworkClass::WebApp));
// commit routing table edits
editor_public_internet.commit().await;
editor_public_internet.commit();
self.inner.lock().network_started = true;
Ok(())
@ -339,8 +389,7 @@ impl Network {
.clear_dial_info_details()
.set_network_class(None)
.clear_relay_node()
.commit()
.await;
.commit();
// Cancels all async background tasks by dropping join handles
*self.inner.lock() = Self::new_inner();
@ -356,6 +405,14 @@ impl Network {
Vec::new()
}
pub fn get_local_port(&self, protocol_type: ProtocolType) -> Option<u16> {
None
}
pub fn get_preferred_local_address(&self, dial_info: &DialInfo) -> Option<SocketAddr> {
None
}
//////////////////////////////////////////
pub fn set_needs_public_dial_info_check(

2
veilid-core/src/network_manager/wasm/protocol/mod.rs
View File

@ -19,7 +19,7 @@ impl ProtocolNetworkConnection {
timeout_ms: u32,
address_filter: AddressFilter,
) -> io::Result<NetworkResult<ProtocolNetworkConnection>> {
if address_filter.is_punished(dial_info.address().to_ip_addr()) {
if address_filter.is_ip_addr_punished(dial_info.address().to_ip_addr()) {
return Ok(NetworkResult::no_connection_other("punished"));
}
match dial_info.protocol_type() {

15
veilid-core/src/routing_table/bucket.rs
View File

@ -1,6 +1,5 @@
use super::*;
use core::sync::atomic::Ordering;
use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
/// Routing Table Bucket
/// Stores map of public keys to entries, which may be in multiple routing tables per crypto kind
@ -16,15 +15,13 @@ pub struct Bucket {
pub(super) type EntriesIter<'a> =
alloc::collections::btree_map::Iter<'a, PublicKey, Arc<BucketEntry>>;
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
struct SerializedBucketEntryData {
key: PublicKey,
value: u32, // index into serialized entries list
}
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
struct SerializedBucketData {
entries: Vec<SerializedBucketEntryData>,
}
@ -50,7 +47,7 @@ impl Bucket {
data: Vec<u8>,
all_entries: &[Arc<BucketEntry>],
) -> EyreResult<()> {
let bucket_data: SerializedBucketData = from_rkyv(data)?;
let bucket_data: SerializedBucketData = deserialize_json_bytes(&data)?;
for e in bucket_data.entries {
self.entries
@ -64,7 +61,7 @@ impl Bucket {
&self,
all_entries: &mut Vec<Arc<BucketEntry>>,
entry_map: &mut HashMap<*const BucketEntry, u32>,
) -> EyreResult<Vec<u8>> {
) -> Vec<u8> {
let mut entries = Vec::new();
for (k, v) in &self.entries {
let entry_index = entry_map.entry(Arc::as_ptr(v)).or_insert_with(|| {
@ -78,8 +75,8 @@ impl Bucket {
});
}
let bucket_data = SerializedBucketData { entries };
let out = to_rkyv(&bucket_data)?;
Ok(out)
let out = serialize_json_bytes(&bucket_data);
out
}
/// Create a new entry with a node_id of this crypto kind and return it

46
veilid-core/src/routing_table/bucket_entry.rs
View File

@ -39,8 +39,7 @@ pub enum BucketEntryState {
pub struct LastConnectionKey(ProtocolType, AddressType);
/// Bucket entry information specific to the LocalNetwork RoutingDomain
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryPublicInternet {
/// The PublicInternet node info
signed_node_info: Option<Box<SignedNodeInfo>>,
@ -51,8 +50,7 @@ pub struct BucketEntryPublicInternet {
}
/// Bucket entry information specific to the LocalNetwork RoutingDomain
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryLocalNetwork {
/// The LocalNetwork node info
signed_node_info: Option<Box<SignedNodeInfo>>,
@ -63,8 +61,7 @@ pub struct BucketEntryLocalNetwork {
}
/// The data associated with each bucket entry
#[derive(Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Serialize, Deserialize)]
pub struct BucketEntryInner {
/// The node ids matching this bucket entry, with the cryptography versions supported by this node as the 'kind' field
validated_node_ids: TypedKeyGroup,
@ -79,7 +76,7 @@ pub struct BucketEntryInner {
/// unreachable may now be reachable with the same SignedNodeInfo/DialInfo
updated_since_last_network_change: bool,
/// The last connection descriptors used to contact this node, per protocol type
#[with(Skip)]
#[serde(skip)]
last_connections: BTreeMap<LastConnectionKey, (ConnectionDescriptor, Timestamp)>,
/// The node info for this entry on the publicinternet routing domain
public_internet: BucketEntryPublicInternet,
@ -88,18 +85,21 @@ pub struct BucketEntryInner {
/// Statistics gathered for the peer
peer_stats: PeerStats,
/// The accounting for the latency statistics
#[with(Skip)]
#[serde(skip)]
latency_stats_accounting: LatencyStatsAccounting,
/// The accounting for the transfer statistics
#[with(Skip)]
#[serde(skip)]
transfer_stats_accounting: TransferStatsAccounting,
/// If the entry is being punished and should be considered dead
#[serde(skip)]
is_punished: bool,
/// Tracking identifier for NodeRef debugging
#[cfg(feature = "tracking")]
#[with(Skip)]
#[serde(skip)]
next_track_id: usize,
/// Backtraces for NodeRef debugging
#[cfg(feature = "tracking")]
#[with(Skip)]
#[serde(skip)]
node_ref_tracks: HashMap<usize, backtrace::Backtrace>,
}
@ -259,6 +259,7 @@ impl BucketEntryInner {
};
// See if we have an existing signed_node_info to update or not
let mut node_info_changed = false;
if let Some(current_sni) = opt_current_sni {
// Always allow overwriting invalid/unsigned node
if current_sni.has_any_signature() {
@ -277,6 +278,11 @@ impl BucketEntryInner {
}
return;
}
// See if anything has changed in this update beside the timestamp
if signed_node_info.node_info() != current_sni.node_info() {
node_info_changed = true;
}
}
}
@ -294,7 +300,9 @@ impl BucketEntryInner {
// because the dial info could have changed and its safer to just reconnect.
// The latest connection would have been the once we got the new node info
// over so that connection is still valid.
self.clear_last_connections_except_latest();
if node_info_changed {
self.clear_last_connections_except_latest();
}
}
pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool {
@ -406,6 +414,10 @@ impl BucketEntryInner {
// Stores a connection descriptor in this entry's table of last connections
pub fn set_last_connection(&mut self, last_connection: ConnectionDescriptor, timestamp: Timestamp) {
if self.is_punished {
// Don't record connection if this entry is currently punished
return;
}
let key = self.descriptor_to_key(last_connection);
self.last_connections
.insert(key, (last_connection, timestamp));
@ -534,6 +546,9 @@ impl BucketEntryInner {
}
pub fn state(&self, cur_ts: Timestamp) -> BucketEntryState {
if self.is_punished {
return BucketEntryState::Dead;
}
if self.check_reliable(cur_ts) {
BucketEntryState::Reliable
} else if self.check_dead(cur_ts) {
@ -542,6 +557,12 @@ impl BucketEntryInner {
BucketEntryState::Unreliable
}
}
pub fn set_punished(&mut self, punished: bool) {
self.is_punished = punished;
if punished {
self.clear_last_connections();
}
}
pub fn peer_stats(&self) -> &PeerStats {
&self.peer_stats
@ -848,6 +869,7 @@ impl BucketEntry {
},
latency_stats_accounting: LatencyStatsAccounting::new(),
transfer_stats_accounting: TransferStatsAccounting::new(),
is_punished: false,
#[cfg(feature = "tracking")]
next_track_id: 0,
#[cfg(feature = "tracking")]

17
veilid-core/src/routing_table/debug.rs
View File

@ -90,19 +90,16 @@ impl RoutingTable {
for (n, gdi) in gdis.iter().enumerate() {
out += &format!(" {:>2}: {:?}\n", n, gdi);
}
out
}
out += "LocalNetwork PeerInfo:\n";
pub(crate) fn debug_info_peerinfo(&self, routing_domain: RoutingDomain) -> String {
let mut out = String::new();
out += &format!(
" {:#?}\n",
self.get_own_peer_info(RoutingDomain::LocalNetwork)
"{:?} PeerInfo:\n {:#?}\n",
routing_domain,
self.get_own_peer_info(routing_domain)
);
out += "PublicInternet PeerInfo:\n";
out += &format!(
" {:#?}\n",
self.get_own_peer_info(RoutingDomain::PublicInternet)
);
out
}

41
veilid-core/src/routing_table/mod.rs
View File

@ -48,6 +48,12 @@ pub const PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS: u32 = 1;
// We should ping them with some frequency and 30 seconds is typical timeout
pub const CONNECTIONLESS_TIMEOUT_SECS: u32 = 29;
// Table store keys
const ALL_ENTRY_BYTES: &[u8] = b"all_entry_bytes";
const ROUTING_TABLE: &str = "routing_table";
const SERIALIZED_BUCKET_MAP: &[u8] = b"serialized_bucket_map";
const CACHE_VALIDITY_KEY: &[u8] = b"cache_validity_key";
pub type LowLevelProtocolPorts = BTreeSet<(LowLevelProtocolType, AddressType, u16)>;
pub type ProtocolToPortMapping = BTreeMap<(ProtocolType, AddressType), (LowLevelProtocolType, u16)>;
#[derive(Clone, Debug)]
@ -295,7 +301,7 @@ impl RoutingTable {
}
/// Serialize the routing table.
fn serialized_buckets(&self) -> EyreResult<(SerializedBucketMap, SerializedBuckets)> {
fn serialized_buckets(&self) -> (SerializedBucketMap, SerializedBuckets) {
// Since entries are shared by multiple buckets per cryptokind
// we need to get the list of all unique entries when serializing
let mut all_entries: Vec<Arc<BucketEntry>> = Vec::new();
@ -309,7 +315,7 @@ impl RoutingTable {
let buckets = inner.buckets.get(&ck).unwrap();
let mut serialized_buckets = Vec::new();
for bucket in buckets.iter() {
serialized_buckets.push(bucket.save_bucket(&mut all_entries, &mut entry_map)?)
serialized_buckets.push(bucket.save_bucket(&mut all_entries, &mut entry_map))
}
serialized_bucket_map.insert(ck, serialized_buckets);
}
@ -319,25 +325,25 @@ impl RoutingTable {
let mut all_entry_bytes = Vec::with_capacity(all_entries.len());
for entry in all_entries {
// Serialize entry
let entry_bytes = entry.with_inner(|e| to_rkyv(e))?;
let entry_bytes = entry.with_inner(|e| serialize_json_bytes(e));
all_entry_bytes.push(entry_bytes);
}
Ok((serialized_bucket_map, all_entry_bytes))
(serialized_bucket_map, all_entry_bytes)
}
/// Write the serialized routing table to the table store.
async fn save_buckets(&self) -> EyreResult<()> {
let (serialized_bucket_map, all_entry_bytes) = self.serialized_buckets()?;
let (serialized_bucket_map, all_entry_bytes) = self.serialized_buckets();
let table_store = self.unlocked_inner.network_manager().table_store();
let tdb = table_store.open("routing_table", 1).await?;
let tdb = table_store.open(ROUTING_TABLE, 1).await?;
let dbx = tdb.transact();
if let Err(e) = dbx.store_rkyv(0, b"serialized_bucket_map", &serialized_bucket_map) {
if let Err(e) = dbx.store_json(0, SERIALIZED_BUCKET_MAP, &serialized_bucket_map) {
dbx.rollback();
return Err(e.into());
}
if let Err(e) = dbx.store_rkyv(0, b"all_entry_bytes", &all_entry_bytes) {
if let Err(e) = dbx.store_json(0, ALL_ENTRY_BYTES, &all_entry_bytes) {
dbx.rollback();
return Err(e.into());
}
@ -362,9 +368,9 @@ impl RoutingTable {
// Deserialize bucket map and all entries from the table store
let table_store = self.unlocked_inner.network_manager().table_store();
let db = table_store.open("routing_table", 1).await?;
let db = table_store.open(ROUTING_TABLE, 1).await?;
let caches_valid = match db.load(0, b"cache_validity_key").await? {
let caches_valid = match db.load(0, CACHE_VALIDITY_KEY).await? {
Some(v) => v == cache_validity_key,
None => false,
};
@ -372,19 +378,18 @@ impl RoutingTable {
// Caches not valid, start over
log_rtab!(debug "cache validity key changed, emptying routing table");
drop(db);
table_store.delete("routing_table").await?;
let db = table_store.open("routing_table", 1).await?;
db.store(0, b"cache_validity_key", &cache_validity_key)
.await?;
table_store.delete(ROUTING_TABLE).await?;
let db = table_store.open(ROUTING_TABLE, 1).await?;
db.store(0, CACHE_VALIDITY_KEY, &cache_validity_key).await?;
return Ok(());
}
// Caches valid, load saved routing table
let Some(serialized_bucket_map): Option<SerializedBucketMap> = db.load_rkyv(0, b"serialized_bucket_map").await? else {
let Some(serialized_bucket_map): Option<SerializedBucketMap> = db.load_json(0, SERIALIZED_BUCKET_MAP).await? else {
log_rtab!(debug "no bucket map in saved routing table");
return Ok(());
};
let Some(all_entry_bytes): Option<SerializedBuckets> = db.load_rkyv(0, b"all_entry_bytes").await? else {
let Some(all_entry_bytes): Option<SerializedBuckets> = db.load_json(0, ALL_ENTRY_BYTES).await? else {
log_rtab!(debug "no all_entry_bytes in saved routing table");
return Ok(());
};
@ -405,8 +410,8 @@ impl RoutingTable {
) -> EyreResult<()> {
let mut all_entries: Vec<Arc<BucketEntry>> = Vec::with_capacity(all_entry_bytes.len());
for entry_bytes in all_entry_bytes {
let entryinner =
from_rkyv(entry_bytes).wrap_err("failed to deserialize bucket entry")?;
let entryinner = deserialize_json_bytes(&entry_bytes)
.wrap_err("failed to deserialize bucket entry")?;
let entry = Arc::new(BucketEntry::new_with_inner(entryinner));
// Keep strong reference in table

11
veilid-core/src/routing_table/route_spec_store/route_set_spec_detail.rs
View File

@ -1,7 +1,6 @@
use super::*;
#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RouteSpecDetail {
/// Crypto kind
pub crypto_kind: CryptoKind,
@ -11,20 +10,18 @@ pub struct RouteSpecDetail {
pub hops: Vec<PublicKey>,
}
#[derive(Clone, Debug, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RouteSetSpecDetail {
/// Route set per crypto kind
route_set: BTreeMap<PublicKey, RouteSpecDetail>,
/// Route noderefs
#[with(Skip)]
#[serde(skip)]
hop_node_refs: Vec<NodeRef>,
/// Published private route, do not reuse for ephemeral routes
/// Not serialized because all routes should be re-published when restarting
#[with(Skip)]
#[serde(skip)]
published: bool,
/// Directions this route is guaranteed to work in
#[with(RkyvEnumSet)]
directions: DirectionSet,
/// Stability preference (prefer reliable nodes over faster)
stability: Stability,

13
veilid-core/src/routing_table/route_spec_store/route_spec_store_content.rs
View File

@ -1,8 +1,7 @@
use super::*;
/// The core representation of the RouteSpecStore that can be serialized
#[derive(Debug, Clone, Default, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RouteSpecStoreContent {
/// All of the route sets we have allocated so far indexed by key
id_by_key: HashMap<PublicKey, RouteId>,
@ -23,7 +22,7 @@ impl RouteSpecStoreContent {
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
let mut content: RouteSpecStoreContent =
rsstdb.load_rkyv(0, b"content").await?.unwrap_or_default();
rsstdb.load_json(0, b"content").await?.unwrap_or_default();
// Look up all route hop noderefs since we can't serialize those
let mut dead_ids = Vec::new();
@ -63,7 +62,7 @@ impl RouteSpecStoreContent {
// This skips #[with(Skip)] saving the secret keys, we save them in the protected store instead
let table_store = routing_table.network_manager().table_store();
let rsstdb = table_store.open("RouteSpecStore", 1).await?;
rsstdb.store_rkyv(0, b"content", self).await?;
rsstdb.store_json(0, b"content", self).await?;
Ok(())
}
@ -96,9 +95,9 @@ impl RouteSpecStoreContent {
pub fn get_id_by_key(&self, key: &PublicKey) -> Option<RouteId> {
self.id_by_key.get(key).cloned()
}
pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
self.details.keys()
}
// pub fn iter_ids(&self) -> std::collections::hash_map::Keys<RouteId, RouteSetSpecDetail> {
// self.details.keys()
// }
pub fn iter_details(&self) -> std::collections::hash_map::Iter<RouteId, RouteSetSpecDetail> {
self.details.iter()
}

17
veilid-core/src/routing_table/route_spec_store/route_stats.rs
View File

@ -1,34 +1,33 @@
use super::*;
#[derive(Clone, Debug, Default, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct RouteStats {
/// Consecutive failed to send count
#[with(Skip)]
#[serde(skip)]
pub failed_to_send: u32,
/// Questions lost
#[with(Skip)]
#[serde(skip)]
pub questions_lost: u32,
/// Timestamp of when the route was created
pub created_ts: Timestamp,
/// Timestamp of when the route was last checked for validity
#[with(Skip)]
#[serde(skip)]
pub last_tested_ts: Option<Timestamp>,
/// Timestamp of when the route was last sent to
#[with(Skip)]
#[serde(skip)]
pub last_sent_ts: Option<Timestamp>,
/// Timestamp of when the route was last received over
#[with(Skip)]
#[serde(skip)]
pub last_received_ts: Option<Timestamp>,
/// Transfers up and down
pub transfer_stats_down_up: TransferStatsDownUp,
/// Latency stats
pub latency_stats: LatencyStats,
/// Accounting mechanism for this route's RPC latency
#[with(Skip)]
#[serde(skip)]
latency_stats_accounting: LatencyStatsAccounting,
/// Accounting mechanism for the bandwidth across this route
#[with(Skip)]
#[serde(skip)]
transfer_stats_accounting: TransferStatsAccounting,
}

28
veilid-core/src/routing_table/tests/mod.rs
View File

@ -1 +1,29 @@
use super::*;
pub mod test_serialize_routing_table;
pub(crate) fn mock_routing_table() -> routing_table::RoutingTable {
let veilid_config = VeilidConfig::new();
#[cfg(feature = "unstable-blockstore")]
let block_store = BlockStore::new(veilid_config.clone());
let protected_store = ProtectedStore::new(veilid_config.clone());
let table_store = TableStore::new(veilid_config.clone(), protected_store.clone());
let crypto = Crypto::new(veilid_config.clone(), table_store.clone());
let storage_manager = storage_manager::StorageManager::new(
veilid_config.clone(),
crypto.clone(),
table_store.clone(),
#[cfg(feature = "unstable-blockstore")]
block_store.clone(),
);
let network_manager = network_manager::NetworkManager::new(
veilid_config.clone(),
storage_manager,
protected_store.clone(),
table_store.clone(),
#[cfg(feature = "unstable-blockstore")]
block_store.clone(),
crypto.clone(),
);
RoutingTable::new(network_manager)
}

35
veilid-core/src/routing_table/tests/test_serialize_routing_table.rs
View File

@ -1,41 +1,14 @@
use crate::*;
use routing_table::*;
fn fake_routing_table() -> routing_table::RoutingTable {
let veilid_config = VeilidConfig::new();
#[cfg(feature = "unstable-blockstore")]
let block_store = BlockStore::new(veilid_config.clone());
let protected_store = ProtectedStore::new(veilid_config.clone());
let table_store = TableStore::new(veilid_config.clone(), protected_store.clone());
let crypto = Crypto::new(veilid_config.clone(), table_store.clone());
let storage_manager = storage_manager::StorageManager::new(
veilid_config.clone(),
crypto.clone(),
table_store.clone(),
#[cfg(feature = "unstable-blockstore")]
block_store.clone(),
);
let network_manager = network_manager::NetworkManager::new(
veilid_config.clone(),
storage_manager,
protected_store.clone(),
table_store.clone(),
#[cfg(feature = "unstable-blockstore")]
block_store.clone(),
crypto.clone(),
);
RoutingTable::new(network_manager)
}
use super::*;
pub async fn test_routingtable_buckets_round_trip() {
let original = fake_routing_table();
let copy = fake_routing_table();
let original = mock_routing_table();
let copy = mock_routing_table();
original.init().await.unwrap();
copy.init().await.unwrap();
// Add lots of routes to `original` here to exercise all various types.
let (serialized_bucket_map, all_entry_bytes) = original.serialized_buckets().unwrap();
let (serialized_bucket_map, all_entry_bytes) = original.serialized_buckets();
copy.populate_routing_table(
&mut copy.inner.write(),

16
veilid-core/src/routing_table/types/dial_info_detail.rs
View File

@ -1,21 +1,7 @@
use super::*;
// Keep member order appropriate for sorting < preference
#[derive(
Debug,
Clone,
PartialEq,
PartialOrd,
Ord,
Eq,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize)]
pub struct DialInfoDetail {
pub class: DialInfoClass,
pub dial_info: DialInfo,

20
veilid-core/src/routing_table/types/direction.rs
View File

@ -1,22 +1,12 @@
#![allow(non_snake_case)]
use super::*;
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
PartialOrd,
Ord,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, PartialOrd, Ord, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum Direction {
Inbound,
Outbound,
Inbound = 0,
Outbound = 1,
}
pub type DirectionSet = EnumSet<Direction>;

53
veilid-core/src/routing_table/types/node_info.rs
View File

@ -12,61 +12,10 @@ pub const CAP_APPMESSAGE: Capability = FourCC(*b"APPM");
#[cfg(feature = "unstable-blockstore")]
pub const CAP_BLOCKSTORE: Capability = FourCC(*b"BLOC");
cfg_if! {
if #[cfg(all(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 8;
} else if #[cfg(any(feature = "unstable-blockstore", feature="unstable-tunnels"))] {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 7;
} else {
const PUBLIC_INTERNET_CAPABILITIES_LEN: usize = 6;
}
}
pub const PUBLIC_INTERNET_CAPABILITIES: [Capability; PUBLIC_INTERNET_CAPABILITIES_LEN] = [
CAP_ROUTE,
#[cfg(feature = "unstable-tunnels")]
CAP_TUNNEL,
CAP_SIGNAL,
CAP_RELAY,
CAP_VALIDATE_DIAL_INFO,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
#[cfg(feature = "unstable-blockstore")]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 4;
#[cfg(not(feature = "unstable-blockstore"))]
const LOCAL_NETWORK_CAPABILITIES_LEN: usize = 3;
pub const LOCAL_NETWORK_CAPABILITIES: [Capability; LOCAL_NETWORK_CAPABILITIES_LEN] = [
CAP_RELAY,
CAP_DHT,
CAP_APPMESSAGE,
#[cfg(feature = "unstable-blockstore")]
CAP_BLOCKSTORE,
];
pub const MAX_CAPABILITIES: usize = 64;
#[derive(
Clone,
Default,
PartialEq,
Eq,
Debug,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct NodeInfo {
network_class: NetworkClass,
#[with(RkyvEnumSet)]
outbound_protocols: ProtocolTypeSet,
#[with(RkyvEnumSet)]
address_types: AddressTypeSet,
envelope_support: Vec<u8>,
crypto_support: Vec<CryptoKind>,

3
veilid-core/src/routing_table/types/node_status.rs
View File

@ -2,8 +2,7 @@ use super::*;
/// Non-nodeinfo status for each node is returned by the StatusA call
#[derive(Clone, Debug, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct NodeStatus {
// Reserved for expansion
}

5
veilid-core/src/routing_table/types/peer_info.rs
View File

@ -1,9 +1,6 @@
use super::*;
#[derive(
Clone, Debug, Serialize, Deserialize, PartialEq, Eq, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct PeerInfo {
node_ids: TypedKeyGroup,
signed_node_info: SignedNodeInfo,

14
veilid-core/src/routing_table/types/routing_domain.rs
View File

@ -4,20 +4,8 @@ use super::*;
// Routing domain here is listed in order of preference, keep in order
#[allow(clippy::derive_hash_xor_eq)]
#[derive(
Debug,
Ord,
PartialOrd,
Hash,
EnumSetType,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[derive(Debug, Ord, PartialOrd, Hash, EnumSetType, Serialize, Deserialize)]
#[enumset(repr = "u8")]
#[archive_attr(repr(u8), derive(CheckBytes))]
pub enum RoutingDomain {
LocalNetwork = 0,
PublicInternet = 1,

5
veilid-core/src/routing_table/types/signed_direct_node_info.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Signed NodeInfo that can be passed around amongst peers and verifiable
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedDirectNodeInfo {
node_info: NodeInfo,
timestamp: Timestamp,

5
veilid-core/src/routing_table/types/signed_node_info.rs
View File

@ -1,9 +1,6 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SignedNodeInfo {
Direct(SignedDirectNodeInfo),
Relayed(SignedRelayedNodeInfo),

5
veilid-core/src/routing_table/types/signed_relayed_node_info.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Signed NodeInfo with a relay that can be passed around amongst peers and verifiable
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedRelayedNodeInfo {
node_info: NodeInfo,
relay_ids: TypedKeyGroup,

47
veilid-core/src/rpc_processor/mod.rs
View File

@ -1454,28 +1454,39 @@ impl RPCProcessor {
&self,
encoded_msg: RPCMessageEncoded,
) -> Result<NetworkResult<()>, RPCError> {
let address_filter = self.network_manager.address_filter();
// Decode operation appropriately based on header detail
let msg = match &encoded_msg.header.detail {
RPCMessageHeaderDetail::Direct(detail) => {
// Get sender node id
let sender_node_id = TypedKey::new(
detail.envelope.get_crypto_kind(),
detail.envelope.get_sender_id(),
);
// Decode and validate the RPC operation
let operation = match self.decode_rpc_operation(&encoded_msg) {
Ok(v) => v,
Err(e) => return Ok(NetworkResult::invalid_message(e)),
Err(e) => {
// Punish nodes that send direct undecodable crap
if matches!(e, RPCError::Protocol(_) | RPCError::InvalidFormat(_)) {
address_filter.punish_node_id(sender_node_id);
}
return Ok(NetworkResult::invalid_message(e));
}
};
// Get the routing domain this message came over
let routing_domain = detail.routing_domain;
// Get the sender noderef, incorporating sender's peer info
let sender_node_id = TypedKey::new(
detail.envelope.get_crypto_kind(),
detail.envelope.get_sender_id(),
);
let mut opt_sender_nr: Option<NodeRef> = None;
if let Some(sender_peer_info) = operation.sender_peer_info() {
// Ensure the sender peer info is for the actual sender specified in the envelope
if !sender_peer_info.node_ids().contains(&sender_node_id) {
// Attempted to update peer info for the wrong node id
address_filter.punish_node_id(sender_node_id);
return Ok(NetworkResult::invalid_message(
"attempt to update peer info for non-sender node id",
));
@ -1487,6 +1498,7 @@ impl RPCProcessor {
sender_peer_info.signed_node_info(),
&[],
) {
address_filter.punish_node_id(sender_node_id);
return Ok(NetworkResult::invalid_message(
"sender peerinfo has invalid peer scope",
));
@ -1497,7 +1509,10 @@ impl RPCProcessor {
false,
) {
Ok(v) => Some(v),
Err(e) => return Ok(NetworkResult::invalid_message(e)),
Err(e) => {
address_filter.punish_node_id(sender_node_id);
return Ok(NetworkResult::invalid_message(e));
}
}
}
@ -1505,7 +1520,10 @@ impl RPCProcessor {
if opt_sender_nr.is_none() {
opt_sender_nr = match self.routing_table().lookup_node_ref(sender_node_id) {
Ok(v) => v,
Err(e) => return Ok(NetworkResult::invalid_message(e)),
Err(e) => {
address_filter.punish_node_id(sender_node_id);
return Ok(NetworkResult::invalid_message(e));
}
}
}
@ -1525,7 +1543,14 @@ impl RPCProcessor {
}
RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => {
// Decode and validate the RPC operation
let operation = self.decode_rpc_operation(&encoded_msg)?;
let operation = match self.decode_rpc_operation(&encoded_msg) {
Ok(v) => v,
Err(e) => {
// Punish routes that send routed undecodable crap
// address_filter.punish_route_id(xxx);
return Ok(NetworkResult::invalid_message(e));
}
};
// Make the RPC message
RPCMessage {
@ -1612,7 +1637,7 @@ impl RPCProcessor {
let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker recv");
// xxx: causes crash (Missing otel data span extensions)
// rpc_worker_span.follows_from(span_id);
network_result_value_or_log!(match self
.process_rpc_message(msg)
.instrument(rpc_worker_span)
@ -1623,7 +1648,9 @@ impl RPCProcessor {
continue;
}
Ok(v) => v,
Ok(v) => {
v
}
} => [ format!(": msg.header={:?}", msg.header) ] {});
}
}

6
veilid-core/src/rpc_processor/rpc_signal.rs
View File

@ -52,8 +52,8 @@ impl RPCProcessor {
// Can't allow anything other than direct packets here, as handling reverse connections
// or anything like via signals over private routes would deanonymize the route
match &msg.header.detail {
RPCMessageHeaderDetail::Direct(_) => {}
let connection_descriptor = match &msg.header.detail {
RPCMessageHeaderDetail::Direct(d) => d.connection_descriptor,
RPCMessageHeaderDetail::SafetyRouted(_) | RPCMessageHeaderDetail::PrivateRouted(_) => {
return Ok(NetworkResult::invalid_message("signal must be direct"));
}
@ -73,7 +73,7 @@ impl RPCProcessor {
let network_manager = self.network_manager();
let signal_info = signal.destructure();
network_manager
.handle_signal(signal_info)
.handle_signal(connection_descriptor, signal_info)
.await
.map_err(RPCError::network)
}

22
veilid-core/src/storage_manager/record_store.rs
View File

@ -9,9 +9,7 @@ use hashlink::LruCache;
pub struct RecordStore<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize + for<'d> Deserialize<'d>,
{
table_store: TableStore,
name: String,
@ -41,9 +39,7 @@ pub struct SubkeyResult {
impl<D> RecordStore<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize + for<'d> Deserialize<'d>,
{
pub fn new(table_store: TableStore, name: &str, limits: RecordStoreLimits) -> Self {
let subkey_cache_size = limits.subkey_cache_size as usize;
@ -93,7 +89,7 @@ where
let mut record_index_saved: Vec<(RecordTableKey, Record<D>)> =
Vec::with_capacity(record_table_keys.len());
for rtk in record_table_keys {
if let Some(vr) = record_table.load_rkyv::<Record<D>>(0, &rtk).await? {
if let Some(vr) = record_table.load_json::<Record<D>>(0, &rtk).await? {
let rik = RecordTableKey::try_from(rtk.as_ref())?;
record_index_saved.push((rik, vr));
}
@ -265,7 +261,7 @@ where
for rtk in changed_records {
// Get the changed record and save it to the table
if let Some(r) = self.record_index.peek(&rtk) {
if let Err(e) = rt_xact.store_rkyv(0, &rtk.bytes(), r) {
if let Err(e) = rt_xact.store_json(0, &rtk.bytes(), r) {
log_stor!(error "failed to save record: {}", e);
}
}
@ -303,7 +299,7 @@ where
// Save to record table
record_table
.store_rkyv(0, &rtk.bytes(), &record)
.store_json(0, &rtk.bytes(), &record)
.await
.map_err(VeilidAPIError::internal)?;
@ -451,7 +447,7 @@ where
}
// If not in cache, try to pull from table store if it is in our stored subkey set
let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk.bytes())
.load_json::<RecordData>(0, &stk.bytes())
.await
.map_err(VeilidAPIError::internal)? else {
apibail_internal!("failed to get subkey that was stored");
@ -517,7 +513,7 @@ where
}
// If not in cache, try to pull from table store if it is in our stored subkey set
let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk.bytes())
.load_json::<RecordData>(0, &stk.bytes())
.await
.map_err(VeilidAPIError::internal)? else {
apibail_internal!("failed to peek subkey that was stored");
@ -575,7 +571,7 @@ where
} else {
// If not in cache, try to pull from table store
if let Some(record_data) = subkey_table
.load_rkyv::<RecordData>(0, &stk_bytes)
.load_json::<RecordData>(0, &stk_bytes)
.await
.map_err(VeilidAPIError::internal)?
{
@ -606,7 +602,7 @@ where
// Write subkey
subkey_table
.store_rkyv(0, &stk_bytes, &record_data)
.store_json(0, &stk_bytes, &record_data)
.await
.map_err(VeilidAPIError::internal)?;

15
veilid-core/src/storage_manager/storage_manager_inner.rs
View File

@ -1,5 +1,8 @@
use super::*;
const STORAGE_MANAGER_METADATA: &str = "storage_manager_metadata";
const OFFLINE_SUBKEY_WRITES: &[u8] = b"offline_subkey_writes";
/// Locked structure for storage manager
pub(super) struct StorageManagerInner {
unlocked_inner: Arc<StorageManagerUnlockedInner>,
@ -68,7 +71,7 @@ impl StorageManagerInner {
let metadata_db = self.unlocked_inner
.table_store
.open(&format!("storage_manager_metadata"), 1)
.open(STORAGE_MANAGER_METADATA, 1)
.await?;
let local_limits = local_limits_from_config(self.unlocked_inner.config.clone());
@ -146,7 +149,7 @@ impl StorageManagerInner {
async fn save_metadata(&mut self) -> EyreResult<()>{
if let Some(metadata_db) = &self.metadata_db {
let tx = metadata_db.transact();
tx.store_rkyv(0, b"offline_subkey_writes", &self.offline_subkey_writes)?;
tx.store_json(0, OFFLINE_SUBKEY_WRITES, &self.offline_subkey_writes)?;
tx.commit().await.wrap_err("failed to commit")?
}
Ok(())
@ -154,10 +157,10 @@ impl StorageManagerInner {
async fn load_metadata(&mut self) -> EyreResult<()> {
if let Some(metadata_db) = &self.metadata_db {
self.offline_subkey_writes = match metadata_db.load_rkyv(0, b"offline_subkey_writes").await {
self.offline_subkey_writes = match metadata_db.load_json(0, OFFLINE_SUBKEY_WRITES).await {
Ok(v) => v.unwrap_or_default(),
Err(_) => {
if let Err(e) = metadata_db.delete(0,b"offline_subkey_writes").await {
if let Err(e) = metadata_db.delete(0, OFFLINE_SUBKEY_WRITES).await {
debug!("offline_subkey_writes format changed, clearing: {}", e);
}
Default::default()
@ -483,9 +486,7 @@ impl StorageManagerInner {
/// # DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
fn get_key<D>(vcrypto: CryptoSystemVersion, record: &Record<D>) -> TypedKey
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
let compiled = record.descriptor().schema_data();
let mut hash_data = Vec::<u8>::with_capacity(PUBLIC_KEY_LENGTH + 4 + compiled.len());

5
veilid-core/src/storage_manager/types/local_record_detail.rs
View File

@ -1,10 +1,7 @@
use super::*;
/// Information required to handle locally opened records
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalRecordDetail {
/// The last 'safety selection' used when creating/opening this record.
/// Even when closed, this safety selection applies to re-publication attempts by the system.

19
veilid-core/src/storage_manager/types/record.rs
View File

@ -1,14 +1,9 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Record<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
descriptor: SignedValueDescriptor,
subkey_count: usize,
@ -20,9 +15,7 @@ where
impl<D> Record<D>
where
D: fmt::Debug + Clone + RkyvArchive + RkyvSerialize<DefaultVeilidRkyvSerializer>,
for<'t> <D as RkyvArchive>::Archived: CheckBytes<RkyvDefaultValidator<'t>>,
<D as RkyvArchive>::Archived: RkyvDeserialize<D, VeilidSharedDeserializeMap>,
D: fmt::Debug + Clone + Serialize,
{
pub fn new(
cur_ts: Timestamp,
@ -84,9 +77,9 @@ where
mem::size_of::<Record<D>>() + self.descriptor.total_size() + self.record_data_size
}
pub fn detail(&self) -> &D {
&self.detail
}
// pub fn detail(&self) -> &D {
// &self.detail
// }
pub fn detail_mut(&mut self) -> &mut D {
&mut self.detail
}

15
veilid-core/src/storage_manager/types/record_data.rs
View File

@ -1,19 +1,6 @@
use super::*;
#[derive(
Clone,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct RecordData {
signed_value_data: SignedValueData,
}

5
veilid-core/src/storage_manager/types/remote_record_detail.rs
View File

@ -1,7 +1,4 @@
use super::*;
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RemoteRecordDetail {}

15
veilid-core/src/storage_manager/types/signed_value_data.rs
View File

@ -3,20 +3,7 @@ use super::*;
/////////////////////////////////////////////////////////////////////////////////////////////////////
///
#[derive(
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)]
pub struct SignedValueData {
value_data: ValueData,
signature: Signature,

14
veilid-core/src/storage_manager/types/signed_value_descriptor.rs
View File

@ -3,19 +3,7 @@ use super::*;
/////////////////////////////////////////////////////////////////////////////////////////////////////
///
#[derive(
Clone,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Clone, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)]
pub struct SignedValueDescriptor {
owner: PublicKey,
schema_data: Vec<u8>,

88
veilid-core/src/table_store/table_db.rs
View File

@ -102,13 +102,14 @@ impl TableDB {
/// but if the contents are guaranteed to be unique, then a nonce
/// can be generated from the hash of the contents and the encryption key itself
fn maybe_encrypt(&self, data: &[u8], keyed_nonce: bool) -> Vec<u8> {
let data = compress_prepend_size(data);
if let Some(ei) = &self.unlocked_inner.encrypt_info {
let mut out = unsafe { unaligned_u8_vec_uninit(NONCE_LENGTH + data.len()) };
if keyed_nonce {
// Key content nonce
let mut noncedata = Vec::with_capacity(data.len() + PUBLIC_KEY_LENGTH);
noncedata.extend_from_slice(data);
noncedata.extend_from_slice(&data);
noncedata.extend_from_slice(&ei.key.bytes);
let noncehash = ei.vcrypto.generate_hash(&noncedata);
out[0..NONCE_LENGTH].copy_from_slice(&noncehash[0..NONCE_LENGTH])
@ -119,23 +120,23 @@ impl TableDB {
let (nonce, encout) = out.split_at_mut(NONCE_LENGTH);
ei.vcrypto.crypt_b2b_no_auth(
data,
&data,
encout,
(nonce as &[u8]).try_into().unwrap(),
&ei.key,
);
out
} else {
data.to_vec()
data
}
}
/// Decrypt buffer using decrypt key with nonce prepended to input
fn maybe_decrypt(&self, data: &[u8]) -> Vec<u8> {
fn maybe_decrypt(&self, data: &[u8]) -> std::io::Result<Vec<u8>> {
if let Some(di) = &self.unlocked_inner.decrypt_info {
assert!(data.len() >= NONCE_LENGTH);
if data.len() == NONCE_LENGTH {
return Vec::new();
return Ok(Vec::new());
}
let mut out = unsafe { unaligned_u8_vec_uninit(data.len() - NONCE_LENGTH) };
@ -146,9 +147,11 @@ impl TableDB {
(&data[0..NONCE_LENGTH]).try_into().unwrap(),
&di.key,
);
out
decompress_size_prepended(&out)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
} else {
data.to_vec()
decompress_size_prepended(data)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
}
}
@ -163,7 +166,8 @@ impl TableDB {
let db = self.unlocked_inner.database.clone();
let mut out = Vec::new();
db.iter_keys(col, None, |k| {
out.push(self.maybe_decrypt(k));
let key = self.maybe_decrypt(k)?;
out.push(key);
Ok(Option::<()>::None)
})
.await
@ -195,15 +199,6 @@ impl TableDB {
db.write(dbt).await.map_err(VeilidAPIError::generic)
}
/// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let value = to_rkyv(value)?;
self.store(col, key, &value).await
}
/// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
@ -223,26 +218,10 @@ impl TableDB {
}
let db = self.unlocked_inner.database.clone();
let key = self.maybe_encrypt(key, true);
Ok(db
.get(col, &key)
.await
.map_err(VeilidAPIError::from)?
.map(|v| self.maybe_decrypt(&v)))
}
/// Read an rkyv key from a column in the TableDB immediately
pub async fn load_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = match self.load(col, key).await? {
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(out)
match db.get(col, &key).await.map_err(VeilidAPIError::from)? {
Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)),
None => Ok(None),
}
}
/// Read an serde-json key from a column in the TableDB immediately
@ -268,27 +247,11 @@ impl TableDB {
let key = self.maybe_encrypt(key, true);
let db = self.unlocked_inner.database.clone();
let old_value = db
.delete(col, &key)
.await
.map_err(VeilidAPIError::from)?
.map(|v| self.maybe_decrypt(&v));
Ok(old_value)
}
/// Delete rkyv key with from a column in the TableDB
pub async fn delete_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let old_value = match self.delete(col, key).await? {
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(old_value)
match db.delete(col, &key).await.map_err(VeilidAPIError::from)? {
Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)),
None => Ok(None),
}
}
/// Delete serde-json key with from a column in the TableDB
@ -377,16 +340,7 @@ impl TableDBTransaction {
Ok(())
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let value = to_rkyv(value)?;
self.store(col, key, &value)
}
/// Store a key in rkyv format with a value in a column in the TableDB
/// Store a key in json format with a value in a column in the TableDB
pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: serde::Serialize,

11
veilid-core/src/table_store/table_store.rs
View File

@ -1,6 +1,8 @@
use super::*;
use keyvaluedb::*;
const ALL_TABLE_NAMES: &[u8] = b"all_table_names";
struct TableStoreInner {
opened: BTreeMap<String, Weak<TableDBUnlockedInner>>,
encryption_key: Option<TypedSharedSecret>,
@ -52,12 +54,11 @@ impl TableStore {
async fn flush(&self) {
let (all_table_names_value, all_tables_db) = {
let inner = self.inner.lock();
let all_table_names_value =
to_rkyv(&inner.all_table_names).expect("failed to archive all_table_names");
let all_table_names_value = serialize_json_bytes(&inner.all_table_names);
(all_table_names_value, inner.all_tables_db.clone().unwrap())
};
let mut dbt = DBTransaction::new();
dbt.put(0, b"all_table_names", &all_table_names_value);
dbt.put(0, ALL_TABLE_NAMES, &all_table_names_value);
if let Err(e) = all_tables_db.write(dbt).await {
error!("failed to write all tables db: {}", e);
}
@ -373,8 +374,8 @@ impl TableStore {
.open("__veilid_all_tables", 1)
.await
.wrap_err("failed to create all tables table")?;
match all_tables_db.get(0, b"all_table_names").await {
Ok(Some(v)) => match from_rkyv::<HashMap<String, String>>(v) {
match all_tables_db.get(0, ALL_TABLE_NAMES).await {
Ok(Some(v)) => match deserialize_json_bytes::<HashMap<String, String>>(&v) {
Ok(all_table_names) => {
let mut inner = self.inner.lock();
inner.all_table_names = all_table_names;

44
veilid-core/src/table_store/tests/test_table_store.rs
View File

@ -145,7 +145,6 @@ pub async fn test_transaction(ts: TableStore) {
let tx = db.transact();
assert!(tx.store(0, b"aaa", b"a-value").is_ok());
assert!(tx.store_json(1, b"bbb", &"b-value".to_owned()).is_ok());
assert!(tx.store_rkyv(2, b"ccc", &"c-value".to_owned()).is_ok());
assert!(tx.store(3, b"ddd", b"d-value").is_err());
assert!(tx.store(0, b"ddd", b"d-value").is_ok());
assert!(tx.delete(0, b"ddd").is_ok());
@ -160,51 +159,9 @@ pub async fn test_transaction(ts: TableStore) {
db.load_json::<String>(1, b"bbb").await,
Ok(Some("b-value".to_owned()))
);
assert_eq!(
db.load_rkyv::<String>(2, b"ccc").await,
Ok(Some("c-value".to_owned()))
);
assert_eq!(db.load(0, b"ddd").await, Ok(None));
}
pub async fn test_rkyv(vcrypto: CryptoSystemVersion, ts: TableStore) {
trace!("test_rkyv");
let _ = ts.delete("test");
let db = ts.open("test", 3).await.expect("should have opened");
let keypair = vcrypto.generate_keypair();
assert!(db.store_rkyv(0, b"asdf", &keypair).await.is_ok());
assert_eq!(db.load_rkyv::<KeyPair>(0, b"qwer").await.unwrap(), None);
let d = match db.load_rkyv::<KeyPair>(0, b"asdf").await {
Ok(x) => x,
Err(e) => {
panic!("couldn't decode: {}", e);
}
};
assert_eq!(d, Some(keypair), "keys should be equal");
let d = match db.delete_rkyv::<KeyPair>(0, b"asdf").await {
Ok(x) => x,
Err(e) => {
panic!("couldn't decode: {}", e);
}
};
assert_eq!(d, Some(keypair), "keys should be equal");
assert!(
db.store(1, b"foo", b"1234567890").await.is_ok(),
"should store new key"
);
assert!(
db.load_rkyv::<TypedKey>(1, b"foo").await.is_err(),
"should fail to unfreeze"
);
}
pub async fn test_json(vcrypto: CryptoSystemVersion, ts: TableStore) {
trace!("test_json");
@ -304,7 +261,6 @@ pub async fn test_all() {
test_delete_open_delete(ts.clone()).await;
test_store_delete_load(ts.clone()).await;
test_transaction(ts.clone()).await;
test_rkyv(vcrypto.clone(), ts.clone()).await;
test_json(vcrypto, ts.clone()).await;
let _ = ts.delete("test").await;
}

4
veilid-core/src/tests/native/mod.rs
View File

@ -35,8 +35,6 @@ pub async fn run_all_tests() {
test_envelope_receipt::test_all().await;
info!("TEST: veilid_api::tests::test_serialize_json");
veilid_api::tests::test_serialize_json::test_all().await;
info!("TEST: veilid_api::tests::test_serialize_rkyv");
veilid_api::tests::test_serialize_rkyv::test_all().await;
info!("TEST: routing_table::test_serialize_routing_table");
routing_table::tests::test_serialize_routing_table::test_all().await;
@ -138,8 +136,6 @@ cfg_if! {
run_test!(veilid_api, test_serialize_json);
run_test!(veilid_api, test_serialize_rkyv);
run_test!(routing_table, test_serialize_routing_table);
}
}

44
veilid-core/src/veilid_api/debug.rs
View File

@ -467,6 +467,23 @@ impl VeilidAPI {
let routing_table = self.network_manager()?.routing_table();
Ok(routing_table.debug_info_dialinfo())
}
async fn debug_peerinfo(&self, args: String) -> VeilidAPIResult<String> {
// Dump routing table peerinfo
let args: Vec<String> = args.split_whitespace().map(|s| s.to_owned()).collect();
let routing_table = self.network_manager()?.routing_table();
let routing_domain = get_debug_argument_at(
&args,
0,
"debug_peerinfo",
"routing_domain",
get_routing_domain,
)
.ok()
.unwrap_or(RoutingDomain::PublicInternet);
Ok(routing_table.debug_info_peerinfo(routing_domain))
}
async fn debug_txtrecord(&self, _args: String) -> VeilidAPIResult<String> {
// Dump routing table txt record
@ -1068,22 +1085,24 @@ impl VeilidAPI {
let routing_table = netman.routing_table();
let crypto = self.crypto()?;
let csv = get_debug_argument_at(
&args,
1,
"debug_record_create",
"kind",
get_crypto_system_version(crypto.clone()),
)
.unwrap_or_else(|_| crypto.best());
let schema = get_debug_argument_at(
&args,
2,
1,
"debug_record_create",
"dht_schema",
get_dht_schema,
)
.unwrap_or_else(|_| DHTSchema::dflt(1));
let csv = get_debug_argument_at(
&args,
2,
"debug_record_create",
"kind",
get_crypto_system_version(crypto.clone()),
)
.unwrap_or_else(|_| crypto.best());
let ss = get_debug_argument_at(
&args,
3,
@ -1106,7 +1125,7 @@ impl VeilidAPI {
};
// Do a record get
let record = match rc.create_dht_record(csv.kind(), schema).await {
let record = match rc.create_dht_record(schema, Some(csv.kind())).await {
Err(e) => return Ok(format!("Can't open DHT record: {}", e)),
Ok(v) => v,
};
@ -1325,6 +1344,7 @@ impl VeilidAPI {
pub async fn debug_help(&self, _args: String) -> VeilidAPIResult<String> {
Ok(r#"buckets [dead|reliable]
dialinfo
peerinfo [routingdomain]
entries [dead|reliable]
entry <node>
nodeinfo
@ -1348,7 +1368,7 @@ route allocate [ord|*ord] [rel] [<count>] [in|out]
test <route>
record list <local|remote>
purge <local|remote> [bytes]
create <cryptokind> <dhtschema> <safety>
create <dhtschema> [<cryptokind> [<safety>]]
set <key>[+<safety>] <subkey> <writer> <data>
get <key>[+<safety>] <subkey> [force]
delete <key>
@ -1398,6 +1418,8 @@ record list <local|remote>
self.debug_buckets(rest).await
} else if arg == "dialinfo" {
self.debug_dialinfo(rest).await
} else if arg == "peerinfo" {
self.debug_peerinfo(rest).await
} else if arg == "txtrecord" {
self.debug_txtrecord(rest).await
} else if arg == "keypair" {

21
veilid-core/src/veilid_api/error.rs
View File

@ -105,21 +105,8 @@ macro_rules! apibail_already_initialized {
}
#[derive(
ThisError,
Clone,
Debug,
PartialOrd,
PartialEq,
Eq,
Ord,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
ThisError, Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize, JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[serde(tag = "kind")]
pub enum VeilidAPIError {
#[error("Not initialized")]
@ -137,9 +124,9 @@ pub enum VeilidAPIError {
#[error("No connection: {message}")]
NoConnection { message: String },
#[error("Key not found: {key}")]
KeyNotFound {
#[schemars(with="String")]
key: TypedKey
KeyNotFound {
#[schemars(with = "String")]
key: TypedKey,
},
#[error("Internal: {message}")]
Internal { message: String },

32
veilid-core/src/veilid_api/json_api/crypto_system.rs
View File

@ -35,24 +35,24 @@ pub enum CryptoSystemRequestOp {
},
DefaultSaltLength,
HashPassword {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
salt: Vec<u8>,
},
VerifyPassword {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
password_hash: String,
},
DeriveSharedSecret {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
password: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
salt: Vec<u8>,
},
@ -60,7 +60,7 @@ pub enum CryptoSystemRequestOp {
RandomSharedSecret,
GenerateKeyPair,
GenerateHash {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},
@ -71,7 +71,7 @@ pub enum CryptoSystemRequestOp {
secret: SecretKey,
},
ValidateHash {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "String")]
@ -88,14 +88,14 @@ pub enum CryptoSystemRequestOp {
key: PublicKey,
#[schemars(with = "String")]
secret: SecretKey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},
Verify {
#[schemars(with = "String")]
key: PublicKey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "String")]
@ -103,31 +103,31 @@ pub enum CryptoSystemRequestOp {
},
AeadOverhead,
DecryptAead {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
nonce: Nonce,
#[schemars(with = "String")]
shared_secret: SharedSecret,
#[serde(with = "opt_json_as_base64")]
#[serde(with = "as_human_opt_base64")]
#[schemars(with = "Option<String>")]
associated_data: Option<Vec<u8>>,
},
EncryptAead {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
nonce: Nonce,
#[schemars(with = "String")]
shared_secret: SharedSecret,
#[serde(with = "opt_json_as_base64")]
#[serde(with = "as_human_opt_base64")]
#[schemars(with = "Option<String>")]
associated_data: Option<Vec<u8>>,
},
CryptNoAuth {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
body: Vec<u8>,
#[schemars(with = "String")]
@ -152,7 +152,7 @@ pub enum CryptoSystemResponseOp {
result: ApiResultWithString<SharedSecret>,
},
RandomBytes {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
@ -221,7 +221,7 @@ pub enum CryptoSystemResponseOp {
result: ApiResultWithVecU8,
},
CryptNoAuth {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},

14
veilid-core/src/veilid_api/json_api/mod.rs
View File

@ -58,7 +58,7 @@ pub enum RequestOp {
sequencing: Sequencing,
},
ImportRemotePrivateRoute {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
blob: Vec<u8>,
},
@ -69,7 +69,7 @@ pub enum RequestOp {
AppCallReply {
#[schemars(with = "String")]
call_id: OperationId,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
@ -96,14 +96,14 @@ pub enum RequestOp {
VerifySignatures {
#[schemars(with = "Vec<String>")]
node_ids: Vec<TypedKey>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "Vec<String>")]
signatures: Vec<TypedSignature>,
},
GenerateSignatures {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
#[schemars(with = "Vec<String>")]
@ -126,7 +126,7 @@ pub enum RequestOp {
pub struct NewPrivateRouteResult {
#[schemars(with = "String")]
route_id: RouteId,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
blob: Vec<u8>,
}
@ -260,7 +260,7 @@ where
#[serde(untagged)]
pub enum ApiResultWithVecU8 {
Ok {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
@ -271,7 +271,7 @@ pub enum ApiResultWithVecU8 {
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
#[serde(transparent)]
pub struct VecU8 {
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
}

4
veilid-core/src/veilid_api/json_api/process.rs
View File

@ -277,10 +277,10 @@ impl JsonRequestProcessor {
),
}
}
RoutingContextRequestOp::CreateDhtRecord { kind, schema } => {
RoutingContextRequestOp::CreateDhtRecord { schema, kind } => {
RoutingContextResponseOp::CreateDhtRecord {
result: to_json_api_result(
routing_context.create_dht_record(kind, schema).await,
routing_context.create_dht_record(schema, kind).await,
),
}
}

10
veilid-core/src/veilid_api/json_api/routing_context.rs
View File

@ -27,20 +27,20 @@ pub enum RoutingContextRequestOp {
},
AppCall {
target: String,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
AppMessage {
target: String,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
},
CreateDhtRecord {
#[schemars(with = "String")]
kind: CryptoKind,
schema: DHTSchema,
#[schemars(with = "Option<String>")]
kind: Option<CryptoKind>,
},
OpenDhtRecord {
#[schemars(with = "String")]
@ -66,7 +66,7 @@ pub enum RoutingContextRequestOp {
#[schemars(with = "String")]
key: TypedKey,
subkey: ValueSubkey,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
data: Vec<u8>,
},

14
veilid-core/src/veilid_api/json_api/table_db.rs
View File

@ -25,22 +25,22 @@ pub enum TableDbRequestOp {
Transact,
Store {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
Load {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},
Delete {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},
@ -101,16 +101,16 @@ pub enum TableDbTransactionRequestOp {
Rollback,
Store {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
value: Vec<u8>,
},
Delete {
col: u32,
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
key: Vec<u8>,
},

3
veilid-core/src/veilid_api/routing_context.rs
View File

@ -195,9 +195,10 @@ impl RoutingContext {
/// Returns the newly allocated DHT record's key if successful. The records is considered 'open' after the create operation succeeds.
pub async fn create_dht_record(
&self,
kind: CryptoKind,
schema: DHTSchema,
kind: Option<CryptoKind>,
) -> VeilidAPIResult<DHTRecordDescriptor> {
let kind = kind.unwrap_or(best_crypto_kind());
let storage_manager = self.api.storage_manager()?;
storage_manager
.create_record(kind, schema, self.unlocked_inner.safety_selection)

13
veilid-core/src/veilid_api/serialize_helpers/mod.rs
View File

@ -1,14 +1,7 @@
mod rkyv_enum_set;
mod rkyv_range_set_blaze;
pub mod serialize_arc;
mod serialize_json;
pub mod serialize_range_set_blaze;
mod veilid_rkyv;
use super::*;
use core::fmt::Debug;
pub use rkyv_enum_set::*;
pub use rkyv_range_set_blaze::*;
pub mod serialize_arc;
mod serialize_json;
pub mod serialize_range_set_blaze;
pub use serialize_json::*;
pub use veilid_rkyv::*;

53
veilid-core/src/veilid_api/serialize_helpers/rkyv_enum_set.rs
View File

@ -1,53 +0,0 @@
use super::*;
pub struct RkyvEnumSet;
impl<T> rkyv::with::ArchiveWith<EnumSet<T>> for RkyvEnumSet
where
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Archive,
{
type Archived = rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>;
type Resolver = rkyv::Resolver<<T as EnumSetTypeWithRepr>::Repr>;
#[inline]
unsafe fn resolve_with(
field: &EnumSet<T>,
pos: usize,
resolver: Self::Resolver,
out: *mut Self::Archived,
) {
let r = field.as_repr();
r.resolve(pos, resolver, out);
}
}
impl<T, S> rkyv::with::SerializeWith<EnumSet<T>, S> for RkyvEnumSet
where
S: rkyv::Fallible + ?Sized,
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Serialize<S>,
{
fn serialize_with(field: &EnumSet<T>, serializer: &mut S) -> Result<Self::Resolver, S::Error> {
let r = field.as_repr();
r.serialize(serializer)
}
}
impl<T, D>
rkyv::with::DeserializeWith<rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>, EnumSet<T>, D>
for RkyvEnumSet
where
D: rkyv::Fallible + ?Sized,
T: EnumSetType + EnumSetTypeWithRepr,
<T as EnumSetTypeWithRepr>::Repr: rkyv::Archive,
rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>:
rkyv::Deserialize<<T as EnumSetTypeWithRepr>::Repr, D>,
{
fn deserialize_with(
field: &rkyv::Archived<<T as EnumSetTypeWithRepr>::Repr>,
deserializer: &mut D,
) -> Result<EnumSet<T>, D::Error> {
Ok(EnumSet::<T>::from_repr(field.deserialize(deserializer)?))
}
}

73
veilid-core/src/veilid_api/serialize_helpers/rkyv_range_set_blaze.rs
View File

@ -1,73 +0,0 @@
use super::*;
use range_set_blaze::*;
pub struct RkyvRangeSetBlaze;
impl<T> rkyv::with::ArchiveWith<RangeSetBlaze<T>> for RkyvRangeSetBlaze
where
T: rkyv::Archive + Integer,
{
type Archived = rkyv::Archived<Vec<T>>;
type Resolver = rkyv::Resolver<Vec<T>>;
#[inline]
unsafe fn resolve_with(
field: &RangeSetBlaze<T>,
pos: usize,
resolver: Self::Resolver,
out: *mut Self::Archived,
) {
let mut r = Vec::<T>::with_capacity(field.ranges_len() * 2);
for range in field.ranges() {
r.push(*range.start());
r.push(*range.end());
}
r.resolve(pos, resolver, out);
}
}
impl<T, S> rkyv::with::SerializeWith<RangeSetBlaze<T>, S> for RkyvRangeSetBlaze
where
S: rkyv::Fallible + ?Sized,
Vec<T>: rkyv::Serialize<S>,
T: rkyv::Archive + Integer,
{
fn serialize_with(
field: &RangeSetBlaze<T>,
serializer: &mut S,
) -> Result<Self::Resolver, S::Error> {
let mut r = Vec::<T>::with_capacity(field.ranges_len() * 2);
for range in field.ranges() {
r.push(*range.start());
r.push(*range.end());
}
r.serialize(serializer)
}
}
impl<T, D> rkyv::with::DeserializeWith<rkyv::Archived<Vec<T>>, RangeSetBlaze<T>, D>
for RkyvRangeSetBlaze
where
D: rkyv::Fallible + ?Sized,
T: rkyv::Archive + Integer,
rkyv::Archived<T>: rkyv::Deserialize<T, D>,
D::Error: From<String>,
{
fn deserialize_with(
field: &rkyv::Archived<Vec<T>>,
deserializer: &mut D,
) -> Result<RangeSetBlaze<T>, D::Error> {
let mut out = RangeSetBlaze::<T>::new();
if field.len() % 2 == 1 {
return Err("invalid range set length".to_owned().into());
}
let f = field.as_slice();
for i in 0..field.len() / 2 {
let l: T = f[i * 2].deserialize(deserializer)?;
let u: T = f[i * 2 + 1].deserialize(deserializer)?;
out.ranges_insert(l..=u);
}
Ok(out)
}
}

213
veilid-core/src/veilid_api/serialize_helpers/serialize_json.rs
View File

@ -13,6 +13,18 @@ pub fn deserialize_json<'a, T: de::Deserialize<'a> + Debug>(arg: &'a str) -> Vei
),
})
}
pub fn deserialize_json_bytes<'a, T: de::Deserialize<'a> + Debug>(
arg: &'a [u8],
) -> VeilidAPIResult<T> {
serde_json::from_slice(arg).map_err(|e| VeilidAPIError::ParseError {
message: e.to_string(),
value: format!(
"deserialize_json_bytes:\n---\n{:?}\n---\n to type {}",
arg,
std::any::type_name::<T>()
),
})
}
// #[instrument(level = "trace", ret, err)]
pub fn deserialize_opt_json<T: de::DeserializeOwned + Debug>(
@ -28,6 +40,20 @@ pub fn deserialize_opt_json<T: de::DeserializeOwned + Debug>(
deserialize_json(arg)
}
// #[instrument(level = "trace", ret, err)]
pub fn deserialize_opt_json_bytes<T: de::DeserializeOwned + Debug>(
arg: Option<Vec<u8>>,
) -> VeilidAPIResult<T> {
let arg = arg.as_ref().ok_or_else(|| VeilidAPIError::ParseError {
message: "invalid null string".to_owned(),
value: format!(
"deserialize_json_opt: null to type {}",
std::any::type_name::<T>()
),
})?;
deserialize_json_bytes(arg.as_slice())
}
// #[instrument(level = "trace", ret)]
pub fn serialize_json<T: Serialize + Debug>(val: T) -> String {
match serde_json::to_string(&val) {
@ -37,97 +63,138 @@ pub fn serialize_json<T: Serialize + Debug>(val: T) -> String {
}
}
}
pub fn serialize_json_bytes<T: Serialize + Debug>(val: T) -> Vec<u8> {
match serde_json::to_vec(&val) {
Ok(v) => v,
Err(e) => {
panic!(
"failed to serialize json value to bytes: {}\nval={:?}",
e, val
);
}
}
}
pub mod json_as_base64 {
pub mod as_human_base64 {
use data_encoding::BASE64URL_NOPAD;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<S: Serializer>(v: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
let base64 = BASE64URL_NOPAD.encode(v);
String::serialize(&base64, s)
if s.is_human_readable() {
let base64 = BASE64URL_NOPAD.encode(v);
String::serialize(&base64, s)
} else {
Vec::<u8>::serialize(v, s)
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
let base64 = String::deserialize(d)?;
BASE64URL_NOPAD
.decode(base64.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
if d.is_human_readable() {
let base64 = String::deserialize(d)?;
BASE64URL_NOPAD
.decode(base64.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
} else {
Vec::<u8>::deserialize(d)
}
}
}
pub mod opt_json_as_base64 {
pub mod as_human_opt_base64 {
use data_encoding::BASE64URL_NOPAD;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<S: Serializer>(v: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x));
Option::<String>::serialize(&base64, s)
if s.is_human_readable() {
let base64 = v.as_ref().map(|x| BASE64URL_NOPAD.encode(&x));
Option::<String>::serialize(&base64, s)
} else {
Option::<Vec<u8>>::serialize(v, s)
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
let base64 = Option::<String>::deserialize(d)?;
base64
.map(|x| {
BASE64URL_NOPAD
.decode(x.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
})
.transpose()
}
}
pub mod json_as_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serializer};
pub fn serialize<T, S>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
where
T: Display,
S: Serializer,
{
serializer.collect_str(value)
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
where
T: FromStr,
T::Err: Display,
D: Deserializer<'de>,
{
String::deserialize(deserializer)?
.parse()
.map_err(de::Error::custom)
}
}
pub mod opt_json_as_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serializer};
pub fn serialize<T, S>(value: &Option<T>, serializer: S) -> Result<S::Ok, S::Error>
where
T: Display,
S: Serializer,
{
match value {
Some(v) => serializer.collect_str(v),
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
where
T: FromStr,
T::Err: Display,
D: Deserializer<'de>,
{
match Option::<String>::deserialize(deserializer)? {
None => Ok(None),
Some(v) => Ok(Some(v.parse::<T>().map_err(de::Error::custom)?)),
if d.is_human_readable() {
let base64 = Option::<String>::deserialize(d)?;
base64
.map(|x| {
BASE64URL_NOPAD
.decode(x.as_bytes())
.map_err(|e| serde::de::Error::custom(e))
})
.transpose()
} else {
Option::<Vec<u8>>::deserialize(d)
}
}
}
pub mod as_human_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<T, S>(value: &T, s: S) -> Result<S::Ok, S::Error>
where
T: Display + Serialize,
S: Serializer,
{
if s.is_human_readable() {
s.collect_str(value)
} else {
T::serialize(value, s)
}
}
pub fn deserialize<'de, T, D>(d: D) -> Result<T, D::Error>
where
T: FromStr + Deserialize<'de>,
T::Err: Display,
D: Deserializer<'de>,
{
if d.is_human_readable() {
String::deserialize(d)?.parse().map_err(de::Error::custom)
} else {
T::deserialize(d)
}
}
}
pub mod as_human_opt_string {
use std::fmt::Display;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<T, S>(value: &Option<T>, s: S) -> Result<S::Ok, S::Error>
where
T: Display + Serialize,
S: Serializer,
{
if s.is_human_readable() {
match value {
Some(v) => s.collect_str(v),
None => s.serialize_none(),
}
} else {
Option::<T>::serialize(value, s)
}
}
pub fn deserialize<'de, T, D>(d: D) -> Result<Option<T>, D::Error>
where
T: FromStr + Deserialize<'de>,
T::Err: Display,
D: Deserializer<'de>,
{
if d.is_human_readable() {
match Option::<String>::deserialize(d)? {
None => Ok(None),
Some(v) => Ok(Some(v.parse::<T>().map_err(de::Error::custom)?)),
}
} else {
Option::<T>::deserialize(d)
}
}
}

151
veilid-core/src/veilid_api/serialize_helpers/veilid_rkyv.rs
View File

@ -1,151 +0,0 @@
use super::*;
use rkyv::ser::Serializer;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub struct VeilidRkyvSerializer<S> {
inner: S,
}
impl<S> VeilidRkyvSerializer<S> {
pub fn into_inner(self) -> S {
self.inner
}
}
impl<S: rkyv::Fallible> rkyv::Fallible for VeilidRkyvSerializer<S> {
type Error = VeilidRkyvError<S::Error>;
}
impl<S: rkyv::ser::ScratchSpace> rkyv::ser::ScratchSpace for VeilidRkyvSerializer<S> {
unsafe fn push_scratch(
&mut self,
layout: core::alloc::Layout,
) -> Result<core::ptr::NonNull<[u8]>, Self::Error> {
self.inner
.push_scratch(layout)
.map_err(VeilidRkyvError::Inner)
}
unsafe fn pop_scratch(
&mut self,
ptr: core::ptr::NonNull<u8>,
layout: core::alloc::Layout,
) -> Result<(), Self::Error> {
self.inner
.pop_scratch(ptr, layout)
.map_err(VeilidRkyvError::Inner)
}
}
impl<S: rkyv::ser::Serializer> rkyv::ser::Serializer for VeilidRkyvSerializer<S> {
#[inline]
fn pos(&self) -> usize {
self.inner.pos()
}
#[inline]
fn write(&mut self, bytes: &[u8]) -> Result<(), Self::Error> {
self.inner.write(bytes).map_err(VeilidRkyvError::Inner)
}
}
impl<S: Default> Default for VeilidRkyvSerializer<S> {
fn default() -> Self {
Self {
inner: S::default(),
}
}
}
pub type DefaultVeilidRkyvSerializer =
VeilidRkyvSerializer<rkyv::ser::serializers::AllocSerializer<1024>>;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#[derive(Debug, Default)]
pub struct VeilidSharedDeserializeMap {
inner: SharedDeserializeMap,
}
impl VeilidSharedDeserializeMap {
#[inline]
pub fn new() -> Self {
Self {
inner: SharedDeserializeMap::new(),
}
}
}
impl rkyv::Fallible for VeilidSharedDeserializeMap {
type Error = VeilidRkyvError<rkyv::de::deserializers::SharedDeserializeMapError>;
}
impl rkyv::de::SharedDeserializeRegistry for VeilidSharedDeserializeMap {
fn get_shared_ptr(&mut self, ptr: *const u8) -> Option<&dyn rkyv::de::SharedPointer> {
self.inner.get_shared_ptr(ptr)
}
fn add_shared_ptr(
&mut self,
ptr: *const u8,
shared: Box<dyn rkyv::de::SharedPointer>,
) -> Result<(), Self::Error> {
self.inner
.add_shared_ptr(ptr, shared)
.map_err(VeilidRkyvError::Inner)
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#[derive(Debug)]
pub enum VeilidRkyvError<E> {
Inner(E),
StringError(String),
}
impl<E: Debug> From<String> for VeilidRkyvError<E> {
fn from(s: String) -> Self {
Self::StringError(s)
}
}
impl<E: Debug + fmt::Display> fmt::Display for VeilidRkyvError<E> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
VeilidRkyvError::Inner(e) => write!(f, "Inner: {}", e),
VeilidRkyvError::StringError(s) => write!(f, "StringError: {}", s),
}
}
}
impl<E: Debug + fmt::Display> std::error::Error for VeilidRkyvError<E> {}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn to_rkyv<T>(value: &T) -> VeilidAPIResult<Vec<u8>>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let mut serializer = DefaultVeilidRkyvSerializer::default();
serializer
.serialize_value(value)
.map_err(|e| VeilidAPIError::generic(format!("failed to serialize object: {}", e)))?;
Ok(serializer
.into_inner()
.into_serializer()
.into_inner()
.to_vec())
}
pub fn from_rkyv<T>(bytes: Vec<u8>) -> VeilidAPIResult<T>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
rkyv::check_archived_root::<T>(&bytes)
.map_err(|e| VeilidAPIError::generic(format!("checkbytes failed: {}", e)))?
.deserialize(&mut VeilidSharedDeserializeMap::default())
.map_err(|e| VeilidAPIError::generic(format!("failed to deserialize: {}", e)))
}

1
veilid-core/src/veilid_api/tests/mod.rs
View File

@ -1,6 +1,5 @@
mod fixtures;
pub mod test_serialize_json;
pub mod test_serialize_rkyv;
mod test_types;
mod test_types_dht;
mod test_types_dht_schema;

16
veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs
View File

@ -1,16 +0,0 @@
use crate::*;
pub async fn test_simple_string() {
let plain = "basic string".to_string();
let serialized = b"basic string\x0c\x00\x00\x00\xf4\xff\xff\xff".to_vec();
let a = to_rkyv(&plain);
assert_eq!(a.unwrap(), serialized);
let b = from_rkyv::<String>(serialized);
assert_eq!(b.unwrap(), plain);
}
pub async fn test_all() {
test_simple_string().await;
}

21
veilid-core/src/veilid_api/types/aligned_u64.rs
View File

@ -2,29 +2,16 @@ use super::*;
/// Aligned u64
/// Required on 32-bit platforms for serialization because Rust aligns u64 on 4 byte boundaries
/// And zero-copy serialization with Rkyv requires 8-byte alignment
/// Some zero-copy serialization frameworks also want 8-byte alignment
/// Supports serializing to string for JSON as well, since JSON can't handle 64-bit numbers to Javascript
#[derive(
Clone,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Copy,
Hash,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
Clone, Default, PartialEq, Eq, PartialOrd, Ord, Copy, Hash, Serialize, Deserialize, JsonSchema,
)]
#[repr(C, align(8))]
#[archive_attr(repr(C, align(8)), derive(CheckBytes))]
#[serde(transparent)]
pub struct AlignedU64(
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
#[schemars(with = "String")]
u64,
);

38
veilid-core/src/veilid_api/types/app_message_call.rs
View File

@ -1,27 +1,15 @@
use super::*;
/// Direct statement blob passed to hosting application for processing
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidAppMessage {
/// Some(sender) if the message was sent directly, None if received via a private/safety route
#[serde(with = "opt_json_as_string")]
#[serde(with = "as_human_opt_string")]
#[schemars(with = "Option<String>")]
pub sender: Option<TypedKey>,
/// The content of the message to deliver to the application
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
pub message: Vec<u8>,
}
@ -40,32 +28,20 @@ impl VeilidAppMessage {
}
/// Direct question blob passed to hosting application for processing to send an eventual AppReply
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct VeilidAppCall {
/// Some(sender) if the request was sent directly, None if received via a private/safety route
#[serde(with = "opt_json_as_string")]
#[serde(with = "as_human_opt_string")]
#[schemars(with = "Option<String>")]
sender: Option<TypedKey>,
/// The content of the request to deliver to the application
#[serde(with = "json_as_base64")]
#[serde(with = "as_human_base64")]
#[schemars(with = "String")]
message: Vec<u8>,
/// The id to reply to
#[serde(with = "json_as_string")]
#[serde(with = "as_human_string")]
#[schemars(with = "String")]
call_id: OperationId,
}

16
veilid-core/src/veilid_api/types/dht/dht_record_descriptor.rs
View File

@ -1,21 +1,7 @@
use super::*;
/// DHT Record Descriptor
#[derive(
Debug,
Clone,
PartialOrd,
Ord,
PartialEq,
Eq,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct DHTRecordDescriptor {
/// DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
#[schemars(with = "String")]

16
veilid-core/src/veilid_api/types/dht/schema/dflt.rs
View File

@ -1,21 +1,7 @@
use super::*;
/// Default DHT Schema (DFLT)
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Ord,
PartialOrd,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(C), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, JsonSchema)]
pub struct DHTSchemaDFLT {
/// Owner subkey count
pub o_cnt: u16,

16
veilid-core/src/veilid_api/types/dht/schema/mod.rs
View File

@ -7,21 +7,7 @@ pub use dflt::*;
pub use smpl::*;
/// Enum over all the supported DHT Schemas
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Ord,
PartialOrd,
Serialize,
Deserialize,
RkyvArchive,
RkyvSerialize,
RkyvDeserialize,
JsonSchema,
)]
#[archive_attr(repr(u8), derive(CheckBytes))]
#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "kind")]
pub enum DHTSchema {
DFLT(DHTSchemaDFLT),

Some files were not shown because too many files have changed in this diff Show More