Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2025-02-18 13:24:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'startup-lock' into 'main'

Browse Source 
Fix startup/shutdown/attach/detach

See merge request veilid/veilid!302
This commit is contained in:
Christien Rioux 2024-07-22 14:29:44 +00:00
commit e759e50983
85 changed files with 1556 additions and 459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
Cargo.lock generated
View File

@ -243,9 +243,9 @@ dependencies = [
[[package]] [[package]]
name = "arrayref" name = "arrayref"
version = "0.3.7" version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a"
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
@ -470,7 +470,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -500,7 +500,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -745,7 +745,7 @@ checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -831,6 +831,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "byteorder-lite"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.6.1" version = "1.6.1"
@ -857,9 +863,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.1.5" version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "324c74f2155653c90b04f25b2a47a8a631360cb908f92a772695f430c7e31052" checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
[[package]] [[package]]
name = "cesu8" name = "cesu8"
@ -1016,7 +1022,7 @@ dependencies = [
"heck 0.5.0", "heck 0.5.0",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1375,7 +1381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1488,7 +1494,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1544,7 +1550,7 @@ dependencies = [
"ident_case", "ident_case",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1566,7 +1572,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [ dependencies = [
"darling_core 0.20.10", "darling_core 0.20.10",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1733,7 +1739,7 @@ dependencies = [
"heck 0.4.1", "heck 0.4.1",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1753,7 +1759,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1796,7 +1802,7 @@ dependencies = [
"darling 0.20.10", "darling 0.20.10",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -1958,6 +1964,12 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "fixedbitset"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]] [[package]]
name = "flate2" name = "flate2"
version = "1.0.30" version = "1.0.30"
@ -2028,7 +2040,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -2140,7 +2152,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -2680,12 +2692,12 @@ dependencies = [
[[package]] [[package]]
name = "image" name = "image"
version = "0.25.1" version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10"
dependencies = [ dependencies = [
"bytemuck", "bytemuck",
"byteorder", "byteorder-lite",
"num-traits", "num-traits",
"png", "png",
"tiff", "tiff",
@ -2969,9 +2981,9 @@ dependencies = [
[[package]] [[package]]
name = "libloading" name = "libloading"
version = "0.8.4" version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"windows-targets 0.52.6", "windows-targets 0.52.6",
@ -3147,6 +3159,12 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "multimap"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
[[package]] [[package]]
name = "nanorand" name = "nanorand"
version = "0.7.0" version = "0.7.0"
@ -4005,7 +4023,7 @@ dependencies = [
"pest_meta", "pest_meta",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4019,6 +4037,16 @@ dependencies = [
"sha2 0.10.8", "sha2 0.10.8",
] ]
[[package]]
name = "petgraph"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset",
"indexmap 2.2.6",
]
[[package]] [[package]]
name = "pharos" name = "pharos"
version = "0.5.3" version = "0.5.3"
@ -4046,7 +4074,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4145,9 +4173,9 @@ dependencies = [
[[package]] [[package]]
name = "portable-atomic" name = "portable-atomic"
version = "1.6.0" version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
[[package]] [[package]]
name = "portable-atomic-util" name = "portable-atomic-util"
@ -4170,6 +4198,16 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "prettyplease"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
dependencies = [
"proc-macro2",
"syn 2.0.72",
]
[[package]] [[package]]
name = "proc-macro-crate" name = "proc-macro-crate"
version = "0.1.5" version = "0.1.5"
@ -4218,6 +4256,27 @@ dependencies = [
"prost-derive 0.12.6", "prost-derive 0.12.6",
] ]
[[package]]
name = "prost-build"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.5.0",
"itertools 0.12.1",
"log",
"multimap",
"once_cell",
"petgraph",
"prettyplease",
"prost 0.12.6",
"prost-types",
"regex",
"syn 2.0.72",
"tempfile",
]
[[package]] [[package]]
name = "prost-derive" name = "prost-derive"
version = "0.11.9" version = "0.11.9"
@ -4241,7 +4300,7 @@ dependencies = [
"itertools 0.12.1", "itertools 0.12.1",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4259,6 +4318,15 @@ version = "2.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
[[package]]
name = "protobuf-src"
version = "2.0.1+26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ba1cfa4b9dc098926b8cce388bf434b93516db3ecf6e8b1a37eb643d733ee7"
dependencies = [
"cmake",
]
[[package]] [[package]]
name = "quick-error" name = "quick-error"
version = "1.2.3" version = "1.2.3"
@ -4692,7 +4760,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"serde_derive_internals 0.29.1", "serde_derive_internals 0.29.1",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4719,9 +4787,9 @@ dependencies = [
[[package]] [[package]]
name = "sdd" name = "sdd"
version = "1.6.0" version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb0dde0ccd15e337a3cf738a9a38115c6d8e74795d074e73973dad3d229a897" checksum = "85f05a494052771fc5bd0619742363b5e24e5ad72ab3111ec2e27925b8edc5f3"
[[package]] [[package]]
name = "secret-service" name = "secret-service"
@ -4854,7 +4922,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4865,7 +4933,7 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4876,7 +4944,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4898,7 +4966,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4959,7 +5027,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -4970,7 +5038,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -5222,9 +5290,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.71" version = "2.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -5252,6 +5320,18 @@ dependencies = [
"windows 0.52.0", "windows 0.52.0",
] ]
[[package]]
name = "tempfile"
version = "3.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
dependencies = [
"cfg-if 1.0.0",
"fastrand 2.1.0",
"rustix 0.38.34",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "termcolor" name = "termcolor"
version = "1.4.1" version = "1.4.1"
@ -5307,7 +5387,17 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
]
[[package]]
name = "thread-id"
version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe8f25bbdd100db7e1d34acf7fd2dc59c4bf8f7483f505eaa7d4f12f76cc0ea"
dependencies = [
"libc",
"winapi",
] ]
[[package]] [[package]]
@ -5426,7 +5516,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -5505,7 +5595,7 @@ dependencies = [
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
"winnow 0.6.13", "winnow 0.6.14",
] ]
[[package]] [[package]]
@ -5627,7 +5717,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -5744,6 +5834,24 @@ dependencies = [
"tracing-subscriber", "tracing-subscriber",
] ]
[[package]]
name = "tracing-perfetto"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd21777b526dfcb57f11f65aa8a2024d83e1db52841993229b6e282e511978b7"
dependencies = [
"anyhow",
"bytes",
"chrono",
"prost 0.12.6",
"prost-build",
"protobuf-src",
"rand",
"thread-id",
"tracing",
"tracing-subscriber",
]
[[package]] [[package]]
name = "tracing-subscriber" name = "tracing-subscriber"
version = "0.3.18" version = "0.3.18"
@ -5812,7 +5920,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"serde_derive_internals 0.28.0", "serde_derive_internals 0.28.0",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -6034,7 +6142,6 @@ version = "0.3.3"
dependencies = [ dependencies = [
"argon2", "argon2",
"async-io 1.13.0", "async-io 1.13.0",
"async-lock 2.8.0",
"async-std", "async-std",
"async-std-resolver", "async-std-resolver",
"async-tls", "async-tls",
@ -6234,6 +6341,7 @@ dependencies = [
"tracing-flame", "tracing-flame",
"tracing-journald", "tracing-journald",
"tracing-opentelemetry 0.24.0", "tracing-opentelemetry 0.24.0",
"tracing-perfetto",
"tracing-subscriber", "tracing-subscriber",
"url", "url",
"veilid-bugsalot", "veilid-bugsalot",
@ -6247,7 +6355,7 @@ name = "veilid-tools"
version = "0.3.3" version = "0.3.3"
dependencies = [ dependencies = [
"android_logger 0.13.3", "android_logger 0.13.3",
"async-lock 2.8.0", "async-lock 3.4.0",
"async-std", "async-std",
"async_executors", "async_executors",
"backtrace", "backtrace",
@ -6388,7 +6496,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -6422,7 +6530,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -6455,7 +6563,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -6875,9 +6983,9 @@ dependencies = [
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.6.13" version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -7022,7 +7130,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]
@ -7042,7 +7150,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.71", "syn 2.0.72",
] ]
[[package]] [[package]]

8
Earthfile
View File

@ -16,6 +16,8 @@ VERSION 0.7
FROM ubuntu:18.04 FROM ubuntu:18.04
ENV ZIG_VERSION=0.13.0-dev.46+3648d7df1 ENV ZIG_VERSION=0.13.0-dev.46+3648d7df1
ENV CMAKE_VERSION_MINOR=3.30
ENV CMAKE_VERSION_PATCH=3.30.1
ENV RUSTUP_HOME=/usr/local/rustup ENV RUSTUP_HOME=/usr/local/rustup
ENV RUSTUP_DIST_SERVER=https://static.rust-lang.org ENV RUSTUP_DIST_SERVER=https://static.rust-lang.org
ENV CARGO_HOME=/usr/local/cargo ENV CARGO_HOME=/usr/local/cargo
@ -28,7 +30,11 @@ WORKDIR /veilid
# Install build prerequisites & setup required directories # Install build prerequisites & setup required directories
deps-base: deps-base:
RUN apt-get -y update RUN apt-get -y update
RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev RUN apt-get install -y iproute2 curl build-essential libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
RUN curl -O https://cmake.org/files/v$CMAKE_VERSION_MINOR/cmake-$CMAKE_VERSION_PATCH-linux-$(arch).sh
RUN mkdir /opt/cmake
RUN sh cmake-$CMAKE_VERSION_PATCH-linux-$(arch).sh --skip-license --prefix=/opt/cmake
RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
# Install Rust # Install Rust
deps-rust: deps-rust:

6
veilid-cli/Cargo.toml
View File

@ -22,12 +22,12 @@ rt-async-std = [
rt-tokio = ["tokio", "tokio-util", "veilid-tools/rt-tokio", "cursive/rt-tokio"] rt-tokio = ["tokio", "tokio-util", "veilid-tools/rt-tokio", "cursive/rt-tokio"]
[dependencies] [dependencies]
async-std = { version = "^1.12", features = [ async-std = { version = "1.12.0", features = [
"unstable", "unstable",
"attributes", "attributes",
], optional = true } ], optional = true }
tokio = { version = "^1", features = ["full"], optional = true } tokio = { version = "1.38.1", features = ["full", "tracing"], optional = true }
tokio-util = { version = "^0", features = ["compat"], optional = true } tokio-util = { version = "0.7.11", features = ["compat"], optional = true }
async-tungstenite = { version = "^0.23" } async-tungstenite = { version = "^0.23" }
cursive = { git = "https://gitlab.com/veilid/cursive.git", default-features = false, features = [ cursive = { git = "https://gitlab.com/veilid/cursive.git", default-features = false, features = [
"crossterm", "crossterm",

2
veilid-cli/src/client_api_connection.rs
View File

@ -187,7 +187,7 @@ impl ClientApiConnection {
// Request initial server state // Request initial server state
let capi = self.clone(); let capi = self.clone();
spawn_detached_local(async move { spawn_detached_local("get initial server state", async move {
let mut req = json::JsonValue::new_object(); let mut req = json::JsonValue::new_object();
req["op"] = "GetState".into(); req["op"] = "GetState".into();
let Some(resp) = capi.perform_request(req).await else { let Some(resp) = capi.perform_request(req).await else {

20
veilid-cli/src/command_processor.rs
View File

@ -114,7 +114,7 @@ impl CommandProcessor {
trace!("CommandProcessor::cmd_help"); trace!("CommandProcessor::cmd_help");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd help", async move {
let out = match capi.server_debug("help".to_owned()).await { let out = match capi.server_debug("help".to_owned()).await {
Err(e) => { Err(e) => {
error!("Server command 'debug help' failed: {}", e); error!("Server command 'debug help' failed: {}", e);
@ -166,7 +166,7 @@ Server Debug Commands:
trace!("CommandProcessor::cmd_shutdown"); trace!("CommandProcessor::cmd_shutdown");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd shutdown", async move {
if let Err(e) = capi.server_shutdown().await { if let Err(e) = capi.server_shutdown().await {
error!("Server command 'shutdown' failed to execute: {}", e); error!("Server command 'shutdown' failed to execute: {}", e);
} }
@ -179,7 +179,7 @@ Server Debug Commands:
trace!("CommandProcessor::cmd_disconnect"); trace!("CommandProcessor::cmd_disconnect");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd disconnect", async move {
capi.disconnect().await; capi.disconnect().await;
ui.send_callback(callback); ui.send_callback(callback);
}); });
@ -190,7 +190,7 @@ Server Debug Commands:
trace!("CommandProcessor::cmd_debug"); trace!("CommandProcessor::cmd_debug");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd debug", async move {
match capi.server_debug(command_line).await { match capi.server_debug(command_line).await {
Ok(output) => { Ok(output) => {
ui.add_node_event(Level::Info, &output); ui.add_node_event(Level::Info, &output);
@ -213,7 +213,7 @@ Server Debug Commands:
trace!("CommandProcessor::cmd_change_log_level"); trace!("CommandProcessor::cmd_change_log_level");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd change_log_level", async move {
let (layer, rest) = Self::word_split(&rest.unwrap_or_default()); let (layer, rest) = Self::word_split(&rest.unwrap_or_default());
let log_level = match convert_loglevel(&rest.unwrap_or_default()) { let log_level = match convert_loglevel(&rest.unwrap_or_default()) {
Ok(v) => v, Ok(v) => v,
@ -252,7 +252,7 @@ Server Debug Commands:
trace!("CommandProcessor::cmd_change_log_ignore"); trace!("CommandProcessor::cmd_change_log_ignore");
let capi = self.capi(); let capi = self.capi();
let ui = self.ui_sender(); let ui = self.ui_sender();
spawn_detached_local(async move { spawn_detached_local("cmd change_log_ignoe", async move {
let (layer, rest) = Self::word_split(&rest.unwrap_or_default()); let (layer, rest) = Self::word_split(&rest.unwrap_or_default());
let log_ignore = rest.unwrap_or_default(); let log_ignore = rest.unwrap_or_default();
@ -284,7 +284,7 @@ Server Debug Commands:
let ui = self.ui_sender(); let ui = self.ui_sender();
let this = self.clone(); let this = self.clone();
spawn_detached_local(async move { spawn_detached_local("cmd enable", async move {
let flag = rest.clone().unwrap_or_default(); let flag = rest.clone().unwrap_or_default();
match flag.as_str() { match flag.as_str() {
"app_messages" => { "app_messages" => {
@ -306,7 +306,7 @@ Server Debug Commands:
let ui = self.ui_sender(); let ui = self.ui_sender();
let this = self.clone(); let this = self.clone();
spawn_detached_local(async move { spawn_detached_local("cmd disable", async move {
let flag = rest.clone().unwrap_or_default(); let flag = rest.clone().unwrap_or_default();
match flag.as_str() { match flag.as_str() {
"app_messages" => { "app_messages" => {
@ -664,7 +664,7 @@ Server Debug Commands:
pub fn attach(&self) { pub fn attach(&self) {
let capi = self.capi(); let capi = self.capi();
spawn_detached_local(async move { spawn_detached_local("attach", async move {
if let Err(e) = capi.server_attach().await { if let Err(e) = capi.server_attach().await {
error!("Server command 'attach' failed to execute: {}", e); error!("Server command 'attach' failed to execute: {}", e);
} }
@ -674,7 +674,7 @@ Server Debug Commands:
pub fn detach(&self) { pub fn detach(&self) {
let capi = self.capi(); let capi = self.capi();
spawn_detached_local(async move { spawn_detached_local("detach", async move {
if let Err(e) = capi.server_detach().await { if let Err(e) = capi.server_detach().await {
error!("Server command 'detach' failed to execute: {}", e); error!("Server command 'detach' failed to execute: {}", e);
} }

1
veilid-cli/src/log_viewer_ui.rs
View File

@ -86,6 +86,7 @@ impl LogViewerUI {
done.await; done.await;
} else { } else {
while let Ok(Ok(c)) = blocking_wrapper( while let Ok(Ok(c)) = blocking_wrapper(
"LogViewerUI read",
{ {
let term = term.clone(); let term = term.clone();
move || term.read_char() move || term.read_char()

2
veilid-core/Cargo.toml
View File

@ -51,6 +51,7 @@ crypto-test = ["enable-crypto-vld0", "enable-crypto-none"]
crypto-test-none = ["enable-crypto-none"] crypto-test-none = ["enable-crypto-none"]
veilid_core_android_tests = ["dep:paranoid-android"] veilid_core_android_tests = ["dep:paranoid-android"]
veilid_core_ios_tests = ["dep:tracing-oslog"] veilid_core_ios_tests = ["dep:tracing-oslog"]
debug-locks = ["veilid-tools/debug-locks"]
### DEPENDENCIES ### DEPENDENCIES
@ -191,7 +192,6 @@ async_executors = { version = "0.7.0", default-features = false, features = [
"bindgen", "bindgen",
"timer", "timer",
] } ] }
async-lock = "2.8.0"
wasm-bindgen = "0.2.92" wasm-bindgen = "0.2.92"
js-sys = "0.3.69" js-sys = "0.3.69"
wasm-bindgen-futures = "0.4.42" wasm-bindgen-futures = "0.4.42"

7
veilid-core/src/attachment_manager.rs
View File

@ -211,7 +211,7 @@ impl AttachmentManager {
} }
} }
#[instrument(level = "debug", skip_all)] #[instrument(parent = None, level = "debug", skip_all)]
async fn attachment_maintainer(self) { async fn attachment_maintainer(self) {
log_net!(debug "attachment starting"); log_net!(debug "attachment starting");
self.update_attaching_detaching_state(AttachmentState::Attaching); self.update_attaching_detaching_state(AttachmentState::Attaching);
@ -323,7 +323,10 @@ impl AttachmentManager {
return false; return false;
} }
inner.maintain_peers = true; inner.maintain_peers = true;
inner.attachment_maintainer_jh = Some(spawn(self.clone().attachment_maintainer())); inner.attachment_maintainer_jh = Some(spawn(
"attachment maintainer",
self.clone().attachment_maintainer(),
));
true true
} }

2
veilid-core/src/crypto/mod.rs
View File

@ -176,7 +176,7 @@ impl Crypto {
// Schedule flushing // Schedule flushing
let this = self.clone(); let this = self.clone();
let flush_future = interval(60000, move || { let flush_future = interval("crypto flush", 60000, move || {
let this = this.clone(); let this = this.clone();
async move { async move {
if let Err(e) = this.flush().await { if let Err(e) = this.flush().await {

1
veilid-core/src/network_manager/address_filter.rs
View File

@ -337,6 +337,7 @@ impl AddressFilter {
.or_insert(punishment); .or_insert(punishment);
} }
#[instrument(parent = None, level = "trace", skip_all, err)]
pub async fn address_filter_task_routine( pub async fn address_filter_task_routine(
self, self,
_stop_token: StopToken, _stop_token: StopToken,

38
veilid-core/src/network_manager/connection_manager.rs
View File

@ -53,6 +53,7 @@ struct ConnectionManagerArc {
connection_inactivity_timeout_ms: u32, connection_inactivity_timeout_ms: u32,
connection_table: ConnectionTable, connection_table: ConnectionTable,
address_lock_table: AsyncTagLockTable<SocketAddr>, address_lock_table: AsyncTagLockTable<SocketAddr>,
startup_lock: StartupLock,
inner: Mutex<Option<ConnectionManagerInner>>, inner: Mutex<Option<ConnectionManagerInner>>,
} }
impl core::fmt::Debug for ConnectionManagerArc { impl core::fmt::Debug for ConnectionManagerArc {
@ -98,6 +99,7 @@ impl ConnectionManager {
connection_inactivity_timeout_ms, connection_inactivity_timeout_ms,
connection_table: ConnectionTable::new(config, address_filter), connection_table: ConnectionTable::new(config, address_filter),
address_lock_table: AsyncTagLockTable::new(), address_lock_table: AsyncTagLockTable::new(),
startup_lock: StartupLock::new(),
inner: Mutex::new(None), inner: Mutex::new(None),
} }
} }
@ -115,8 +117,11 @@ impl ConnectionManager {
self.arc.connection_inactivity_timeout_ms self.arc.connection_inactivity_timeout_ms
} }
pub async fn startup(&self) { pub async fn startup(&self) -> EyreResult<()> {
let guard = self.arc.startup_lock.startup()?;
log_net!(debug "startup connection manager"); log_net!(debug "startup connection manager");
let mut inner = self.arc.inner.lock(); let mut inner = self.arc.inner.lock();
if inner.is_some() { if inner.is_some() {
panic!("shouldn't start connection manager twice without shutting it down first"); panic!("shouldn't start connection manager twice without shutting it down first");
@ -129,14 +134,26 @@ impl ConnectionManager {
let stop_source = StopSource::new(); let stop_source = StopSource::new();
// Spawn the async processor // Spawn the async processor
let async_processor = spawn(self.clone().async_processor(stop_source.token(), receiver)); let async_processor = spawn(
"connection manager async processor",
self.clone().async_processor(stop_source.token(), receiver),
);
// Store in the inner object // Store in the inner object
*inner = Some(Self::new_inner(stop_source, sender, async_processor)); *inner = Some(Self::new_inner(stop_source, sender, async_processor));
guard.success();
Ok(())
} }
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
log_net!(debug "starting connection manager shutdown"); log_net!(debug "starting connection manager shutdown");
let Ok(guard) = self.arc.startup_lock.shutdown().await else {
log_net!(debug "connection manager is already shut down");
return;
};
// Remove the inner from the lock // Remove the inner from the lock
let mut inner = { let mut inner = {
let mut inner_lock = self.arc.inner.lock(); let mut inner_lock = self.arc.inner.lock();
@ -158,6 +175,8 @@ impl ConnectionManager {
// Wait for the connections to complete // Wait for the connections to complete
log_net!(debug "waiting for connection handlers to complete"); log_net!(debug "waiting for connection handlers to complete");
self.arc.connection_table.join().await; self.arc.connection_table.join().await;
guard.success();
log_net!(debug "finished connection manager shutdown"); log_net!(debug "finished connection manager shutdown");
} }
@ -263,6 +282,9 @@ impl ConnectionManager {
// Returns a network connection if one already is established // Returns a network connection if one already is established
pub fn get_connection(&self, flow: Flow) -> Option<ConnectionHandle> { pub fn get_connection(&self, flow: Flow) -> Option<ConnectionHandle> {
let Ok(_guard) = self.arc.startup_lock.enter() else {
return None;
};
self.arc.connection_table.peek_connection_by_flow(flow) self.arc.connection_table.peek_connection_by_flow(flow)
} }
@ -276,6 +298,9 @@ impl ConnectionManager {
self.arc.connection_table.ref_connection_by_id(id, kind) self.arc.connection_table.ref_connection_by_id(id, kind)
} }
pub fn try_connection_ref_scope(&self, id: NetworkConnectionId) -> Option<ConnectionRefScope> { pub fn try_connection_ref_scope(&self, id: NetworkConnectionId) -> Option<ConnectionRefScope> {
let Ok(_guard) = self.arc.startup_lock.enter() else {
return None;
};
ConnectionRefScope::try_new(self.clone(), id) ConnectionRefScope::try_new(self.clone(), id)
} }
@ -288,6 +313,11 @@ impl ConnectionManager {
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
) -> EyreResult<NetworkResult<ConnectionHandle>> { ) -> EyreResult<NetworkResult<ConnectionHandle>> {
let Ok(_guard) = self.arc.startup_lock.enter() else {
return Ok(NetworkResult::service_unavailable(
"connection manager is not started",
));
};
let peer_address = dial_info.peer_address(); let peer_address = dial_info.peer_address();
let remote_addr = peer_address.socket_addr(); let remote_addr = peer_address.socket_addr();
let mut preferred_local_address = self let mut preferred_local_address = self
@ -387,6 +417,10 @@ impl ConnectionManager {
if !allow_accept { if !allow_accept {
return; return;
} }
let Ok(_guard) = self.arc.startup_lock.enter() else {
return;
};
// Async lock on the remote address for atomicity per remote // Async lock on the remote address for atomicity per remote
let _lock_guard = self let _lock_guard = self
.arc .arc

112
veilid-core/src/network_manager/mod.rs
View File

@ -139,6 +139,7 @@ enum SendDataToExistingFlowResult {
#[derive(Copy, Clone, Debug, Eq, PartialEq)] #[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum StartupDisposition { pub enum StartupDisposition {
Success, Success,
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
BindRetry, BindRetry,
} }
@ -172,6 +173,8 @@ struct NetworkManagerUnlockedInner {
address_filter_task: TickTask<EyreReport>, address_filter_task: TickTask<EyreReport>,
// Network Key // Network Key
network_key: Option<SharedSecret>, network_key: Option<SharedSecret>,
// Startup Lock
startup_lock: StartupLock,
} }
#[derive(Clone)] #[derive(Clone)]
@ -209,10 +212,20 @@ impl NetworkManager {
routing_table: RwLock::new(None), routing_table: RwLock::new(None),
components: RwLock::new(None), components: RwLock::new(None),
update_callback: RwLock::new(None), update_callback: RwLock::new(None),
rolling_transfers_task: TickTask::new(ROLLING_TRANSFERS_INTERVAL_SECS), rolling_transfers_task: TickTask::new(
public_address_check_task: TickTask::new(PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS), "rolling_transfers_task",
address_filter_task: TickTask::new(ADDRESS_FILTER_TASK_INTERVAL_SECS), ROLLING_TRANSFERS_INTERVAL_SECS,
),
public_address_check_task: TickTask::new(
"public_address_check_task",
PUBLIC_ADDRESS_CHECK_TASK_INTERVAL_SECS,
),
address_filter_task: TickTask::new(
"address_filter_task",
ADDRESS_FILTER_TASK_INTERVAL_SECS,
),
network_key, network_key,
startup_lock: StartupLock::new(),
} }
} }
@ -428,7 +441,7 @@ impl NetworkManager {
}); });
// Start network components // Start network components
connection_manager.startup().await; connection_manager.startup().await?;
match net.startup().await? { match net.startup().await? {
StartupDisposition::Success => {} StartupDisposition::Success => {}
StartupDisposition::BindRetry => { StartupDisposition::BindRetry => {
@ -445,27 +458,30 @@ impl NetworkManager {
#[instrument(level = "debug", skip_all, err)] #[instrument(level = "debug", skip_all, err)]
pub async fn startup(&self) -> EyreResult<StartupDisposition> { pub async fn startup(&self) -> EyreResult<StartupDisposition> {
let guard = self.unlocked_inner.startup_lock.startup()?;
match self.internal_startup().await { match self.internal_startup().await {
Ok(StartupDisposition::Success) => { Ok(StartupDisposition::Success) => {
guard.success();
// Inform api clients that things have changed // Inform api clients that things have changed
self.send_network_update(); self.send_network_update();
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
Ok(StartupDisposition::BindRetry) => { Ok(StartupDisposition::BindRetry) => {
self.shutdown().await; self.shutdown_internal().await;
Ok(StartupDisposition::BindRetry) Ok(StartupDisposition::BindRetry)
} }
Err(e) => { Err(e) => {
self.shutdown().await; self.shutdown_internal().await;
Err(e) Err(e)
} }
} }
} }
#[instrument(level = "debug", skip_all)] #[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) { async fn shutdown_internal(&self) {
log_net!(debug "starting network manager shutdown");
// Cancel all tasks // Cancel all tasks
self.cancel_tasks().await; self.cancel_tasks().await;
@ -487,6 +503,20 @@ impl NetworkManager {
{ {
*self.inner.lock() = NetworkManager::new_inner(); *self.inner.lock() = NetworkManager::new_inner();
} }
}
#[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) {
log_net!(debug "starting network manager shutdown");
let Ok(guard) = self.unlocked_inner.startup_lock.shutdown().await else {
log_net!(debug "network manager is already shut down");
return;
};
self.shutdown_internal().await;
guard.success();
// send update // send update
log_net!(debug "sending network state update to api clients"); log_net!(debug "sending network state update to api clients");
@ -546,9 +576,7 @@ impl NetworkManager {
} }
pub fn network_is_started(&self) -> bool { pub fn network_is_started(&self) -> bool {
self.opt_net() self.opt_net().map(|net| net.is_started()).unwrap_or(false)
.and_then(|net| net.is_started())
.unwrap_or(false)
} }
pub fn generate_node_status(&self, _routing_domain: RoutingDomain) -> NodeStatus { pub fn generate_node_status(&self, _routing_domain: RoutingDomain) -> NodeStatus {
@ -556,7 +584,7 @@ impl NetworkManager {
} }
/// Generates a multi-shot/normal receipt /// Generates a multi-shot/normal receipt
#[instrument(level = "trace", skip(self, extra_data, callback), err)] #[instrument(level = "trace", skip(self, extra_data, callback))]
pub fn generate_receipt<D: AsRef<[u8]>>( pub fn generate_receipt<D: AsRef<[u8]>>(
&self, &self,
expiration_us: u64, expiration_us: u64,
@ -564,6 +592,9 @@ impl NetworkManager {
extra_data: D, extra_data: D,
callback: impl ReceiptCallback, callback: impl ReceiptCallback,
) -> EyreResult<Vec<u8>> { ) -> EyreResult<Vec<u8>> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
bail!("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let routing_table = self.routing_table(); let routing_table = self.routing_table();
@ -593,12 +624,16 @@ impl NetworkManager {
} }
/// Generates a single-shot/normal receipt /// Generates a single-shot/normal receipt
#[instrument(level = "trace", skip(self, extra_data), err)] #[instrument(level = "trace", skip(self, extra_data))]
pub fn generate_single_shot_receipt<D: AsRef<[u8]>>( pub fn generate_single_shot_receipt<D: AsRef<[u8]>>(
&self, &self,
expiration_us: u64, expiration_us: u64,
extra_data: D, extra_data: D,
) -> EyreResult<(Vec<u8>, EventualValueFuture<ReceiptEvent>)> { ) -> EyreResult<(Vec<u8>, EventualValueFuture<ReceiptEvent>)> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
bail!("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let routing_table = self.routing_table(); let routing_table = self.routing_table();
@ -635,6 +670,10 @@ impl NetworkManager {
&self, &self,
receipt_data: R, receipt_data: R,
) -> NetworkResult<()> { ) -> NetworkResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return NetworkResult::service_unavailable("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) { let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) {
@ -656,6 +695,10 @@ impl NetworkManager {
receipt_data: R, receipt_data: R,
inbound_noderef: NodeRef, inbound_noderef: NodeRef,
) -> NetworkResult<()> { ) -> NetworkResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return NetworkResult::service_unavailable("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) { let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) {
@ -676,6 +719,10 @@ impl NetworkManager {
&self, &self,
receipt_data: R, receipt_data: R,
) -> NetworkResult<()> { ) -> NetworkResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return NetworkResult::service_unavailable("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) { let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) {
@ -697,6 +744,10 @@ impl NetworkManager {
receipt_data: R, receipt_data: R,
private_route: PublicKey, private_route: PublicKey,
) -> NetworkResult<()> { ) -> NetworkResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return NetworkResult::service_unavailable("network is not started");
};
let receipt_manager = self.receipt_manager(); let receipt_manager = self.receipt_manager();
let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) { let receipt = match Receipt::from_signed_data(self.crypto(), receipt_data.as_ref()) {
@ -712,12 +763,16 @@ impl NetworkManager {
} }
// Process a received signal // Process a received signal
#[instrument(level = "trace", target = "receipt", skip_all)] #[instrument(level = "trace", target = "net", skip_all)]
pub async fn handle_signal( pub async fn handle_signal(
&self, &self,
signal_flow: Flow, signal_flow: Flow,
signal_info: SignalInfo, signal_info: SignalInfo,
) -> EyreResult<NetworkResult<()>> { ) -> EyreResult<NetworkResult<()>> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return Ok(NetworkResult::service_unavailable("network is not started"));
};
match signal_info { match signal_info {
SignalInfo::ReverseConnect { receipt, peer_info } => { SignalInfo::ReverseConnect { receipt, peer_info } => {
let routing_table = self.routing_table(); let routing_table = self.routing_table();
@ -811,7 +866,7 @@ impl NetworkManager {
} }
/// Builds an envelope for sending over the network /// Builds an envelope for sending over the network
#[instrument(level = "trace", target = "receipt", skip_all)] #[instrument(level = "trace", target = "net", skip_all)]
fn build_envelope<B: AsRef<[u8]>>( fn build_envelope<B: AsRef<[u8]>>(
&self, &self,
dest_node_id: TypedKey, dest_node_id: TypedKey,
@ -854,13 +909,17 @@ impl NetworkManager {
/// node_ref is the direct destination to which the envelope will be sent /// node_ref is the direct destination to which the envelope will be sent
/// If 'destination_node_ref' is specified, it can be different than the node_ref being sent to /// If 'destination_node_ref' is specified, it can be different than the node_ref being sent to
/// which will cause the envelope to be relayed /// which will cause the envelope to be relayed
#[instrument(level = "trace", target = "receipt", skip_all)] #[instrument(level = "trace", target = "net", skip_all)]
pub async fn send_envelope<B: AsRef<[u8]>>( pub async fn send_envelope<B: AsRef<[u8]>>(
&self, &self,
node_ref: NodeRef, node_ref: NodeRef,
destination_node_ref: Option<NodeRef>, destination_node_ref: Option<NodeRef>,
body: B, body: B,
) -> EyreResult<NetworkResult<SendDataMethod>> { ) -> EyreResult<NetworkResult<SendDataMethod>> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return Ok(NetworkResult::no_connection_other("network is not started"));
};
let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone(); let destination_node_ref = destination_node_ref.as_ref().unwrap_or(&node_ref).clone();
let best_node_id = destination_node_ref.best_node_id(); let best_node_id = destination_node_ref.best_node_id();
@ -898,6 +957,11 @@ impl NetworkManager {
dial_info: DialInfo, dial_info: DialInfo,
rcpt_data: Vec<u8>, rcpt_data: Vec<u8>,
) -> EyreResult<()> { ) -> EyreResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "not sending out-of-band receipt to {} because network is stopped", dial_info);
return Ok(());
};
// Do we need to validate the outgoing receipt? Probably not // Do we need to validate the outgoing receipt? Probably not
// because it is supposed to be opaque and the // because it is supposed to be opaque and the
// recipient/originator does the validation // recipient/originator does the validation
@ -918,8 +982,12 @@ impl NetworkManager {
// Called when a packet potentially containing an RPC envelope is received by a low-level // Called when a packet potentially containing an RPC envelope is received by a low-level
// network protocol handler. Processes the envelope, authenticates and decrypts the RPC message // network protocol handler. Processes the envelope, authenticates and decrypts the RPC message
// and passes it to the RPC handler // and passes it to the RPC handler
#[instrument(level = "trace", target = "receipt", skip_all)] #[instrument(level = "trace", target = "net", skip_all)]
async fn on_recv_envelope(&self, data: &mut [u8], flow: Flow) -> EyreResult<bool> { async fn on_recv_envelope(&self, data: &mut [u8], flow: Flow) -> EyreResult<bool> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return Ok(false);
};
log_net!("envelope of {} bytes received from {:?}", data.len(), flow); log_net!("envelope of {} bytes received from {:?}", data.len(), flow);
let remote_addr = flow.remote_address().ip_addr(); let remote_addr = flow.remote_address().ip_addr();
@ -1132,7 +1200,13 @@ impl NetworkManager {
source_noderef.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain)); source_noderef.merge_filter(NodeRefFilter::new().with_routing_domain(routing_domain));
// Pass message to RPC system // Pass message to RPC system
rpc.enqueue_direct_message(envelope, source_noderef, flow, routing_domain, body)?; if let Err(e) =
rpc.enqueue_direct_message(envelope, source_noderef, flow, routing_domain, body)
{
// Couldn't enqueue, but not the sender's fault
log_net!(debug "failed to enqueue direct message: {}", e);
return Ok(false);
}
// Inform caller that we dealt with the envelope locally // Inform caller that we dealt with the envelope locally
Ok(true) Ok(true)

7
veilid-core/src/network_manager/native/discovery_context.rs
View File

@ -262,7 +262,7 @@ impl DiscoveryContext {
// Always process two at a time so we get both addresses in parallel if possible // Always process two at a time so we get both addresses in parallel if possible
if unord.len() == 2 { if unord.len() == 2 {
// Process one // Process one
if let Some(Some(ei)) = unord.next().await { if let Some(Some(ei)) = unord.next().in_current_span().await {
external_address_infos.push(ei); external_address_infos.push(ei);
if external_address_infos.len() == 2 { if external_address_infos.len() == 2 {
break; break;
@ -272,7 +272,7 @@ impl DiscoveryContext {
} }
// Finish whatever is left if we need to // Finish whatever is left if we need to
if external_address_infos.len() < 2 { if external_address_infos.len() < 2 {
while let Some(res) = unord.next().await { while let Some(res) = unord.next().in_current_span().await {
if let Some(ei) = res { if let Some(ei) = res {
external_address_infos.push(ei); external_address_infos.push(ei);
if external_address_infos.len() == 2 { if external_address_infos.len() == 2 {
@ -644,6 +644,7 @@ impl DiscoveryContext {
} }
/// Add discovery futures to an unordered set that may detect dialinfo when they complete /// Add discovery futures to an unordered set that may detect dialinfo when they complete
#[instrument(level = "trace", skip(self))]
pub async fn discover( pub async fn discover(
&self, &self,
unord: &mut FuturesUnordered<SendPinBoxFuture<Option<DetectionResult>>>, unord: &mut FuturesUnordered<SendPinBoxFuture<Option<DetectionResult>>>,
@ -681,7 +682,7 @@ impl DiscoveryContext {
} }
}; };
if let Some(clear_network_callback) = some_clear_network_callback { if let Some(clear_network_callback) = some_clear_network_callback {
clear_network_callback().await; clear_network_callback().in_current_span().await;
} }
// UPNP Automatic Mapping // UPNP Automatic Mapping

10
veilid-core/src/network_manager/native/igd_manager.rs
View File

@ -191,7 +191,7 @@ impl IGDManager {
mapped_port: u16, mapped_port: u16,
) -> Option<()> { ) -> Option<()> {
let this = self.clone(); let this = self.clone();
blocking_wrapper(move || { blocking_wrapper("igd unmap_port", move || {
let mut inner = this.inner.lock(); let mut inner = this.inner.lock();
// If we already have this port mapped, just return the existing portmap // If we already have this port mapped, just return the existing portmap
@ -235,7 +235,7 @@ impl IGDManager {
expected_external_address: Option<IpAddr>, expected_external_address: Option<IpAddr>,
) -> Option<SocketAddr> { ) -> Option<SocketAddr> {
let this = self.clone(); let this = self.clone();
blocking_wrapper(move || { blocking_wrapper("igd map_any_port", move || {
let mut inner = this.inner.lock(); let mut inner = this.inner.lock();
// If we already have this port mapped, just return the existing portmap // If we already have this port mapped, just return the existing portmap
@ -310,7 +310,7 @@ impl IGDManager {
.await .await
} }
#[instrument(level = "trace", target = "net", skip_all, err)] #[instrument(level = "trace", target = "net", name = "IGDManager::tick", skip_all, err)]
pub async fn tick(&self) -> EyreResult<bool> { pub async fn tick(&self) -> EyreResult<bool> {
// Refresh mappings if we have them // Refresh mappings if we have them
// If an error is received, then return false to restart the local network // If an error is received, then return false to restart the local network
@ -340,7 +340,7 @@ impl IGDManager {
} }
let this = self.clone(); let this = self.clone();
blocking_wrapper(move || { blocking_wrapper("igd tick", move || {
let mut inner = this.inner.lock(); let mut inner = this.inner.lock();
// Process full renewals // Process full renewals
@ -434,6 +434,6 @@ impl IGDManager {
// Normal exit, no restart // Normal exit, no restart
Ok(true) Ok(true)
}, Err(eyre!("failed to process blocking task"))).in_current_span().await }, Err(eyre!("failed to process blocking task"))).instrument(tracing::trace_span!("igd tick fut")).await
} }
} }

89
veilid-core/src/network_manager/native/mod.rs
View File

@ -72,8 +72,6 @@ pub const MAX_CAPABILITIES: usize = 64;
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
struct NetworkInner { struct NetworkInner {
/// Some(true) if the low-level network is running, Some(false) if it is not, None if it is in transit
network_started: Option<bool>,
/// set if the network needs to be restarted due to a low level configuration change /// set if the network needs to be restarted due to a low level configuration change
/// such as dhcp release or change of address or interfaces being added or removed /// such as dhcp release or change of address or interfaces being added or removed
network_needs_restart: bool, network_needs_restart: bool,
@ -114,6 +112,9 @@ struct NetworkInner {
} }
struct NetworkUnlockedInner { struct NetworkUnlockedInner {
// Startup lock
startup_lock: StartupLock,
// Accessors // Accessors
routing_table: RoutingTable, routing_table: RoutingTable,
network_manager: NetworkManager, network_manager: NetworkManager,
@ -139,7 +140,6 @@ pub(in crate::network_manager) struct Network {
impl Network { impl Network {
fn new_inner() -> NetworkInner { fn new_inner() -> NetworkInner {
NetworkInner { NetworkInner {
network_started: Some(false),
network_needs_restart: false, network_needs_restart: false,
needs_public_dial_info_check: false, needs_public_dial_info_check: false,
network_already_cleared: false, network_already_cleared: false,
@ -168,13 +168,14 @@ impl Network {
) -> NetworkUnlockedInner { ) -> NetworkUnlockedInner {
let config = network_manager.config(); let config = network_manager.config();
NetworkUnlockedInner { NetworkUnlockedInner {
startup_lock: StartupLock::new(),
network_manager, network_manager,
routing_table, routing_table,
connection_manager, connection_manager,
interfaces: NetworkInterfaces::new(), interfaces: NetworkInterfaces::new(),
update_network_class_task: TickTask::new(1), update_network_class_task: TickTask::new("update_network_class_task", 1),
network_interfaces_task: TickTask::new(1), network_interfaces_task: TickTask::new("network_interfaces_task", 1),
upnp_task: TickTask::new(1), upnp_task: TickTask::new("upnp_task", 1),
igd_manager: igd_manager::IGDManager::new(config.clone()), igd_manager: igd_manager::IGDManager::new(config.clone()),
} }
} }
@ -335,12 +336,12 @@ impl Network {
inner.preferred_local_addresses.get(&key).copied() inner.preferred_local_addresses.get(&key).copied()
} }
pub fn is_stable_interface_address(&self, addr: IpAddr) -> bool { pub(crate) fn is_stable_interface_address(&self, addr: IpAddr) -> bool {
let stable_addrs = self.get_stable_interface_addresses(); let stable_addrs = self.get_stable_interface_addresses();
stable_addrs.contains(&addr) stable_addrs.contains(&addr)
} }
pub fn get_stable_interface_addresses(&self) -> Vec<IpAddr> { pub(crate) fn get_stable_interface_addresses(&self) -> Vec<IpAddr> {
let addrs = self.unlocked_inner.interfaces.stable_addresses(); let addrs = self.unlocked_inner.interfaces.stable_addresses();
let mut addrs: Vec<IpAddr> = addrs let mut addrs: Vec<IpAddr> = addrs
.into_iter() .into_iter()
@ -377,7 +378,7 @@ impl Network {
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
// Record DialInfo failures // Record DialInfo failures
pub async fn record_dial_info_failure<T, F: Future<Output = EyreResult<NetworkResult<T>>>>( async fn record_dial_info_failure<T, F: Future<Output = EyreResult<NetworkResult<T>>>>(
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
fut: F, fut: F,
@ -401,6 +402,8 @@ impl Network {
dial_info: DialInfo, dial_info: DialInfo,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<()>> { ) -> EyreResult<NetworkResult<()>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure( self.record_dial_info_failure(
dial_info.clone(), dial_info.clone(),
async move { async move {
@ -476,6 +479,8 @@ impl Network {
data: Vec<u8>, data: Vec<u8>,
timeout_ms: u32, timeout_ms: u32,
) -> EyreResult<NetworkResult<Vec<u8>>> { ) -> EyreResult<NetworkResult<Vec<u8>>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure( self.record_dial_info_failure(
dial_info.clone(), dial_info.clone(),
async move { async move {
@ -513,7 +518,7 @@ impl Network {
let mut out = vec![0u8; MAX_MESSAGE_SIZE]; let mut out = vec![0u8; MAX_MESSAGE_SIZE];
let (recv_len, recv_addr) = network_result_try!(timeout( let (recv_len, recv_addr) = network_result_try!(timeout(
timeout_ms, timeout_ms,
h.recv_message(&mut out).instrument(Span::current()) h.recv_message(&mut out).in_current_span()
) )
.await .await
.into_network_result()) .into_network_result())
@ -564,7 +569,7 @@ impl Network {
let out = network_result_try!(network_result_try!(timeout( let out = network_result_try!(network_result_try!(timeout(
timeout_ms, timeout_ms,
pnc.recv() pnc.recv().in_current_span()
) )
.await .await
.into_network_result()) .into_network_result())
@ -590,6 +595,8 @@ impl Network {
flow: Flow, flow: Flow,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<SendDataToExistingFlowResult> { ) -> EyreResult<SendDataToExistingFlowResult> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
let data_len = data.len(); let data_len = data.len();
// Handle connectionless protocol // Handle connectionless protocol
@ -655,6 +662,8 @@ impl Network {
dial_info: DialInfo, dial_info: DialInfo,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<UniqueFlow>> { ) -> EyreResult<NetworkResult<UniqueFlow>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure( self.record_dial_info_failure(
dial_info.clone(), dial_info.clone(),
async move { async move {
@ -922,22 +931,22 @@ impl Network {
#[instrument(level = "debug", err, skip_all)] #[instrument(level = "debug", err, skip_all)]
pub async fn startup(&self) -> EyreResult<StartupDisposition> { pub async fn startup(&self) -> EyreResult<StartupDisposition> {
self.inner.lock().network_started = None; let guard = self.unlocked_inner.startup_lock.startup()?;
match self.startup_internal().await { match self.startup_internal().await {
Ok(StartupDisposition::Success) => { Ok(StartupDisposition::Success) => {
info!("network started"); info!("network started");
self.inner.lock().network_started = Some(true); guard.success();
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
Ok(StartupDisposition::BindRetry) => { Ok(StartupDisposition::BindRetry) => {
debug!("network bind retry"); debug!("network bind retry");
self.inner.lock().network_started = Some(false); self.shutdown_internal().await;
Ok(StartupDisposition::BindRetry) Ok(StartupDisposition::BindRetry)
} }
Err(e) => { Err(e) => {
debug!("network failed to start"); debug!("network failed to start");
self.inner.lock().network_started = Some(false); self.shutdown_internal().await;
Err(e) Err(e)
} }
} }
@ -947,8 +956,8 @@ impl Network {
self.inner.lock().network_needs_restart self.inner.lock().network_needs_restart
} }
pub fn is_started(&self) -> Option<bool> { pub fn is_started(&self) -> bool {
self.inner.lock().network_started self.unlocked_inner.startup_lock.is_started()
} }
#[instrument(level = "debug", skip_all)] #[instrument(level = "debug", skip_all)]
@ -957,11 +966,7 @@ impl Network {
} }
#[instrument(level = "debug", skip_all)] #[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) { async fn shutdown_internal(&self) {
log_net!(debug "starting low level network shutdown");
self.inner.lock().network_started = None;
let routing_table = self.routing_table(); let routing_table = self.routing_table();
// Stop all tasks // Stop all tasks
@ -1005,7 +1010,19 @@ impl Network {
// Reset state including network class // Reset state including network class
*self.inner.lock() = Self::new_inner(); *self.inner.lock() = Self::new_inner();
}
#[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) {
log_net!(debug "starting low level network shutdown");
let Ok(guard) = self.unlocked_inner.startup_lock.shutdown().await else {
log_net!(debug "low level network is already shut down");
return;
};
self.shutdown_internal().await;
guard.success();
log_net!(debug "finished low level network shutdown"); log_net!(debug "finished low level network shutdown");
} }
@ -1014,12 +1031,20 @@ impl Network {
&self, &self,
punishment: Option<Box<dyn FnOnce() + Send + 'static>>, punishment: Option<Box<dyn FnOnce() + Send + 'static>>,
) { ) {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return;
};
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
inner.needs_public_dial_info_check = true; inner.needs_public_dial_info_check = true;
inner.public_dial_info_check_punishment = punishment; inner.public_dial_info_check_punishment = punishment;
} }
pub fn needs_public_dial_info_check(&self) -> bool { pub fn needs_public_dial_info_check(&self) -> bool {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return false;
};
let inner = self.inner.lock(); let inner = self.inner.lock();
inner.needs_public_dial_info_check inner.needs_public_dial_info_check
} }
@ -1027,7 +1052,7 @@ impl Network {
////////////////////////////////////////// //////////////////////////////////////////
#[instrument(level = "trace", target = "net", skip_all, err)] #[instrument(level = "trace", target = "net", skip_all, err)]
pub async fn network_interfaces_task_routine( async fn network_interfaces_task_routine(
self, self,
_stop_token: StopToken, _stop_token: StopToken,
_l: u64, _l: u64,
@ -1038,13 +1063,8 @@ impl Network {
Ok(()) Ok(())
} }
#[instrument(level = "trace", target = "net", skip_all, err)] #[instrument(parent = None, level = "trace", target = "net", skip_all, err)]
pub async fn upnp_task_routine( async fn upnp_task_routine(self, _stop_token: StopToken, _l: u64, _t: u64) -> EyreResult<()> {
self,
_stop_token: StopToken,
_l: u64,
_t: u64,
) -> EyreResult<()> {
if !self.unlocked_inner.igd_manager.tick().await? { if !self.unlocked_inner.igd_manager.tick().await? {
info!("upnp failed, restarting local network"); info!("upnp failed, restarting local network");
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
@ -1054,8 +1074,13 @@ impl Network {
Ok(()) Ok(())
} }
#[instrument(level = "trace", target = "net", skip_all, err)] #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)]
pub async fn tick(&self) -> EyreResult<()> { pub(crate) async fn tick(&self) -> EyreResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return Ok(());
};
let (detect_address_changes, upnp) = { let (detect_address_changes, upnp) = {
let config = self.network_manager().config(); let config = self.network_manager().config();
let c = config.get(); let c = config.get();

9
veilid-core/src/network_manager/native/network_class_discovery.rs
View File

@ -200,7 +200,12 @@ impl Network {
// Wait for all discovery futures to complete and apply discoverycontexts // Wait for all discovery futures to complete and apply discoverycontexts
let mut all_address_types = AddressTypeSet::new(); let mut all_address_types = AddressTypeSet::new();
loop { loop {
match unord.next().timeout_at(stop_token.clone()).await { match unord
.next()
.timeout_at(stop_token.clone())
.in_current_span()
.await
{
Ok(Some(Some(dr))) => { Ok(Some(Some(dr))) => {
// Found some new dial info for this protocol/address combination // Found some new dial info for this protocol/address combination
self.update_with_detected_dial_info(dr.ddi.clone()).await?; self.update_with_detected_dial_info(dr.ddi.clone()).await?;
@ -277,7 +282,7 @@ impl Network {
Ok(()) Ok(())
} }
#[instrument(level = "trace", skip(self), err)] #[instrument(parent = None, level = "trace", skip(self), err)]
pub async fn update_network_class_task_routine( pub async fn update_network_class_task_routine(
self, self,
stop_token: StopToken, stop_token: StopToken,

65
veilid-core/src/network_manager/native/network_tcp.rs
View File

@ -38,6 +38,7 @@ impl Network {
Ok(acceptor) Ok(acceptor)
} }
#[instrument(level = "trace", skip_all)]
async fn try_tls_handlers( async fn try_tls_handlers(
&self, &self,
tls_acceptor: &TlsAcceptor, tls_acceptor: &TlsAcceptor,
@ -60,7 +61,7 @@ impl Network {
// read a chunk of the stream // read a chunk of the stream
timeout( timeout(
tls_connection_initial_timeout_ms, tls_connection_initial_timeout_ms,
ps.peek_exact(&mut first_packet), ps.peek_exact(&mut first_packet).in_current_span(),
) )
.await .await
.wrap_err("tls initial timeout")? .wrap_err("tls initial timeout")?
@ -70,6 +71,7 @@ impl Network {
.await .await
} }
#[instrument(level = "trace", skip_all)]
async fn try_handlers( async fn try_handlers(
&self, &self,
stream: AsyncPeekStream, stream: AsyncPeekStream,
@ -90,6 +92,7 @@ impl Network {
Ok(None) Ok(None)
} }
#[instrument(level = "trace", skip_all)]
async fn tcp_acceptor( async fn tcp_acceptor(
self, self,
tcp_stream: io::Result<TcpStream>, tcp_stream: io::Result<TcpStream>,
@ -132,33 +135,33 @@ impl Network {
} }
}; };
// #[cfg(all(feature = "rt-async-std", unix))] #[cfg(all(feature = "rt-async-std", unix))]
// { {
// // async-std does not directly support linger on TcpStream yet // async-std does not directly support linger on TcpStream yet
// use std::os::fd::{AsRawFd, FromRawFd}; use std::os::fd::{AsRawFd, FromRawFd};
// if let Err(e) = unsafe { socket2::Socket::from_raw_fd(tcp_stream.as_raw_fd()) } if let Err(e) = unsafe { socket2::Socket::from_raw_fd(tcp_stream.as_raw_fd()) }
// .set_linger(Some(core::time::Duration::from_secs(0))) .set_linger(Some(core::time::Duration::from_secs(0)))
// { {
// log_net!(debug "Couldn't set TCP linger: {}", e); log_net!(debug "Couldn't set TCP linger: {}", e);
// return; return;
// } }
// } }
// #[cfg(all(feature = "rt-async-std", windows))] #[cfg(all(feature = "rt-async-std", windows))]
// { {
// // async-std does not directly support linger on TcpStream yet // async-std does not directly support linger on TcpStream yet
// use std::os::windows::io::{AsRawSocket, FromRawSocket}; use std::os::windows::io::{AsRawSocket, FromRawSocket};
// if let Err(e) = unsafe { socket2::Socket::from_raw_socket(tcp_stream.as_raw_socket()) } if let Err(e) = unsafe { socket2::Socket::from_raw_socket(tcp_stream.as_raw_socket()) }
// .set_linger(Some(core::time::Duration::from_secs(0))) .set_linger(Some(core::time::Duration::from_secs(0)))
// { {
// log_net!(debug "Couldn't set TCP linger: {}", e); log_net!(debug "Couldn't set TCP linger: {}", e);
// return; return;
// } }
// } }
// #[cfg(not(feature = "rt-async-std"))] #[cfg(not(feature = "rt-async-std"))]
// if let Err(e) = tcp_stream.set_linger(Some(core::time::Duration::from_secs(0))) { if let Err(e) = tcp_stream.set_linger(Some(core::time::Duration::from_secs(0))) {
// log_net!(debug "Couldn't set TCP linger: {}", e); log_net!(debug "Couldn't set TCP linger: {}", e);
// return; return;
// } }
if let Err(e) = tcp_stream.set_nodelay(true) { if let Err(e) = tcp_stream.set_nodelay(true) {
log_net!(debug "Couldn't set TCP nodelay: {}", e); log_net!(debug "Couldn't set TCP nodelay: {}", e);
return; return;
@ -180,7 +183,7 @@ impl Network {
// read a chunk of the stream // read a chunk of the stream
if timeout( if timeout(
connection_initial_timeout_ms, connection_initial_timeout_ms,
ps.peek_exact(&mut first_packet), ps.peek_exact(&mut first_packet).in_current_span(),
) )
.await .await
.is_err() .is_err()
@ -237,6 +240,7 @@ impl Network {
} }
} }
#[instrument(level = "trace", skip_all)]
async fn spawn_socket_listener(&self, addr: SocketAddr) -> EyreResult<bool> { async fn spawn_socket_listener(&self, addr: SocketAddr) -> EyreResult<bool> {
// Get config // Get config
let (connection_initial_timeout_ms, tls_connection_initial_timeout_ms) = { let (connection_initial_timeout_ms, tls_connection_initial_timeout_ms) = {
@ -297,7 +301,7 @@ impl Network {
let connection_manager = self.connection_manager(); let connection_manager = self.connection_manager();
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
let jh = spawn(async move { let jh = spawn(&format!("TCP listener {}", addr), async move {
// moves listener object in and get incoming iterator // moves listener object in and get incoming iterator
// when this task exists, the listener will close the socket // when this task exists, the listener will close the socket
@ -344,6 +348,7 @@ impl Network {
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
// TCP listener that multiplexes ports so multiple protocols can exist on a single port // TCP listener that multiplexes ports so multiple protocols can exist on a single port
#[instrument(level = "trace", skip_all)]
pub(super) async fn start_tcp_listener( pub(super) async fn start_tcp_listener(
&self, &self,
bind_set: NetworkBindSet, bind_set: NetworkBindSet,

12
veilid-core/src/network_manager/native/network_udp.rs
View File

@ -3,6 +3,7 @@ use sockets::*;
use stop_token::future::FutureExt; use stop_token::future::FutureExt;
impl Network { impl Network {
#[instrument(level = "trace", skip_all)]
pub(super) async fn create_udp_listener_tasks(&self) -> EyreResult<()> { pub(super) async fn create_udp_listener_tasks(&self) -> EyreResult<()> {
// Spawn socket tasks // Spawn socket tasks
let mut task_count = { let mut task_count = {
@ -16,14 +17,14 @@ impl Network {
} }
} }
log_net!("task_count: {}", task_count); log_net!("task_count: {}", task_count);
for _ in 0..task_count { for task_n in 0..task_count {
log_net!("Spawning UDP listener task"); log_net!("Spawning UDP listener task");
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
// Run thread task to process stream of messages // Run thread task to process stream of messages
let this = self.clone(); let this = self.clone();
let jh = spawn(async move { let jh = spawn(&format!("UDP listener {}", task_n), async move {
log_net!("UDP listener task spawned"); log_net!("UDP listener task spawned");
// Collect all our protocol handlers into a vector // Collect all our protocol handlers into a vector
@ -57,6 +58,7 @@ impl Network {
match ph match ph
.recv_message(&mut data) .recv_message(&mut data)
.timeout_at(stop_token.clone()) .timeout_at(stop_token.clone())
.in_current_span()
.await .await
{ {
Ok(Ok((size, flow))) => { Ok(Ok((size, flow))) => {
@ -89,7 +91,7 @@ impl Network {
// Now we wait for join handles to exit, // Now we wait for join handles to exit,
// if any error out it indicates an error needing // if any error out it indicates an error needing
// us to completely restart the network // us to completely restart the network
while let Some(v) = protocol_handlers_unordered.next().await { while let Some(v) = protocol_handlers_unordered.next().in_current_span().await {
// true = stopped, false = errored // true = stopped, false = errored
if !v { if !v {
// If any protocol handler fails, our socket died and we need to restart the network // If any protocol handler fails, our socket died and we need to restart the network
@ -98,7 +100,7 @@ impl Network {
} }
log_net!("UDP listener task stopped"); log_net!("UDP listener task stopped");
}); }.instrument(trace_span!(parent: None, "UDP Listener")));
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
// Add to join handle // Add to join handle
@ -108,6 +110,7 @@ impl Network {
Ok(()) Ok(())
} }
#[instrument(level = "trace", skip_all)]
async fn create_udp_protocol_handler(&self, addr: SocketAddr) -> EyreResult<bool> { async fn create_udp_protocol_handler(&self, addr: SocketAddr) -> EyreResult<bool> {
log_net!("create_udp_protocol_handler on {:?}", &addr); log_net!("create_udp_protocol_handler on {:?}", &addr);
@ -148,6 +151,7 @@ impl Network {
Ok(true) Ok(true)
} }
#[instrument(level = "trace", skip_all)]
pub(super) async fn create_udp_protocol_handlers( pub(super) async fn create_udp_protocol_handlers(
&self, &self,
bind_set: NetworkBindSet, bind_set: NetworkBindSet,

16
veilid-core/src/network_manager/native/protocol/sockets.rs
View File

@ -91,9 +91,9 @@ pub fn new_default_tcp_socket(domain: Domain) -> io::Result<Socket> {
#[instrument(level = "trace", ret)] #[instrument(level = "trace", ret)]
pub fn new_shared_tcp_socket(domain: Domain) -> io::Result<Socket> { pub fn new_shared_tcp_socket(domain: Domain) -> io::Result<Socket> {
let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP))?; let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP))?;
// if let Err(e) = socket.set_linger(Some(core::time::Duration::from_secs(0))) { if let Err(e) = socket.set_linger(Some(core::time::Duration::from_secs(0))) {
// log_net!(error "Couldn't set TCP linger: {}", e); log_net!(error "Couldn't set TCP linger: {}", e);
// } }
if let Err(e) = socket.set_nodelay(true) { if let Err(e) = socket.set_nodelay(true) {
log_net!(error "Couldn't set TCP nodelay: {}", e); log_net!(error "Couldn't set TCP nodelay: {}", e);
} }
@ -162,10 +162,12 @@ pub async fn nonblocking_connect(
let async_stream = Async::new(std::net::TcpStream::from(socket))?; let async_stream = Async::new(std::net::TcpStream::from(socket))?;
// The stream becomes writable when connected // The stream becomes writable when connected
timeout_or_try!(timeout(timeout_ms, async_stream.writable()) timeout_or_try!(
.await timeout(timeout_ms, async_stream.writable().in_current_span())
.into_timeout_or() .await
.into_result()?); .into_timeout_or()
.into_result()?
);
// Check low level error // Check low level error
let async_stream = match async_stream.get_ref().take_error()? { let async_stream = match async_stream.get_ref().take_error()? {

2
veilid-core/src/network_manager/native/protocol/tcp.rs
View File

@ -134,7 +134,7 @@ impl RawTcpProtocolHandler {
let mut peekbuf: [u8; PEEK_DETECT_LEN] = [0u8; PEEK_DETECT_LEN]; let mut peekbuf: [u8; PEEK_DETECT_LEN] = [0u8; PEEK_DETECT_LEN];
if (timeout( if (timeout(
self.connection_initial_timeout_ms, self.connection_initial_timeout_ms,
ps.peek_exact(&mut peekbuf), ps.peek_exact(&mut peekbuf).in_current_span(),
) )
.await) .await)
.is_err() .is_err()

2
veilid-core/src/network_manager/native/protocol/ws.rs
View File

@ -222,7 +222,7 @@ impl WebsocketProtocolHandler {
let mut peek_buf = [0u8; MAX_WS_BEFORE_BODY]; let mut peek_buf = [0u8; MAX_WS_BEFORE_BODY];
let peek_len = match timeout( let peek_len = match timeout(
self.arc.connection_initial_timeout_ms, self.arc.connection_initial_timeout_ms,
ps.peek(&mut peek_buf), ps.peek(&mut peek_buf).in_current_span(),
) )
.await .await
{ {

5
veilid-core/src/network_manager/native/start_protocols.rs
View File

@ -84,6 +84,7 @@ impl Network {
// Returns a port, a set of ip addresses to bind to, and a // Returns a port, a set of ip addresses to bind to, and a
// bool specifying if multiple ports should be tried // bool specifying if multiple ports should be tried
#[instrument(level = "trace", skip_all)]
async fn convert_listen_address_to_bind_set( async fn convert_listen_address_to_bind_set(
&self, &self,
listen_address: String, listen_address: String,
@ -136,6 +137,7 @@ impl Network {
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
#[instrument(level = "trace", skip_all)]
pub(super) async fn bind_udp_protocol_handlers( pub(super) async fn bind_udp_protocol_handlers(
&self, &self,
editor_public_internet: &mut RoutingDomainEditor, editor_public_internet: &mut RoutingDomainEditor,
@ -249,6 +251,7 @@ impl Network {
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
#[instrument(level = "trace", skip_all)]
pub(super) async fn start_ws_listeners( pub(super) async fn start_ws_listeners(
&self, &self,
editor_public_internet: &mut RoutingDomainEditor, editor_public_internet: &mut RoutingDomainEditor,
@ -364,6 +367,7 @@ impl Network {
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
#[instrument(level = "trace", skip_all)]
pub(super) async fn start_wss_listeners( pub(super) async fn start_wss_listeners(
&self, &self,
editor_public_internet: &mut RoutingDomainEditor, editor_public_internet: &mut RoutingDomainEditor,
@ -463,6 +467,7 @@ impl Network {
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
#[instrument(level = "trace", skip_all)]
pub(super) async fn start_tcp_listeners( pub(super) async fn start_tcp_listeners(
&self, &self,
editor_public_internet: &mut RoutingDomainEditor, editor_public_internet: &mut RoutingDomainEditor,

13
veilid-core/src/network_manager/network_connection.rs
View File

@ -136,7 +136,8 @@ impl NetworkConnection {
let flow = protocol_connection.flow(); let flow = protocol_connection.flow();
// Create handle for sending // Create handle for sending
let (sender, receiver) = flume::bounded(get_concurrency() as usize); //let (sender, receiver) = flume::bounded(get_concurrency() as usize);
let (sender, receiver) = flume::unbounded();
// Create stats // Create stats
let stats = Arc::new(Mutex::new(NetworkConnectionStats { let stats = Arc::new(Mutex::new(NetworkConnectionStats {
@ -148,7 +149,7 @@ impl NetworkConnection {
let local_stop_token = stop_source.token(); let local_stop_token = stop_source.token();
// Spawn connection processor and pass in protocol connection // Spawn connection processor and pass in protocol connection
let processor = spawn(Self::process_connection( let processor = spawn("connection processor", Self::process_connection(
connection_manager, connection_manager,
local_stop_token, local_stop_token,
manager_stop_token, manager_stop_token,
@ -265,7 +266,7 @@ impl NetworkConnection {
// Connection receiver loop // Connection receiver loop
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
//#[instrument(level="trace", target="net", skip_all)] #[instrument(parent = None, level="trace", target="net", skip_all)]
fn process_connection( fn process_connection(
connection_manager: ConnectionManager, connection_manager: ConnectionManager,
local_stop_token: StopToken, local_stop_token: StopToken,
@ -309,10 +310,6 @@ impl NetworkConnection {
match res { match res {
Ok((_span_id, message)) => { Ok((_span_id, message)) => {
// let span = span!(Level::TRACE, "process_connection send");
// span.follows_from(span_id);
// let _enter = span.enter();
// Touch the LRU for this connection // Touch the LRU for this connection
connection_manager.touch_connection_by_id(connection_id); connection_manager.touch_connection_by_id(connection_id);
@ -337,7 +334,7 @@ impl NetworkConnection {
RecvLoopAction::Finish RecvLoopAction::Finish
} }
} }
}); }.in_current_span());
unord.push(system_boxed(sender_fut.in_current_span())); unord.push(system_boxed(sender_fut.in_current_span()));
} }

49
veilid-core/src/network_manager/receipt_manager.rs
View File

@ -158,9 +158,14 @@ struct ReceiptManagerInner {
timeout_task: MustJoinSingleFuture<()>, timeout_task: MustJoinSingleFuture<()>,
} }
struct ReceiptManagerUnlockedInner {
startup_lock: StartupLock,
}
#[derive(Clone)] #[derive(Clone)]
pub(super) struct ReceiptManager { pub(super) struct ReceiptManager {
inner: Arc<Mutex<ReceiptManagerInner>>, inner: Arc<Mutex<ReceiptManagerInner>>,
unlocked_inner: Arc<ReceiptManagerUnlockedInner>,
} }
impl ReceiptManager { impl ReceiptManager {
@ -177,6 +182,9 @@ impl ReceiptManager {
pub fn new(network_manager: NetworkManager) -> Self { pub fn new(network_manager: NetworkManager) -> Self {
Self { Self {
inner: Arc::new(Mutex::new(Self::new_inner(network_manager))), inner: Arc::new(Mutex::new(Self::new_inner(network_manager))),
unlocked_inner: Arc::new(ReceiptManagerUnlockedInner {
startup_lock: StartupLock::new(),
}),
} }
} }
@ -185,6 +193,7 @@ impl ReceiptManager {
} }
pub async fn startup(&self) -> EyreResult<()> { pub async fn startup(&self) -> EyreResult<()> {
let guard = self.unlocked_inner.startup_lock.startup()?;
log_net!(debug "startup receipt manager"); log_net!(debug "startup receipt manager");
// Retrieve config // Retrieve config
@ -195,6 +204,7 @@ impl ReceiptManager {
inner.stop_source = Some(StopSource::new()); inner.stop_source = Some(StopSource::new());
} }
guard.success();
Ok(()) Ok(())
} }
@ -223,7 +233,7 @@ impl ReceiptManager {
} }
#[instrument(level = "trace", target = "receipt", skip_all)] #[instrument(level = "trace", target = "receipt", skip_all)]
pub async fn timeout_task_routine(self, now: Timestamp, stop_token: StopToken) { async fn timeout_task_routine(self, now: Timestamp, stop_token: StopToken) {
// Go through all receipts and build a list of expired nonces // Go through all receipts and build a list of expired nonces
let mut new_next_oldest_ts: Option<Timestamp> = None; let mut new_next_oldest_ts: Option<Timestamp> = None;
let mut expired_records = Vec::new(); let mut expired_records = Vec::new();
@ -271,8 +281,18 @@ impl ReceiptManager {
} }
} }
#[instrument(level = "trace", target = "receipt", skip_all, err)] #[instrument(
level = "trace",
target = "receipt",
name = "ReceiptManager::tick",
skip_all,
err
)]
pub async fn tick(&self) -> EyreResult<()> { pub async fn tick(&self) -> EyreResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return Ok(());
};
let (next_oldest_ts, timeout_task, stop_token) = { let (next_oldest_ts, timeout_task, stop_token) = {
let inner = self.inner.lock(); let inner = self.inner.lock();
let stop_token = match inner.stop_source.as_ref() { let stop_token = match inner.stop_source.as_ref() {
@ -291,10 +311,12 @@ impl ReceiptManager {
// Single-spawn the timeout task routine // Single-spawn the timeout task routine
let _ = timeout_task let _ = timeout_task
.single_spawn( .single_spawn(
"receipt timeout",
self.clone() self.clone()
.timeout_task_routine(now, stop_token) .timeout_task_routine(now, stop_token)
.in_current_span(), .instrument(trace_span!(parent: None, "receipt timeout task")),
) )
.in_current_span()
.await; .await;
} }
} }
@ -303,6 +325,11 @@ impl ReceiptManager {
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
log_net!(debug "starting receipt manager shutdown"); log_net!(debug "starting receipt manager shutdown");
let Ok(guard) = self.unlocked_inner.startup_lock.shutdown().await else {
log_net!(debug "receipt manager is already shut down");
return;
};
let network_manager = self.network_manager(); let network_manager = self.network_manager();
// Stop all tasks // Stop all tasks
@ -320,6 +347,8 @@ impl ReceiptManager {
} }
*self.inner.lock() = Self::new_inner(network_manager); *self.inner.lock() = Self::new_inner(network_manager);
guard.success();
log_net!(debug "finished receipt manager shutdown"); log_net!(debug "finished receipt manager shutdown");
} }
@ -332,6 +361,10 @@ impl ReceiptManager {
expected_returns: u32, expected_returns: u32,
callback: impl ReceiptCallback, callback: impl ReceiptCallback,
) { ) {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return;
};
let receipt_nonce = receipt.get_nonce(); let receipt_nonce = receipt.get_nonce();
event!(target: "receipt", Level::DEBUG, "== New Multiple Receipt ({}) {} ", expected_returns, receipt_nonce.encode()); event!(target: "receipt", Level::DEBUG, "== New Multiple Receipt ({}) {} ", expected_returns, receipt_nonce.encode());
let record = Arc::new(Mutex::new(ReceiptRecord::new( let record = Arc::new(Mutex::new(ReceiptRecord::new(
@ -353,6 +386,10 @@ impl ReceiptManager {
expiration: Timestamp, expiration: Timestamp,
eventual: ReceiptSingleShotType, eventual: ReceiptSingleShotType,
) { ) {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return;
};
let receipt_nonce = receipt.get_nonce(); let receipt_nonce = receipt.get_nonce();
event!(target: "receipt", Level::DEBUG, "== New SingleShot Receipt {}", receipt_nonce.encode()); event!(target: "receipt", Level::DEBUG, "== New SingleShot Receipt {}", receipt_nonce.encode());
@ -385,6 +422,8 @@ impl ReceiptManager {
pub async fn cancel_receipt(&self, nonce: &Nonce) -> EyreResult<()> { pub async fn cancel_receipt(&self, nonce: &Nonce) -> EyreResult<()> {
event!(target: "receipt", Level::DEBUG, "== Cancel Receipt {}", nonce.encode()); event!(target: "receipt", Level::DEBUG, "== Cancel Receipt {}", nonce.encode());
let _guard = self.unlocked_inner.startup_lock.enter()?;
// Remove the record // Remove the record
let record = { let record = {
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
@ -417,6 +456,10 @@ impl ReceiptManager {
receipt: Receipt, receipt: Receipt,
receipt_returned: ReceiptReturned, receipt_returned: ReceiptReturned,
) -> NetworkResult<()> { ) -> NetworkResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
return NetworkResult::service_unavailable("receipt manager not started");
};
let receipt_nonce = receipt.get_nonce(); let receipt_nonce = receipt.get_nonce();
let extra_data = receipt.get_extra_data(); let extra_data = receipt.get_extra_data();

104
veilid-core/src/network_manager/send_data.rs
View File

@ -1,4 +1,5 @@
use super::*; use super::*;
use stop_token::future::FutureExt as _;
impl NetworkManager { impl NetworkManager {
/// Send raw data to a node /// Send raw data to a node
@ -146,7 +147,7 @@ impl NetworkManager {
Ok(NetworkResult::value(send_data_method)) Ok(NetworkResult::value(send_data_method))
} }
.instrument(trace_span!("send_data")), .in_current_span()
) )
} }
@ -559,6 +560,12 @@ impl NetworkManager {
target_nr: NodeRef, target_nr: NodeRef,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<UniqueFlow>> { ) -> EyreResult<NetworkResult<UniqueFlow>> {
// Detect if network is stopping so we can break out of this
let Some(stop_token) = self.unlocked_inner.startup_lock.stop_token() else {
return Ok(NetworkResult::service_unavailable("network is stopping"));
};
// Build a return receipt for the signal // Build a return receipt for the signal
let receipt_timeout = ms_to_us( let receipt_timeout = ms_to_us(
self.unlocked_inner self.unlocked_inner
@ -588,30 +595,38 @@ impl NetworkManager {
let rpc = self.rpc_processor(); let rpc = self.rpc_processor();
network_result_try!(rpc network_result_try!(rpc
.rpc_call_signal( .rpc_call_signal(
Destination::relay(relay_nr, target_nr.clone()), Destination::relay(relay_nr.clone(), target_nr.clone()),
SignalInfo::ReverseConnect { receipt, peer_info }, SignalInfo::ReverseConnect { receipt, peer_info },
) )
.await .await
.wrap_err("failed to send signal")?); .wrap_err("failed to send signal")?);
// Wait for the return receipt // Wait for the return receipt
let inbound_nr = match eventual_value.await.take_value().unwrap() { let inbound_nr = match eventual_value.timeout_at(stop_token).in_current_span().await {
ReceiptEvent::ReturnedPrivate { private_route: _ } Err(_) => {
| ReceiptEvent::ReturnedOutOfBand return Ok(NetworkResult::service_unavailable("network is stopping"));
| ReceiptEvent::ReturnedSafety => {
return Ok(NetworkResult::invalid_message(
"reverse connect receipt should be returned in-band",
));
} }
ReceiptEvent::ReturnedInBand { inbound_noderef } => inbound_noderef, Ok(v) => {
ReceiptEvent::Expired => { let receipt_event = v.take_value().unwrap();
return Ok(NetworkResult::timeout()); match receipt_event {
} ReceiptEvent::ReturnedPrivate { private_route: _ }
ReceiptEvent::Cancelled => { | ReceiptEvent::ReturnedOutOfBand
return Ok(NetworkResult::no_connection_other(format!( | ReceiptEvent::ReturnedSafety => {
"reverse connect receipt cancelled from {}", return Ok(NetworkResult::invalid_message(
target_nr "reverse connect receipt should be returned in-band",
))) ));
}
ReceiptEvent::ReturnedInBand { inbound_noderef } => inbound_noderef,
ReceiptEvent::Expired => {
return Ok(NetworkResult::timeout());
}
ReceiptEvent::Cancelled => {
return Ok(NetworkResult::no_connection_other(format!(
"reverse connect receipt cancelled from {}",
target_nr
)))
}
}
} }
}; };
@ -634,7 +649,9 @@ impl NetworkManager {
)), )),
} }
} else { } else {
bail!("no reverse connection available") return Ok(NetworkResult::no_connection_other(format!(
"reverse connection dropped from {}", target_nr)
));
} }
} }
@ -648,6 +665,11 @@ impl NetworkManager {
target_nr: NodeRef, target_nr: NodeRef,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<UniqueFlow>> { ) -> EyreResult<NetworkResult<UniqueFlow>> {
// Detect if network is stopping so we can break out of this
let Some(stop_token) = self.unlocked_inner.startup_lock.stop_token() else {
return Ok(NetworkResult::service_unavailable("network is stopping"));
};
// Ensure we are filtered down to UDP (the only hole punch protocol supported today) // Ensure we are filtered down to UDP (the only hole punch protocol supported today)
assert!(target_nr assert!(target_nr
.filter_ref() .filter_ref()
@ -706,23 +728,31 @@ impl NetworkManager {
.wrap_err("failed to send signal")?); .wrap_err("failed to send signal")?);
// Wait for the return receipt // Wait for the return receipt
let inbound_nr = match eventual_value.await.take_value().unwrap() { let inbound_nr = match eventual_value.timeout_at(stop_token).in_current_span().await {
ReceiptEvent::ReturnedPrivate { private_route: _ } Err(_) => {
| ReceiptEvent::ReturnedOutOfBand return Ok(NetworkResult::service_unavailable("network is stopping"));
| ReceiptEvent::ReturnedSafety => {
return Ok(NetworkResult::invalid_message(
"hole punch receipt should be returned in-band",
));
} }
ReceiptEvent::ReturnedInBand { inbound_noderef } => inbound_noderef, Ok(v) => {
ReceiptEvent::Expired => { let receipt_event = v.take_value().unwrap();
return Ok(NetworkResult::timeout()); match receipt_event {
} ReceiptEvent::ReturnedPrivate { private_route: _ }
ReceiptEvent::Cancelled => { | ReceiptEvent::ReturnedOutOfBand
return Ok(NetworkResult::no_connection_other(format!( | ReceiptEvent::ReturnedSafety => {
"hole punch receipt cancelled from {}", return Ok(NetworkResult::invalid_message(
target_nr "hole punch receipt should be returned in-band",
))) ));
}
ReceiptEvent::ReturnedInBand { inbound_noderef } => inbound_noderef,
ReceiptEvent::Expired => {
return Ok(NetworkResult::timeout());
}
ReceiptEvent::Cancelled => {
return Ok(NetworkResult::no_connection_other(format!(
"hole punch receipt cancelled from {}",
target_nr
)))
}
}
} }
}; };
@ -749,7 +779,9 @@ impl NetworkManager {
)), )),
} }
} else { } else {
bail!("no hole punch available") return Ok(NetworkResult::no_connection_other(format!(
"hole punch dropped from {}", target_nr)
));
} }
} }
} }

4
veilid-core/src/network_manager/stats.rs
View File

@ -35,7 +35,7 @@ impl Default for NetworkManagerStats {
impl NetworkManager { impl NetworkManager {
// Callbacks from low level network for statistics gathering // Callbacks from low level network for statistics gathering
pub fn stats_packet_sent(&self, addr: IpAddr, bytes: ByteCount) { pub(crate) fn stats_packet_sent(&self, addr: IpAddr, bytes: ByteCount) {
let inner = &mut *self.inner.lock(); let inner = &mut *self.inner.lock();
inner inner
.stats .stats
@ -52,7 +52,7 @@ impl NetworkManager {
.add_up(bytes); .add_up(bytes);
} }
pub fn stats_packet_rcvd(&self, addr: IpAddr, bytes: ByteCount) { pub(crate) fn stats_packet_rcvd(&self, addr: IpAddr, bytes: ByteCount) {
let inner = &mut *self.inner.lock(); let inner = &mut *self.inner.lock();
inner inner
.stats .stats

1
veilid-core/src/network_manager/tasks/mod.rs
View File

@ -48,6 +48,7 @@ impl NetworkManager {
} }
} }
#[instrument(level = "trace", name = "NetworkManager::tick", skip_all, err)]
pub async fn tick(&self) -> EyreResult<()> { pub async fn tick(&self) -> EyreResult<()> {
let routing_table = self.routing_table(); let routing_table = self.routing_table();
let net = self.net(); let net = self.net();

4
veilid-core/src/network_manager/tasks/public_address_check.rs
View File

@ -2,10 +2,10 @@ use super::*;
impl NetworkManager { impl NetworkManager {
// Clean up the public address check tables, removing entries that have timed out // Clean up the public address check tables, removing entries that have timed out
#[instrument(level = "trace", skip(self), err)] #[instrument(parent = None, level = "trace", skip_all, err)]
pub(crate) async fn public_address_check_task_routine( pub(crate) async fn public_address_check_task_routine(
self, self,
stop_token: StopToken, _stop_token: StopToken,
_last_ts: Timestamp, _last_ts: Timestamp,
cur_ts: Timestamp, cur_ts: Timestamp,
) -> EyreResult<()> { ) -> EyreResult<()> {

70
veilid-core/src/network_manager/wasm/mod.rs
View File

@ -52,12 +52,14 @@ pub const MAX_CAPABILITIES: usize = 64;
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
struct NetworkInner { struct NetworkInner {
network_started: Option<bool>,
network_needs_restart: bool, network_needs_restart: bool,
protocol_config: ProtocolConfig, protocol_config: ProtocolConfig,
} }
struct NetworkUnlockedInner { struct NetworkUnlockedInner {
// Startup lock
startup_lock: StartupLock,
// Accessors // Accessors
routing_table: RoutingTable, routing_table: RoutingTable,
network_manager: NetworkManager, network_manager: NetworkManager,
@ -74,7 +76,6 @@ pub(in crate::network_manager) struct Network {
impl Network { impl Network {
fn new_inner() -> NetworkInner { fn new_inner() -> NetworkInner {
NetworkInner { NetworkInner {
network_started: Some(false),
network_needs_restart: false, network_needs_restart: false,
protocol_config: Default::default(), protocol_config: Default::default(),
} }
@ -86,6 +87,7 @@ impl Network {
connection_manager: ConnectionManager, connection_manager: ConnectionManager,
) -> NetworkUnlockedInner { ) -> NetworkUnlockedInner {
NetworkUnlockedInner { NetworkUnlockedInner {
startup_lock: StartupLock::new(),
network_manager, network_manager,
routing_table, routing_table,
connection_manager, connection_manager,
@ -121,7 +123,7 @@ impl Network {
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
// Record DialInfo failures // Record DialInfo failures
pub async fn record_dial_info_failure<T, F: Future<Output = EyreResult<NetworkResult<T>>>>( async fn record_dial_info_failure<T, F: Future<Output = EyreResult<NetworkResult<T>>>>(
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
fut: F, fut: F,
@ -135,12 +137,18 @@ impl Network {
Ok(network_result) Ok(network_result)
} }
#[cfg_attr(feature="verbose-tracing", instrument(level="trace", err, skip(self, data), fields(data.len = data.len())))] // Send data to a dial info, unbound, using a new connection from a random port
// This creates a short-lived connection in the case of connection-oriented protocols
// for the purpose of sending this one message.
// This bypasses the connection table as it is not a 'node to node' connection.
#[instrument(level="trace", target="net", err, skip(self, data), fields(data.len = data.len()))]
pub async fn send_data_unbound_to_dial_info( pub async fn send_data_unbound_to_dial_info(
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<()>> { ) -> EyreResult<NetworkResult<()>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure(dial_info.clone(), async move { self.record_dial_info_failure(dial_info.clone(), async move {
let data_len = data.len(); let data_len = data.len();
let timeout_ms = { let timeout_ms = {
@ -187,13 +195,15 @@ impl Network {
// This creates a short-lived connection in the case of connection-oriented protocols // This creates a short-lived connection in the case of connection-oriented protocols
// for the purpose of sending this one message. // for the purpose of sending this one message.
// This bypasses the connection table as it is not a 'node to node' connection. // This bypasses the connection table as it is not a 'node to node' connection.
#[cfg_attr(feature="verbose-tracing", instrument(level="trace", err, skip(self, data), fields(data.len = data.len())))] #[instrument(level="trace", target="net", err, skip(self, data), fields(data.len = data.len()))]
pub async fn send_recv_data_unbound_to_dial_info( pub async fn send_recv_data_unbound_to_dial_info(
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
data: Vec<u8>, data: Vec<u8>,
timeout_ms: u32, timeout_ms: u32,
) -> EyreResult<NetworkResult<Vec<u8>>> { ) -> EyreResult<NetworkResult<Vec<u8>>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure(dial_info.clone(), async move { self.record_dial_info_failure(dial_info.clone(), async move {
let data_len = data.len(); let data_len = data.len();
let connect_timeout_ms = { let connect_timeout_ms = {
@ -247,12 +257,14 @@ impl Network {
.await .await
} }
#[cfg_attr(feature="verbose-tracing", instrument(level="trace", err, skip(self, data), fields(data.len = data.len())))] #[instrument(level="trace", target="net", err, skip(self, data), fields(data.len = data.len()))]
pub async fn send_data_to_existing_flow( pub async fn send_data_to_existing_flow(
&self, &self,
flow: Flow, flow: Flow,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<SendDataToExistingFlowResult> { ) -> EyreResult<SendDataToExistingFlowResult> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
let data_len = data.len(); let data_len = data.len();
match flow.protocol_type() { match flow.protocol_type() {
ProtocolType::UDP => { ProtocolType::UDP => {
@ -292,12 +304,16 @@ impl Network {
Ok(SendDataToExistingFlowResult::NotSent(data)) Ok(SendDataToExistingFlowResult::NotSent(data))
} }
#[cfg_attr(feature="verbose-tracing", instrument(level="trace", err, skip(self, data), fields(data.len = data.len())))] // Send data directly to a dial info, possibly without knowing which node it is going to
// Returns a flow for the connection used to send the data
#[instrument(level="trace", target="net", err, skip(self, data), fields(data.len = data.len()))]
pub async fn send_data_to_dial_info( pub async fn send_data_to_dial_info(
&self, &self,
dial_info: DialInfo, dial_info: DialInfo,
data: Vec<u8>, data: Vec<u8>,
) -> EyreResult<NetworkResult<UniqueFlow>> { ) -> EyreResult<NetworkResult<UniqueFlow>> {
let _guard = self.unlocked_inner.startup_lock.enter()?;
self.record_dial_info_failure(dial_info.clone(), async move { self.record_dial_info_failure(dial_info.clone(), async move {
let data_len = data.len(); let data_len = data.len();
if dial_info.protocol_type() == ProtocolType::UDP { if dial_info.protocol_type() == ProtocolType::UDP {
@ -399,23 +415,22 @@ impl Network {
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
#[instrument(level = "debug", err, skip_all)]
pub async fn startup(&self) -> EyreResult<StartupDisposition> { pub async fn startup(&self) -> EyreResult<StartupDisposition> {
self.inner.lock().network_started = None; let guard = self.unlocked_inner.startup_lock.startup()?;
match self.startup_internal().await { match self.startup_internal().await {
Ok(StartupDisposition::Success) => { Ok(StartupDisposition::Success) => {
info!("network started"); info!("network started");
self.inner.lock().network_started = Some(true); guard.success();
Ok(StartupDisposition::Success) Ok(StartupDisposition::Success)
} }
Ok(StartupDisposition::BindRetry) => { Ok(StartupDisposition::BindRetry) => {
debug!("network bind retry"); debug!("network bind retry");
self.inner.lock().network_started = Some(false);
Ok(StartupDisposition::BindRetry) Ok(StartupDisposition::BindRetry)
} }
Err(e) => { Err(e) => {
debug!("network failed to start"); debug!("network failed to start");
self.inner.lock().network_started = Some(false);
Err(e) Err(e)
} }
} }
@ -425,16 +440,22 @@ impl Network {
self.inner.lock().network_needs_restart self.inner.lock().network_needs_restart
} }
pub fn is_started(&self) -> Option<bool> { pub fn is_started(&self) -> bool {
self.inner.lock().network_started self.unlocked_inner.startup_lock.is_started()
} }
#[instrument(level = "debug", skip_all)]
pub fn restart_network(&self) { pub fn restart_network(&self) {
self.inner.lock().network_needs_restart = true; self.inner.lock().network_needs_restart = true;
} }
#[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
log_net!(debug "stopping network"); log_net!(debug "starting low level network shutdown");
let Ok(guard) = self.unlocked_inner.startup_lock.shutdown().await else {
log_net!(debug "low level network is already shut down");
return;
};
// Reset state // Reset state
let routing_table = self.routing_table(); let routing_table = self.routing_table();
@ -451,7 +472,8 @@ impl Network {
// Cancels all async background tasks by dropping join handles // Cancels all async background tasks by dropping join handles
*self.inner.lock() = Self::new_inner(); *self.inner.lock() = Self::new_inner();
log_net!(debug "network stopped"); guard.success();
log_net!(debug "finished low level network shutdown");
} }
pub fn get_preferred_local_address(&self, _dial_info: &DialInfo) -> Option<SocketAddr> { pub fn get_preferred_local_address(&self, _dial_info: &DialInfo) -> Option<SocketAddr> {
@ -472,15 +494,29 @@ impl Network {
&self, &self,
_punishment: Option<Box<dyn FnOnce() + Send + 'static>>, _punishment: Option<Box<dyn FnOnce() + Send + 'static>>,
) { ) {
// let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return;
};
} }
pub fn needs_public_dial_info_check(&self) -> bool { pub fn needs_public_dial_info_check(&self) -> bool {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return false;
};
false false
} }
////////////////////////////////////////// //////////////////////////////////////////
pub async fn tick(&self) -> EyreResult<()> { #[instrument(level = "trace", target = "net", name = "Network::tick", skip_all, err)]
pub(crate) async fn tick(&self) -> EyreResult<()> {
let Ok(_guard) = self.unlocked_inner.startup_lock.enter() else {
log_net!(debug "ignoring due to not started up");
return Ok(());
};
Ok(()) Ok(())
} }
} }

28
veilid-core/src/routing_table/mod.rs
View File

@ -221,14 +221,26 @@ impl RoutingTable {
node_id: c.network.routing_table.node_id.clone(), node_id: c.network.routing_table.node_id.clone(),
node_id_secret: c.network.routing_table.node_id_secret.clone(), node_id_secret: c.network.routing_table.node_id_secret.clone(),
kick_queue: Mutex::new(BTreeSet::default()), kick_queue: Mutex::new(BTreeSet::default()),
rolling_transfers_task: TickTask::new(ROLLING_TRANSFERS_INTERVAL_SECS), rolling_transfers_task: TickTask::new(
kick_buckets_task: TickTask::new(1), "rolling_transfers_task",
bootstrap_task: TickTask::new(1), ROLLING_TRANSFERS_INTERVAL_SECS,
peer_minimum_refresh_task: TickTask::new(1), ),
closest_peers_refresh_task: TickTask::new_ms(c.network.dht.min_peer_refresh_time_ms), kick_buckets_task: TickTask::new("kick_buckets_task", 1),
ping_validator_task: TickTask::new(1), bootstrap_task: TickTask::new("bootstrap_task", 1),
relay_management_task: TickTask::new(RELAY_MANAGEMENT_INTERVAL_SECS), peer_minimum_refresh_task: TickTask::new("peer_minimum_refresh_task", 1),
private_route_management_task: TickTask::new(PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS), closest_peers_refresh_task: TickTask::new_ms(
"closest_peers_refresh_task",
c.network.dht.min_peer_refresh_time_ms,
),
ping_validator_task: TickTask::new("ping_validator_task", 1),
relay_management_task: TickTask::new(
"relay_management_task",
RELAY_MANAGEMENT_INTERVAL_SECS,
),
private_route_management_task: TickTask::new(
"private_route_management_task",
PRIVATE_ROUTE_MANAGEMENT_INTERVAL_SECS,
),
} }
} }
pub fn new(network_manager: NetworkManager) -> Self { pub fn new(network_manager: NetworkManager) -> Self {

1
veilid-core/src/routing_table/tasks/mod.rs
View File

@ -112,6 +112,7 @@ impl RoutingTable {
/// Ticks about once per second /// Ticks about once per second
/// to run tick tasks which may run at slower tick rates as configured /// to run tick tasks which may run at slower tick rates as configured
#[instrument(level = "trace", name = "RoutingTable::tick", skip_all, err)]
pub async fn tick(&self) -> EyreResult<()> { pub async fn tick(&self) -> EyreResult<()> {
// Don't tick if paused // Don't tick if paused
let opt_tick_guard = { let opt_tick_guard = {

28
veilid-core/src/routing_table/tasks/ping_validator.rs
View File

@ -114,7 +114,6 @@ impl RoutingTable {
rpc.rpc_call_status(Destination::direct(relay_nr_filtered)) rpc.rpc_call_status(Destination::direct(relay_nr_filtered))
.await .await
} }
.instrument(Span::current())
.boxed(), .boxed(),
); );
} }
@ -159,9 +158,7 @@ impl RoutingTable {
log_rtab!("--> Watch ping to {:?}", watch_nr); log_rtab!("--> Watch ping to {:?}", watch_nr);
futurequeue.push_back( futurequeue.push_back(
async move { rpc.rpc_call_status(Destination::direct(watch_nr)).await } async move { rpc.rpc_call_status(Destination::direct(watch_nr)).await }.boxed(),
.instrument(Span::current())
.boxed(),
); );
} }
Ok(()) Ok(())
@ -198,9 +195,7 @@ impl RoutingTable {
let rpc = rpc.clone(); let rpc = rpc.clone();
log_rtab!("--> Validator ping to {:?}", nr); log_rtab!("--> Validator ping to {:?}", nr);
futurequeue.push_back( futurequeue.push_back(
async move { rpc.rpc_call_status(Destination::direct(nr)).await } async move { rpc.rpc_call_status(Destination::direct(nr)).await }.boxed(),
.instrument(Span::current())
.boxed(),
); );
} }
@ -226,9 +221,7 @@ impl RoutingTable {
// Just do a single ping with the best protocol for all the nodes // Just do a single ping with the best protocol for all the nodes
futurequeue.push_back( futurequeue.push_back(
async move { rpc.rpc_call_status(Destination::direct(nr)).await } async move { rpc.rpc_call_status(Destination::direct(nr)).await }.boxed(),
.instrument(Span::current())
.boxed(),
); );
} }
@ -258,10 +251,19 @@ impl RoutingTable {
let mut unord = FuturesUnordered::new(); let mut unord = FuturesUnordered::new();
while !unord.is_empty() || !futurequeue.is_empty() { while !unord.is_empty() || !futurequeue.is_empty() {
#[cfg(feature = "verbose-tracing")] log_rtab!(
log_rtab!(debug "Ping validation queue: {} remaining, {} in progress", futurequeue.len(), unord.len()); "Ping validation queue: {} remaining, {} in progress",
futurequeue.len(),
unord.len()
);
// Process one unordered futures if we have some // Process one unordered futures if we have some
match unord.next().timeout_at(stop_token.clone()).await { match unord
.next()
.timeout_at(stop_token.clone())
.in_current_span()
.await
{
Ok(Some(_)) => { Ok(Some(_)) => {
// Some ping completed // Some ping completed
} }

14
veilid-core/src/rpc_processor/fanout_call.rs
View File

@ -321,13 +321,17 @@ where
} }
} }
// Wait for them to complete // Wait for them to complete
timeout(timeout_ms, async { timeout(
while let Some(is_done) = unord.next().await { timeout_ms,
if is_done { async {
break; while let Some(is_done) = unord.next().in_current_span().await {
if is_done {
break;
}
} }
} }
}) .in_current_span(),
)
.await .await
.into_timeout_or() .into_timeout_or()
.map(|_| { .map(|_| {

60
veilid-core/src/rpc_processor/mod.rs
View File

@ -277,7 +277,7 @@ enum RPCKind {
///////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////
struct RPCProcessorInner { struct RPCProcessorInner {
send_channel: Option<flume::Sender<(Option<Id>, RPCMessageEncoded)>>, send_channel: Option<flume::Sender<(Span, RPCMessageEncoded)>>,
stop_source: Option<StopSource>, stop_source: Option<StopSource>,
worker_join_handles: Vec<MustJoinHandle<()>>, worker_join_handles: Vec<MustJoinHandle<()>>,
} }
@ -292,6 +292,7 @@ struct RPCProcessorUnlockedInner {
update_callback: UpdateCallback, update_callback: UpdateCallback,
waiting_rpc_table: OperationWaiter<RPCMessage, Option<QuestionContext>>, waiting_rpc_table: OperationWaiter<RPCMessage, Option<QuestionContext>>,
waiting_app_call_table: OperationWaiter<Vec<u8>, ()>, waiting_app_call_table: OperationWaiter<Vec<u8>, ()>,
startup_lock: StartupLock,
} }
#[derive(Clone)] #[derive(Clone)]
@ -345,6 +346,7 @@ impl RPCProcessor {
update_callback, update_callback,
waiting_rpc_table: OperationWaiter::new(), waiting_rpc_table: OperationWaiter::new(),
waiting_app_call_table: OperationWaiter::new(), waiting_app_call_table: OperationWaiter::new(),
startup_lock: StartupLock::new(),
} }
} }
pub fn new(network_manager: NetworkManager, update_callback: UpdateCallback) -> Self { pub fn new(network_manager: NetworkManager, update_callback: UpdateCallback) -> Self {
@ -377,6 +379,7 @@ impl RPCProcessor {
#[instrument(level = "debug", skip_all, err)] #[instrument(level = "debug", skip_all, err)]
pub async fn startup(&self) -> EyreResult<()> { pub async fn startup(&self) -> EyreResult<()> {
log_rpc!(debug "startup rpc processor"); log_rpc!(debug "startup rpc processor");
let guard = self.unlocked_inner.startup_lock.startup()?;
{ {
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
@ -389,10 +392,10 @@ impl RPCProcessor {
"Spinning up {} RPC workers", "Spinning up {} RPC workers",
self.unlocked_inner.concurrency self.unlocked_inner.concurrency
); );
for _ in 0..self.unlocked_inner.concurrency { for task_n in 0..self.unlocked_inner.concurrency {
let this = self.clone(); let this = self.clone();
let receiver = channel.1.clone(); let receiver = channel.1.clone();
let jh = spawn(Self::rpc_worker( let jh = spawn(&format!("rpc worker {}",task_n), Self::rpc_worker(
this, this,
inner.stop_source.as_ref().unwrap().token(), inner.stop_source.as_ref().unwrap().token(),
receiver, receiver,
@ -405,13 +408,18 @@ impl RPCProcessor {
self.storage_manager self.storage_manager
.set_rpc_processor(Some(self.clone())) .set_rpc_processor(Some(self.clone()))
.await; .await;
guard.success();
Ok(()) Ok(())
} }
#[instrument(level = "debug", skip_all)] #[instrument(level = "debug", skip_all)]
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
log_rpc!(debug "starting rpc processor shutdown"); log_rpc!(debug "starting rpc processor shutdown");
let Ok(guard) = self.unlocked_inner.startup_lock.shutdown().await else {
log_rpc!(debug "rpc processor already shut down");
return;
};
// Stop storage manager from using us // Stop storage manager from using us
self.storage_manager.set_rpc_processor(None).await; self.storage_manager.set_rpc_processor(None).await;
@ -437,6 +445,7 @@ impl RPCProcessor {
// Release the rpc processor // Release the rpc processor
*self.inner.lock() = Self::new_inner(); *self.inner.lock() = Self::new_inner();
guard.success();
log_rpc!(debug "finished rpc processor shutdown"); log_rpc!(debug "finished rpc processor shutdown");
} }
@ -536,9 +545,11 @@ impl RPCProcessor {
&self, &self,
node_id: TypedKey, node_id: TypedKey,
safety_selection: SafetySelection, safety_selection: SafetySelection,
) -> SendPinBoxFuture<Result<Option<NodeRef>, RPCError>> { ) -> SendPinBoxFuture<Result<Option<NodeRef>, RPCError>> {
let this = self.clone(); let this = self.clone();
Box::pin(async move { Box::pin(async move {
let _guard = this.unlocked_inner.startup_lock.enter().map_err(RPCError::map_try_again("not started up"))?;
let routing_table = this.routing_table(); let routing_table = this.routing_table();
// First see if we have the node in our routing table already // First see if we have the node in our routing table already
@ -579,7 +590,7 @@ impl RPCProcessor {
}; };
Ok(nr) Ok(nr)
}) }.in_current_span())
} }
#[instrument(level="trace", target="rpc", skip_all)] #[instrument(level="trace", target="rpc", skip_all)]
@ -1651,31 +1662,31 @@ impl RPCProcessor {
RPCStatementDetail::AppMessage(_) => self.process_app_message(msg).await, RPCStatementDetail::AppMessage(_) => self.process_app_message(msg).await,
}, },
RPCOperationKind::Answer(_) => { RPCOperationKind::Answer(_) => {
self.unlocked_inner let op_id = msg.operation.op_id();
if let Err(e) = self.unlocked_inner
.waiting_rpc_table .waiting_rpc_table
.complete_op_waiter(msg.operation.op_id(), msg) .complete_op_waiter(op_id, msg) {
.await?; log_rpc!(debug "Operation id {} did not complete: {}", op_id, e);
// Don't throw an error here because it's okay if the original operation timed out
}
Ok(NetworkResult::value(())) Ok(NetworkResult::value(()))
} }
} }
} }
#[instrument(level="trace", target="rpc", skip_all)]
async fn rpc_worker( async fn rpc_worker(
self, self,
stop_token: StopToken, stop_token: StopToken,
receiver: flume::Receiver<(Option<Id>, RPCMessageEncoded)>, receiver: flume::Receiver<(Span, RPCMessageEncoded)>,
) { ) {
while let Ok(Ok((_span_id, msg))) = while let Ok(Ok((prev_span, msg))) =
receiver.recv_async().timeout_at(stop_token.clone()).await receiver.recv_async().timeout_at(stop_token.clone()).await
{ {
//let rpc_worker_span = span!(parent: None, Level::TRACE, "rpc_worker recv"); let rpc_message_span = tracing::trace_span!("rpc message");
// xxx: causes crash (Missing otel data span extensions) rpc_message_span.follows_from(prev_span);
// rpc_worker_span.follows_from(span_id);
network_result_value_or_log!(match self network_result_value_or_log!(match self
.process_rpc_message(msg).in_current_span() .process_rpc_message(msg).instrument(rpc_message_span)
//.instrument(rpc_worker_span)
.await .await
{ {
Err(e) => { Err(e) => {
@ -1699,6 +1710,8 @@ impl RPCProcessor {
routing_domain: RoutingDomain, routing_domain: RoutingDomain,
body: Vec<u8>, body: Vec<u8>,
) -> EyreResult<()> { ) -> EyreResult<()> {
let _guard = self.unlocked_inner.startup_lock.enter().map_err(RPCError::map_try_again("not started up"))?;
let header = RPCMessageHeader { let header = RPCMessageHeader {
detail: RPCMessageHeaderDetail::Direct(RPCMessageHeaderDetailDirect { detail: RPCMessageHeaderDetail::Direct(RPCMessageHeaderDetailDirect {
envelope, envelope,
@ -1722,9 +1735,8 @@ impl RPCProcessor {
}; };
send_channel send_channel
}; };
let span_id = Span::current().id();
send_channel send_channel
.try_send((span_id, msg)) .try_send((Span::current(), msg))
.map_err(|e| eyre!("failed to enqueue direct RPC message: {}", e))?; .map_err(|e| eyre!("failed to enqueue direct RPC message: {}", e))?;
Ok(()) Ok(())
} }
@ -1758,9 +1770,8 @@ impl RPCProcessor {
}; };
send_channel send_channel
}; };
let span_id = Span::current().id();
send_channel send_channel
.try_send((span_id, msg)) .try_send((Span::current(), msg))
.map_err(|e| eyre!("failed to enqueue safety routed RPC message: {}", e))?; .map_err(|e| eyre!("failed to enqueue safety routed RPC message: {}", e))?;
Ok(()) Ok(())
} }
@ -1797,9 +1808,8 @@ impl RPCProcessor {
}; };
send_channel send_channel
}; };
let span_id = Span::current().id();
send_channel send_channel
.try_send((span_id, msg)) .try_send((Span::current(), msg))
.map_err(|e| eyre!("failed to enqueue private routed RPC message: {}", e))?; .map_err(|e| eyre!("failed to enqueue private routed RPC message: {}", e))?;
Ok(()) Ok(())
} }

53
veilid-core/src/rpc_processor/operation_waiter.rs
View File

@ -8,7 +8,7 @@ where
{ {
waiter: OperationWaiter<T, C>, waiter: OperationWaiter<T, C>,
op_id: OperationId, op_id: OperationId,
eventual_instance: Option<EventualValueFuture<(Option<Id>, T)>>, result_receiver: Option<flume::Receiver<(Span, T)>>,
} }
impl<T, C> Drop for OperationWaitHandle<T, C> impl<T, C> Drop for OperationWaitHandle<T, C>
@ -17,7 +17,7 @@ where
C: Unpin + Clone, C: Unpin + Clone,
{ {
fn drop(&mut self) { fn drop(&mut self) {
if self.eventual_instance.is_some() { if self.result_receiver.is_some() {
self.waiter.cancel_op_waiter(self.op_id); self.waiter.cancel_op_waiter(self.op_id);
} }
} }
@ -31,7 +31,7 @@ where
{ {
context: C, context: C,
timestamp: Timestamp, timestamp: Timestamp,
eventual: EventualValue<(Option<Id>, T)>, result_sender: flume::Sender<(Span, T)>,
} }
#[derive(Debug)] #[derive(Debug)]
@ -80,11 +80,11 @@ where
/// Set up wait for operation to complete /// Set up wait for operation to complete
pub fn add_op_waiter(&self, op_id: OperationId, context: C) -> OperationWaitHandle<T, C> { pub fn add_op_waiter(&self, op_id: OperationId, context: C) -> OperationWaitHandle<T, C> {
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
let e = EventualValue::new(); let (result_sender, result_receiver) = flume::bounded(1);
let waiting_op = OperationWaitingOp { let waiting_op = OperationWaitingOp {
context, context,
timestamp: get_aligned_timestamp(), timestamp: get_aligned_timestamp(),
eventual: e.clone(), result_sender,
}; };
if inner.waiting_op_table.insert(op_id, waiting_op).is_some() { if inner.waiting_op_table.insert(op_id, waiting_op).is_some() {
error!( error!(
@ -96,7 +96,7 @@ where
OperationWaitHandle { OperationWaitHandle {
waiter: self.clone(), waiter: self.clone(),
op_id, op_id,
eventual_instance: Some(e.instance()), result_receiver: Some(result_receiver),
} }
} }
@ -122,14 +122,15 @@ where
} }
/// Remove wait for op /// Remove wait for op
#[instrument(level = "trace", target = "rpc", skip_all)]
fn cancel_op_waiter(&self, op_id: OperationId) { fn cancel_op_waiter(&self, op_id: OperationId) {
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
inner.waiting_op_table.remove(&op_id); inner.waiting_op_table.remove(&op_id);
} }
/// Complete the app call /// Complete the waiting op
#[instrument(level = "trace", target = "rpc", skip_all)] #[instrument(level = "trace", target = "rpc", skip_all)]
pub async fn complete_op_waiter(&self, op_id: OperationId, message: T) -> Result<(), RPCError> { pub fn complete_op_waiter(&self, op_id: OperationId, message: T) -> Result<(), RPCError> {
let waiting_op = { let waiting_op = {
let mut inner = self.inner.lock(); let mut inner = self.inner.lock();
inner inner
@ -141,10 +142,9 @@ where
)))? )))?
}; };
waiting_op waiting_op
.eventual .result_sender
.resolve((Span::current().id(), message)) .send((Span::current(), message))
.await; .map_err(RPCError::ignore)
Ok(())
} }
/// Wait for operation to complete /// Wait for operation to complete
@ -156,29 +156,30 @@ where
) -> Result<TimeoutOr<(T, TimestampDuration)>, RPCError> { ) -> Result<TimeoutOr<(T, TimestampDuration)>, RPCError> {
let timeout_ms = us_to_ms(timeout_us.as_u64()).map_err(RPCError::internal)?; let timeout_ms = us_to_ms(timeout_us.as_u64()).map_err(RPCError::internal)?;
// Take the instance // Take the receiver
// After this, we must manually cancel since the cancel on handle drop is disabled // After this, we must manually cancel since the cancel on handle drop is disabled
let eventual_instance = handle.eventual_instance.take().unwrap(); let result_receiver = handle.result_receiver.take().unwrap();
let result_fut = result_receiver.recv_async().in_current_span();
// wait for eventualvalue // wait for eventualvalue
let start_ts = get_aligned_timestamp(); let start_ts = get_aligned_timestamp();
let res = timeout(timeout_ms, eventual_instance) let res = timeout(timeout_ms, result_fut).await.into_timeout_or();
.await
.into_timeout_or(); match res {
Ok(res TimeoutOr::Timeout => {
.on_timeout(|| {
// log_rpc!(debug "op wait timed out: {}", handle.op_id);
// debug_print_backtrace();
self.cancel_op_waiter(handle.op_id); self.cancel_op_waiter(handle.op_id);
}) Ok(TimeoutOr::Timeout)
.map(|res| { }
let (_span_id, ret) = res.take_value().unwrap(); TimeoutOr::Value(Ok((_span_id, ret))) => {
let end_ts = get_aligned_timestamp(); let end_ts = get_aligned_timestamp();
//xxx: causes crash (Missing otel data span extensions) //xxx: causes crash (Missing otel data span extensions)
// Span::current().follows_from(span_id); // Span::current().follows_from(span_id);
(ret, end_ts.saturating_sub(start_ts)) Ok(TimeoutOr::Value((ret, end_ts.saturating_sub(start_ts))))
})) }
TimeoutOr::Value(Err(e)) => Err(RPCError::ignore(e)),
}
} }
} }

19
veilid-core/src/rpc_processor/rpc_app_call.rs
View File

@ -9,6 +9,12 @@ impl RPCProcessor {
dest: Destination, dest: Destination,
message: Vec<u8>, message: Vec<u8>,
) -> RPCNetworkResult<Answer<Vec<u8>>> { ) -> RPCNetworkResult<Answer<Vec<u8>>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
let debug_string = format!("AppCall(message(len)={}) => {}", message.len(), dest); let debug_string = format!("AppCall(message(len)={}) => {}", message.len(), dest);
let app_call_q = RPCOperationAppCallQ::new(message)?; let app_call_q = RPCOperationAppCallQ::new(message)?;
@ -147,14 +153,15 @@ impl RPCProcessor {
/// Exposed to API for apps to return app call answers /// Exposed to API for apps to return app call answers
#[instrument(level = "trace", target = "rpc", skip_all)] #[instrument(level = "trace", target = "rpc", skip_all)]
pub async fn app_call_reply( pub fn app_call_reply(&self, call_id: OperationId, message: Vec<u8>) -> Result<(), RPCError> {
&self, let _guard = self
call_id: OperationId, .unlocked_inner
message: Vec<u8>, .startup_lock
) -> Result<(), RPCError> { .enter()
.map_err(RPCError::map_try_again("not started up"))?;
self.unlocked_inner self.unlocked_inner
.waiting_app_call_table .waiting_app_call_table
.complete_op_waiter(call_id, message) .complete_op_waiter(call_id, message)
.await .map_err(RPCError::ignore)
} }
} }

6
veilid-core/src/rpc_processor/rpc_app_message.rs
View File

@ -9,6 +9,12 @@ impl RPCProcessor {
dest: Destination, dest: Destination,
message: Vec<u8>, message: Vec<u8>,
) -> RPCNetworkResult<()> { ) -> RPCNetworkResult<()> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
let app_message = RPCOperationAppMessage::new(message)?; let app_message = RPCOperationAppMessage::new(message)?;
let statement = RPCStatement::new(RPCStatementDetail::AppMessage(Box::new(app_message))); let statement = RPCStatement::new(RPCStatementDetail::AppMessage(Box::new(app_message)));

6
veilid-core/src/rpc_processor/rpc_find_node.rs
View File

@ -14,6 +14,12 @@ impl RPCProcessor {
node_id: TypedKey, node_id: TypedKey,
capabilities: Vec<Capability>, capabilities: Vec<Capability>,
) -> RPCNetworkResult<Answer<Vec<PeerInfo>>> { ) -> RPCNetworkResult<Answer<Vec<PeerInfo>>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
if matches!( if matches!(
dest, dest,

6
veilid-core/src/rpc_processor/rpc_get_value.rs
View File

@ -30,6 +30,12 @@ impl RPCProcessor {
subkey: ValueSubkey, subkey: ValueSubkey,
last_descriptor: Option<SignedValueDescriptor>, last_descriptor: Option<SignedValueDescriptor>,
) ->RPCNetworkResult<Answer<GetValueAnswer>> { ) ->RPCNetworkResult<Answer<GetValueAnswer>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
// and get the target noderef so we can validate the response // and get the target noderef so we can validate the response
let Some(target) = dest.node() else { let Some(target) = dest.node() else {

6
veilid-core/src/rpc_processor/rpc_inspect_value.rs
View File

@ -32,6 +32,12 @@ impl RPCProcessor {
subkeys: ValueSubkeyRangeSet, subkeys: ValueSubkeyRangeSet,
last_descriptor: Option<SignedValueDescriptor>, last_descriptor: Option<SignedValueDescriptor>,
) -> RPCNetworkResult<Answer<InspectValueAnswer>> { ) -> RPCNetworkResult<Answer<InspectValueAnswer>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
// and get the target noderef so we can validate the response // and get the target noderef so we can validate the response
let Some(target) = dest.node() else { let Some(target) = dest.node() else {

6
veilid-core/src/rpc_processor/rpc_return_receipt.rs
View File

@ -9,6 +9,12 @@ impl RPCProcessor {
dest: Destination, dest: Destination,
receipt: D, receipt: D,
) -> RPCNetworkResult<()> { ) -> RPCNetworkResult<()> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
let receipt = receipt.as_ref().to_vec(); let receipt = receipt.as_ref().to_vec();
let return_receipt = RPCOperationReturnReceipt::new(receipt)?; let return_receipt = RPCOperationReturnReceipt::new(receipt)?;

6
veilid-core/src/rpc_processor/rpc_set_value.rs
View File

@ -34,6 +34,12 @@ impl RPCProcessor {
descriptor: SignedValueDescriptor, descriptor: SignedValueDescriptor,
send_descriptor: bool, send_descriptor: bool,
) ->RPCNetworkResult<Answer<SetValueAnswer>> { ) ->RPCNetworkResult<Answer<SetValueAnswer>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
// and get the target noderef so we can validate the response // and get the target noderef so we can validate the response
let Some(target) = dest.node() else { let Some(target) = dest.node() else {

6
veilid-core/src/rpc_processor/rpc_signal.rs
View File

@ -9,6 +9,12 @@ impl RPCProcessor {
dest: Destination, dest: Destination,
signal_info: SignalInfo, signal_info: SignalInfo,
) -> RPCNetworkResult<()> { ) -> RPCNetworkResult<()> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
if matches!( if matches!(
dest, dest,

6
veilid-core/src/rpc_processor/rpc_status.rs
View File

@ -20,6 +20,12 @@ impl RPCProcessor {
self, self,
dest: Destination, dest: Destination,
) -> RPCNetworkResult<Answer<Option<SenderInfo>>> { ) -> RPCNetworkResult<Answer<Option<SenderInfo>>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Determine routing domain and node status to send // Determine routing domain and node status to send
let (opt_target_nr, routing_domain, node_status) = if let Some(UnsafeRoutingInfo { let (opt_target_nr, routing_domain, node_status) = if let Some(UnsafeRoutingInfo {
opt_node, opt_node,

55
veilid-core/src/rpc_processor/rpc_validate_dial_info.rs
View File

@ -10,6 +10,17 @@ impl RPCProcessor {
dial_info: DialInfo, dial_info: DialInfo,
redirect: bool, redirect: bool,
) -> Result<bool, RPCError> { ) -> Result<bool, RPCError> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
let stop_token = self
.unlocked_inner
.startup_lock
.stop_token()
.ok_or(RPCError::try_again("not started up"))?;
let network_manager = self.network_manager(); let network_manager = self.network_manager();
let receipt_time = ms_to_us(self.unlocked_inner.validate_dial_info_receipt_time_ms); let receipt_time = ms_to_us(self.unlocked_inner.validate_dial_info_receipt_time_ms);
@ -32,23 +43,35 @@ impl RPCProcessor {
); );
// Wait for receipt // Wait for receipt
match eventual_value.await.take_value().unwrap() { match eventual_value
ReceiptEvent::ReturnedPrivate { private_route: _ } .timeout_at(stop_token)
| ReceiptEvent::ReturnedInBand { inbound_noderef: _ } .in_current_span()
| ReceiptEvent::ReturnedSafety => { .await
log_net!(debug "validate_dial_info receipt should be returned out-of-band"); {
Ok(false) Err(_) => {
return Err(RPCError::try_again("not started up"));
} }
ReceiptEvent::ReturnedOutOfBand => { Ok(v) => {
log_net!(debug "validate_dial_info receipt returned"); let receipt_event = v.take_value().unwrap();
Ok(true) match receipt_event {
} ReceiptEvent::ReturnedPrivate { private_route: _ }
ReceiptEvent::Expired => { | ReceiptEvent::ReturnedInBand { inbound_noderef: _ }
log_net!(debug "validate_dial_info receipt expired"); | ReceiptEvent::ReturnedSafety => {
Ok(false) log_net!(debug "validate_dial_info receipt should be returned out-of-band");
} Ok(false)
ReceiptEvent::Cancelled => { }
Err(RPCError::internal("receipt was dropped before expiration")) ReceiptEvent::ReturnedOutOfBand => {
log_net!(debug "validate_dial_info receipt returned");
Ok(true)
}
ReceiptEvent::Expired => {
log_net!(debug "validate_dial_info receipt expired");
Ok(false)
}
ReceiptEvent::Cancelled => {
Err(RPCError::internal("receipt was dropped before expiration"))
}
}
} }
} }
} }

6
veilid-core/src/rpc_processor/rpc_value_changed.rs
View File

@ -13,6 +13,12 @@ impl RPCProcessor {
watch_id: u64, watch_id: u64,
value: Option<SignedValueData>, value: Option<SignedValueData>,
) -> RPCNetworkResult<()> { ) -> RPCNetworkResult<()> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination is never using a safety route // Ensure destination is never using a safety route
if matches!(dest.get_safety_selection(), SafetySelection::Safe(_)) { if matches!(dest.get_safety_selection(), SafetySelection::Safe(_)) {
return Err(RPCError::internal( return Err(RPCError::internal(

6
veilid-core/src/rpc_processor/rpc_watch_value.rs
View File

@ -32,6 +32,12 @@ impl RPCProcessor {
watcher: KeyPair, watcher: KeyPair,
watch_id: Option<u64>, watch_id: Option<u64>,
) -> RPCNetworkResult<Answer<WatchValueAnswer>> { ) -> RPCNetworkResult<Answer<WatchValueAnswer>> {
let _guard = self
.unlocked_inner
.startup_lock
.enter()
.map_err(RPCError::map_try_again("not started up"))?;
// Ensure destination never has a private route // Ensure destination never has a private route
// and get the target noderef so we can validate the response // and get the target noderef so we can validate the response
let Some(target) = dest.node() else { let Some(target) = dest.node() else {

8
veilid-core/src/storage_manager/get_value.rs
View File

@ -182,7 +182,7 @@ impl StorageManager {
log_network_result!(debug "GetValue fanout call returned peers {}", gva.answer.peers.len()); log_network_result!(debug "GetValue fanout call returned peers {}", gva.answer.peers.len());
Ok(NetworkResult::value(gva.answer.peers)) Ok(NetworkResult::value(gva.answer.peers))
}.in_current_span() }.instrument(tracing::trace_span!("outbound_get_value fanout routine"))
} }
}; };
@ -225,7 +225,7 @@ impl StorageManager {
}; };
// Call the fanout in a spawned task // Call the fanout in a spawned task
spawn(Box::pin(async move { spawn("outbound_get_value fanout", Box::pin(async move {
let fanout_call = FanoutCall::new( let fanout_call = FanoutCall::new(
routing_table.clone(), routing_table.clone(),
key, key,
@ -271,7 +271,7 @@ impl StorageManager {
})) { })) {
log_dht!(debug "Sending GetValue result failed: {}", e); log_dht!(debug "Sending GetValue result failed: {}", e);
} }
}.in_current_span())) }.instrument(tracing::trace_span!("outbound_get_value result"))))
.detach(); .detach();
Ok(out_rx) Ok(out_rx)
@ -319,7 +319,7 @@ impl StorageManager {
// Return done // Return done
false false
}.in_current_span()) }.instrument(tracing::trace_span!("outbound_get_value deferred results")))
}, },
), ),
); );

2
veilid-core/src/storage_manager/inspect_value.rs
View File

@ -228,7 +228,7 @@ impl StorageManager {
log_network_result!(debug "InspectValue fanout call returned peers {}", answer.peers.len()); log_network_result!(debug "InspectValue fanout call returned peers {}", answer.peers.len());
Ok(NetworkResult::value(answer.peers)) Ok(NetworkResult::value(answer.peers))
}.in_current_span() }.instrument(tracing::trace_span!("outbound_inspect_value fanout call"))
}; };
// Routine to call to check if we're done at each step // Routine to call to check if we're done at each step

25
veilid-core/src/storage_manager/mod.rs
View File

@ -89,11 +89,26 @@ impl StorageManager {
table_store, table_store,
#[cfg(feature = "unstable-blockstore")] #[cfg(feature = "unstable-blockstore")]
block_store, block_store,
flush_record_stores_task: TickTask::new(FLUSH_RECORD_STORES_INTERVAL_SECS), flush_record_stores_task: TickTask::new(
offline_subkey_writes_task: TickTask::new(OFFLINE_SUBKEY_WRITES_INTERVAL_SECS), "flush_record_stores_task",
send_value_changes_task: TickTask::new(SEND_VALUE_CHANGES_INTERVAL_SECS), FLUSH_RECORD_STORES_INTERVAL_SECS,
check_active_watches_task: TickTask::new(CHECK_ACTIVE_WATCHES_INTERVAL_SECS), ),
check_watched_records_task: TickTask::new(CHECK_WATCHED_RECORDS_INTERVAL_SECS), offline_subkey_writes_task: TickTask::new(
"offline_subkey_writes_task",
OFFLINE_SUBKEY_WRITES_INTERVAL_SECS,
),
send_value_changes_task: TickTask::new(
"send_value_changes_task",
SEND_VALUE_CHANGES_INTERVAL_SECS,
),
check_active_watches_task: TickTask::new(
"check_active_watches_task",
CHECK_ACTIVE_WATCHES_INTERVAL_SECS,
),
check_watched_records_task: TickTask::new(
"check_watched_records_task",
CHECK_WATCHED_RECORDS_INTERVAL_SECS,
),
anonymous_watch_keys, anonymous_watch_keys,
} }

8
veilid-core/src/storage_manager/set_value.rs
View File

@ -177,7 +177,7 @@ impl StorageManager {
ctx.send_partial_update = true; ctx.send_partial_update = true;
Ok(NetworkResult::value(sva.answer.peers)) Ok(NetworkResult::value(sva.answer.peers))
}.in_current_span() }.instrument(tracing::trace_span!("fanout call_routine"))
} }
}; };
@ -224,7 +224,7 @@ impl StorageManager {
}; };
// Call the fanout in a spawned task // Call the fanout in a spawned task
spawn(Box::pin(async move { spawn("outbound_set_value fanout", Box::pin(async move {
let fanout_call = FanoutCall::new( let fanout_call = FanoutCall::new(
routing_table.clone(), routing_table.clone(),
key, key,
@ -267,7 +267,7 @@ impl StorageManager {
})) { })) {
log_dht!(debug "Sending SetValue result failed: {}", e); log_dht!(debug "Sending SetValue result failed: {}", e);
} }
}.in_current_span())) }.instrument(tracing::trace_span!("outbound_set_value fanout routine"))))
.detach(); .detach();
Ok(out_rx) Ok(out_rx)
@ -329,7 +329,7 @@ impl StorageManager {
// Return done // Return done
false false
}.in_current_span()) }.instrument(tracing::trace_span!("outbound_set_value deferred results")))
}, },
), ),
); );

2
veilid-core/src/storage_manager/storage_manager_inner.rs
View File

@ -133,7 +133,7 @@ impl StorageManagerInner {
self.deferred_result_processor.init().await; self.deferred_result_processor.init().await;
// Schedule tick // Schedule tick
let tick_future = interval(1000, move || { let tick_future = interval("storage manager tick", 1000, move || {
let this = outer_self.clone(); let this = outer_self.clone();
async move { async move {
if let Err(e) = this.tick().await { if let Err(e) = this.tick().await {

2
veilid-core/src/storage_manager/tasks/mod.rs
View File

@ -80,6 +80,7 @@ impl StorageManager {
} }
} }
#[instrument(parent = None, level = "trace", target = "stor", name = "StorageManager::tick", skip_all, err)]
pub async fn tick(&self) -> EyreResult<()> { pub async fn tick(&self) -> EyreResult<()> {
// Run the flush stores task // Run the flush stores task
self.unlocked_inner.flush_record_stores_task.tick().await?; self.unlocked_inner.flush_record_stores_task.tick().await?;
@ -109,6 +110,7 @@ impl StorageManager {
Ok(()) Ok(())
} }
#[instrument(level = "trace", target = "stor", skip_all)]
pub(crate) async fn cancel_tasks(&self) { pub(crate) async fn cancel_tasks(&self) {
log_stor!(debug "stopping check watched records task"); log_stor!(debug "stopping check watched records task");
if let Err(e) = self.unlocked_inner.check_watched_records_task.stop().await { if let Err(e) = self.unlocked_inner.check_watched_records_task.stop().await {

19
veilid-core/src/storage_manager/tasks/send_value_changes.rs
View File

@ -32,15 +32,24 @@ impl StorageManager {
// Add a future for each value change // Add a future for each value change
for vc in value_changes { for vc in value_changes {
let this = self.clone(); let this = self.clone();
unord.push(async move { unord.push(
if let Err(e) = this.send_value_change(vc).await { async move {
log_stor!(debug "Failed to send value change: {}", e); if let Err(e) = this.send_value_change(vc).await {
log_stor!(debug "Failed to send value change: {}", e);
}
} }
}); .in_current_span(),
);
} }
while !unord.is_empty() { while !unord.is_empty() {
match unord.next().timeout_at(stop_token.clone()).await { match unord
.next()
.in_current_span()
.timeout_at(stop_token.clone())
.in_current_span()
.await
{
Ok(Some(_)) => { Ok(Some(_)) => {
// Some ValueChanged completed // Some ValueChanged completed
} }

2
veilid-core/src/storage_manager/watch_value.rs
View File

@ -294,7 +294,7 @@ impl StorageManager {
log_network_result!(debug "WatchValue fanout call returned peers {} ({})", wva.answer.peers.len(), next_node); log_network_result!(debug "WatchValue fanout call returned peers {} ({})", wva.answer.peers.len(), next_node);
Ok(NetworkResult::value(wva.answer.peers)) Ok(NetworkResult::value(wva.answer.peers))
}.in_current_span() }.instrument(tracing::trace_span!("outbound_watch_value call routine"))
}; };
// Routine to call to check if we're done at each step // Routine to call to check if we're done at each step

3
veilid-core/src/veilid_api/api.rs
View File

@ -15,7 +15,7 @@ impl fmt::Debug for VeilidAPIInner {
impl Drop for VeilidAPIInner { impl Drop for VeilidAPIInner {
fn drop(&mut self) { fn drop(&mut self) {
if let Some(context) = self.context.take() { if let Some(context) = self.context.take() {
spawn_detached(api_shutdown(context)); spawn_detached("api shutdown", api_shutdown(context));
} }
} }
} }
@ -366,7 +366,6 @@ impl VeilidAPI {
let rpc_processor = self.rpc_processor()?; let rpc_processor = self.rpc_processor()?;
rpc_processor rpc_processor
.app_call_reply(call_id, message) .app_call_reply(call_id, message)
.await
.map_err(|e| e.into()) .map_err(|e| e.into())
} }

6
veilid-core/src/veilid_api/debug.rs
View File

@ -867,9 +867,11 @@ impl VeilidAPI {
.purge_last_connections(); .purge_last_connections();
if let Some(connection_manager) = &opt_connection_manager { if let Some(connection_manager) = &opt_connection_manager {
connection_manager.startup().await; connection_manager
.startup()
.await
.map_err(VeilidAPIError::internal)?;
} }
Ok("Connections purged".to_owned()) Ok("Connections purged".to_owned())
} else if args[0] == "routes" { } else if args[0] == "routes" {
// Purge route spec store // Purge route spec store

2
veilid-server/Cargo.toml
View File

@ -38,6 +38,7 @@ rt-tokio = [
] ]
tracking = ["veilid-core/tracking"] tracking = ["veilid-core/tracking"]
debug-json-api = [] debug-json-api = []
debug-locks = ["veilid-core/debug-locks"]
[dependencies] [dependencies]
veilid-core = { path = "../veilid-core", default-features = false } veilid-core = { path = "../veilid-core", default-features = false }
@ -81,6 +82,7 @@ stop-token = { version = "^0", default-features = false }
sysinfo = { version = "^0.30.13" } sysinfo = { version = "^0.30.13" }
wg = { version = "^0.9.1", features = ["future"] } wg = { version = "^0.9.1", features = ["future"] }
tracing-flame = "0.2.0" tracing-flame = "0.2.0"
tracing-perfetto = "0.1.1"
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
windows-service = "^0" windows-service = "^0"

14
veilid-server/src/client_api.rs
View File

@ -145,7 +145,11 @@ impl ClientApi {
let t_awg = awg.clone(); let t_awg = awg.clone();
// Process the connection // Process the connection
spawn(self.clone().handle_ipc_connection(stream, t_awg)).detach(); spawn(
"client_api handle_ipc_connection",
self.clone().handle_ipc_connection(stream, t_awg),
)
.detach();
} }
// Wait for all connections to terminate // Wait for all connections to terminate
@ -183,7 +187,11 @@ impl ClientApi {
let t_awg = awg.clone(); let t_awg = awg.clone();
// Process the connection // Process the connection
spawn(self.clone().handle_tcp_connection(stream, t_awg)).detach(); spawn(
"client_api handle_tcp_connection",
self.clone().handle_tcp_connection(stream, t_awg),
)
.detach();
} }
// Wait for all connections to terminate // Wait for all connections to terminate
@ -543,6 +551,6 @@ impl ClientApi {
} }
let bind_futures_join = join_all(bind_futures); let bind_futures_join = join_all(bind_futures);
self.inner.lock().join_handle = Some(spawn(bind_futures_join)); self.inner.lock().join_handle = Some(spawn("client_api bind_futures", bind_futures_join));
} }
} }

18
veilid-server/src/main.rs
View File

@ -84,10 +84,14 @@ pub struct CmdlineArgs {
#[arg(long, value_name = "endpoint")] #[arg(long, value_name = "endpoint")]
otlp: Option<String>, otlp: Option<String>,
/// Turn on flamegraph tracing (experimental, isn't terribly useful) /// Turn on flamegraph tracing (experimental)
#[arg(long, hide = true, value_name = "PATH", num_args=0..=1, require_equals=true, default_missing_value = "")] #[arg(long, hide = true, value_name = "PATH", num_args=0..=1, require_equals=true, default_missing_value = "")]
flame: Option<OsString>, flame: Option<OsString>,
/// Turn on perfetto tracing (experimental)
#[arg(long, hide = true, value_name = "PATH", num_args=0..=1, require_equals=true, default_missing_value = "")]
perfetto: Option<OsString>,
/// Run as an extra daemon on the same machine for testing purposes, specify a number greater than zero to offset the listening ports /// Run as an extra daemon on the same machine for testing purposes, specify a number greater than zero to offset the listening ports
#[arg(long)] #[arg(long)]
subnode_index: Option<u16>, subnode_index: Option<u16>,
@ -223,6 +227,18 @@ fn main() -> EyreResult<()> {
settingsrw.logging.flame.enabled = true; settingsrw.logging.flame.enabled = true;
settingsrw.logging.flame.path = flame; settingsrw.logging.flame.path = flame;
} }
if let Some(perfetto) = args.perfetto {
let perfetto = if perfetto.is_empty() {
Settings::get_default_perfetto_path(settingsrw.testing.subnode_index)
.to_string_lossy()
.to_string()
} else {
perfetto.to_string_lossy().to_string()
};
println!("Enabling perfetto output to {}", perfetto);
settingsrw.logging.perfetto.enabled = true;
settingsrw.logging.perfetto.path = perfetto;
}
if args.no_attach { if args.no_attach {
settingsrw.auto_attach = false; settingsrw.auto_attach = false;

34
veilid-server/src/server.rs
View File

@ -108,25 +108,29 @@ pub async fn run_veilid_server(
let capi2 = capi.clone(); let capi2 = capi.clone();
let update_receiver_shutdown = SingleShotEventual::new(Some(())); let update_receiver_shutdown = SingleShotEventual::new(Some(()));
let mut update_receiver_shutdown_instance = update_receiver_shutdown.instance().fuse(); let mut update_receiver_shutdown_instance = update_receiver_shutdown.instance().fuse();
let update_receiver_jh = spawn_local(async move { let update_receiver_jh = spawn_local(
loop { "update_receiver",
select! { async move {
res = receiver.recv_async() => { loop {
if let Ok(change) = res { select! {
if let Some(capi) = &capi2 { res = receiver.recv_async() => {
// Handle state changes on main thread for capnproto rpc if let Ok(change) = res {
capi.clone().handle_update(change); if let Some(capi) = &capi2 {
// Handle state changes on main thread for capnproto rpc
capi.clone().handle_update(change);
}
} else {
break;
} }
} else { }
_ = update_receiver_shutdown_instance => {
break; break;
} }
} };
_ = update_receiver_shutdown_instance => { }
break;
}
};
} }
}); .in_current_span(),
);
// Auto-attach if desired // Auto-attach if desired
let mut out = Ok(()); let mut out = Ok(());

23
veilid-server/src/settings.rs
View File

@ -69,6 +69,9 @@ logging:
flame: flame:
enabled: false enabled: false
path: '' path: ''
perfetto:
enabled: false
path: ''
console: console:
enabled: false enabled: false
testing: testing:
@ -451,6 +454,12 @@ pub struct Flame {
pub path: String, pub path: String,
} }
#[derive(Debug, Deserialize, Serialize)]
pub struct Perfetto {
pub enabled: bool,
pub path: String,
}
#[derive(Debug, Deserialize, Serialize)] #[derive(Debug, Deserialize, Serialize)]
pub struct Console { pub struct Console {
pub enabled: bool, pub enabled: bool,
@ -503,6 +512,7 @@ pub struct Logging {
pub api: Api, pub api: Api,
pub otlp: Otlp, pub otlp: Otlp,
pub flame: Flame, pub flame: Flame,
pub perfetto: Perfetto,
pub console: Console, pub console: Console,
} }
@ -873,6 +883,15 @@ impl Settings {
}) })
} }
/// Determine default perfetto output path
pub fn get_default_perfetto_path(subnode_index: u16) -> PathBuf {
std::env::temp_dir().join(if subnode_index == 0 {
"veilid-server.pftrace".to_owned()
} else {
format!("veilid-server-{}.pftrace", subnode_index)
})
}
#[allow(dead_code)] #[allow(dead_code)]
fn get_or_create_private_directory<P: AsRef<Path>>(path: P, group_read: bool) -> bool { fn get_or_create_private_directory<P: AsRef<Path>>(path: P, group_read: bool) -> bool {
let path = path.as_ref(); let path = path.as_ref();
@ -996,6 +1015,8 @@ impl Settings {
set_config_value!(inner.logging.otlp.ignore_log_targets, value); set_config_value!(inner.logging.otlp.ignore_log_targets, value);
set_config_value!(inner.logging.flame.enabled, value); set_config_value!(inner.logging.flame.enabled, value);
set_config_value!(inner.logging.flame.path, value); set_config_value!(inner.logging.flame.path, value);
set_config_value!(inner.logging.perfetto.enabled, value);
set_config_value!(inner.logging.perfetto.path, value);
set_config_value!(inner.logging.console.enabled, value); set_config_value!(inner.logging.console.enabled, value);
set_config_value!(inner.testing.subnode_index, value); set_config_value!(inner.testing.subnode_index, value);
set_config_value!(inner.core.capabilities.disable, value); set_config_value!(inner.core.capabilities.disable, value);
@ -1565,6 +1586,8 @@ mod tests {
); );
assert!(!s.logging.flame.enabled); assert!(!s.logging.flame.enabled);
assert_eq!(s.logging.flame.path, ""); assert_eq!(s.logging.flame.path, "");
assert!(!s.logging.perfetto.enabled);
assert_eq!(s.logging.perfetto.path, "");
assert!(!s.logging.console.enabled); assert!(!s.logging.console.enabled);
assert_eq!(s.testing.subnode_index, 0); assert_eq!(s.testing.subnode_index, 0);

2
veilid-server/src/unix.rs
View File

@ -34,7 +34,7 @@ pub async fn run_veilid_server_with_signals(
Signals::new([SIGHUP, SIGTERM, SIGINT, SIGQUIT]).wrap_err("failed to init signals")?; Signals::new([SIGHUP, SIGTERM, SIGINT, SIGQUIT]).wrap_err("failed to init signals")?;
let handle = signals.handle(); let handle = signals.handle();
let signals_task = spawn(handle_signals(signals)); let signals_task = spawn("signals", handle_signals(signals));
// Run veilid server // Run veilid server
let res = run_veilid_server(settings, server_mode, veilid_logs).await; let res = run_veilid_server(settings, server_mode, veilid_logs).await;

32
veilid-server/src/veilid_logs.rs
View File

@ -18,6 +18,7 @@ use std::path::*;
use std::sync::Arc; use std::sync::Arc;
use tracing_appender::*; use tracing_appender::*;
use tracing_flame::FlameLayer; use tracing_flame::FlameLayer;
use tracing_perfetto::PerfettoLayer;
use tracing_subscriber::prelude::*; use tracing_subscriber::prelude::*;
use tracing_subscriber::*; use tracing_subscriber::*;
@ -48,14 +49,15 @@ impl VeilidLogs {
#[cfg(feature = "rt-tokio")] #[cfg(feature = "rt-tokio")]
if settingsr.logging.console.enabled { if settingsr.logging.console.enabled {
let filter = veilid_core::VeilidLayerFilter::new_no_default(
veilid_core::VeilidConfigLogLevel::Trace,
&[],
);
let layer = ConsoleLayer::builder() let layer = ConsoleLayer::builder()
.with_default_env() .with_default_env()
.spawn() .spawn()
.with_filter( .with_filter(filter);
filter::Targets::new()
.with_target("tokio", Level::TRACE)
.with_target("runtime", Level::TRACE),
);
layers.push(layer.boxed()); layers.push(layer.boxed());
} }
@ -93,6 +95,26 @@ impl VeilidLogs {
); );
} }
// Perfetto logger
if settingsr.logging.perfetto.enabled {
let filter = veilid_core::VeilidLayerFilter::new_no_default(
veilid_core::VeilidConfigLogLevel::Trace,
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()),
);
let perfetto_layer = PerfettoLayer::new(std::sync::Mutex::new(std::fs::File::create(
&settingsr.logging.perfetto.path,
)?));
// Do not include this in change_log_level changes, so we keep trace level
// filters.insert("flame", filter.clone());
layers.push(
perfetto_layer
.with_debug_annotations(true)
.with_filter(filter)
.boxed(),
);
}
// OpenTelemetry logger // OpenTelemetry logger
#[cfg(feature = "opentelemetry-otlp")] #[cfg(feature = "opentelemetry-otlp")]
if settingsr.logging.otlp.enabled { if settingsr.logging.otlp.enabled {

5
veilid-tools/Cargo.toml
View File

@ -33,7 +33,8 @@ rt-wasm-bindgen = ["async_executors/bindgen", "async_executors/timer"]
veilid_tools_android_tests = ["dep:paranoid-android"] veilid_tools_android_tests = ["dep:paranoid-android"]
veilid_tools_ios_tests = ["dep:tracing", "dep:oslog", "dep:tracing-oslog"] veilid_tools_ios_tests = ["dep:tracing", "dep:oslog", "dep:tracing-oslog"]
tracing = ["dep:tracing", "dep:tracing-subscriber"] tracing = ["dep:tracing", "dep:tracing-subscriber", "tokio/tracing"]
debug-locks = []
[dependencies] [dependencies]
tracing = { version = "0.1.40", features = [ tracing = { version = "0.1.40", features = [
@ -52,6 +53,7 @@ futures-util = { version = "0.3.30", default-features = false, features = [
"alloc", "alloc",
] } ] }
parking_lot = "0.12.3" parking_lot = "0.12.3"
async-lock = "3.4.0"
once_cell = "1.19.0" once_cell = "1.19.0"
stop-token = { version = "0.7.0", default-features = false } stop-token = { version = "0.7.0", default-features = false }
rand = "0.8.5" rand = "0.8.5"
@ -87,7 +89,6 @@ wasm-bindgen-futures = "0.4.42"
async_executors = { version = "0.7.0", default-features = false } async_executors = { version = "0.7.0", default-features = false }
getrandom = { version = "0.2", features = ["js"] } getrandom = { version = "0.2", features = ["js"] }
async-lock = "2.8.0"
send_wrapper = { version = "0.6.0", features = ["futures"] } send_wrapper = { version = "0.6.0", features = ["futures"] }
# Dependencies for Linux or Android # Dependencies for Linux or Android

5
veilid-tools/src/deferred_stream_processor.rs
View File

@ -32,7 +32,10 @@ impl DeferredStreamProcessor {
self.opt_stopper = Some(stopper); self.opt_stopper = Some(stopper);
let (dsc_tx, dsc_rx) = flume::unbounded::<SendPinBoxFuture<()>>(); let (dsc_tx, dsc_rx) = flume::unbounded::<SendPinBoxFuture<()>>();
self.opt_deferred_stream_channel = Some(dsc_tx); self.opt_deferred_stream_channel = Some(dsc_tx);
self.opt_join_handle = Some(spawn(Self::processor(stop_token, dsc_rx))); self.opt_join_handle = Some(spawn(
"deferred stream processor",
Self::processor(stop_token, dsc_rx),
));
} }
/// Terminate the processor and ensure all streams are closed /// Terminate the processor and ensure all streams are closed

2
veilid-tools/src/eventual.rs
View File

@ -104,7 +104,7 @@ where
match out { match out {
None => task::Poll::<Self::Output>::Pending, None => task::Poll::<Self::Output>::Pending,
Some(wakers) => { Some(wakers) => {
// Wake all EventualResolvedFutures // Wake all other instance futures
for w in wakers { for w in wakers {
w.wake(); w.wake();
} }

2
veilid-tools/src/eventual_value.rs
View File

@ -81,7 +81,7 @@ impl<T: Unpin> Future for EventualValueFuture<T> {
match out { match out {
None => task::Poll::<Self::Output>::Pending, None => task::Poll::<Self::Output>::Pending,
Some(wakers) => { Some(wakers) => {
// Wake all EventualResolvedFutures // Wake all other instance futures
for w in wakers { for w in wakers {
w.wake(); w.wake();
} }

2
veilid-tools/src/eventual_value_clone.rs
View File

@ -77,7 +77,7 @@ impl<T: Unpin + Clone> Future for EventualValueCloneFuture<T> {
match out { match out {
None => task::Poll::<Self::Output>::Pending, None => task::Poll::<Self::Output>::Pending,
Some(wakers) => { Some(wakers) => {
// Wake all EventualResolvedFutures // Wake all other instance futures
for w in wakers { for w in wakers {
w.wake(); w.wake();
} }

8
veilid-tools/src/interval.rs
View File

@ -3,7 +3,7 @@ use super::*;
cfg_if! { cfg_if! {
if #[cfg(target_arch = "wasm32")] { if #[cfg(target_arch = "wasm32")] {
pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SendPinBoxFuture<()> pub fn interval<F, FUT>(name: &str, freq_ms: u32, callback: F) -> SendPinBoxFuture<()>
where where
F: Fn() -> FUT + Send + Sync + 'static, F: Fn() -> FUT + Send + Sync + 'static,
FUT: Future<Output = ()> + Send, FUT: Future<Output = ()> + Send,
@ -11,7 +11,7 @@ cfg_if! {
let e = Eventual::new(); let e = Eventual::new();
let ie = e.clone(); let ie = e.clone();
let jh = spawn(Box::pin(async move { let jh = spawn(name, Box::pin(async move {
while timeout(freq_ms, ie.instance_clone(())).await.is_err() { while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
callback().await; callback().await;
} }
@ -25,7 +25,7 @@ cfg_if! {
} else { } else {
pub fn interval<F, FUT>(freq_ms: u32, callback: F) -> SendPinBoxFuture<()> pub fn interval<F, FUT>(name: &str, freq_ms: u32, callback: F) -> SendPinBoxFuture<()>
where where
F: Fn() -> FUT + Send + Sync + 'static, F: Fn() -> FUT + Send + Sync + 'static,
FUT: Future<Output = ()> + Send, FUT: Future<Output = ()> + Send,
@ -33,7 +33,7 @@ cfg_if! {
let e = Eventual::new(); let e = Eventual::new();
let ie = e.clone(); let ie = e.clone();
let jh = spawn(async move { let jh = spawn(name, async move {
while timeout(freq_ms, ie.instance_clone(())).await.is_err() { while timeout(freq_ms, ie.instance_clone(())).await.is_err() {
callback().await; callback().await;
} }

33
veilid-tools/src/lib.rs
View File

@ -49,6 +49,7 @@ pub mod single_shot_eventual;
pub mod sleep; pub mod sleep;
pub mod spawn; pub mod spawn;
pub mod split_url; pub mod split_url;
pub mod startup_lock;
pub mod tick_task; pub mod tick_task;
pub mod timeout; pub mod timeout;
pub mod timeout_or; pub mod timeout_or;
@ -106,7 +107,7 @@ pub use std::str::FromStr;
#[doc(no_inline)] #[doc(no_inline)]
pub use std::string::{String, ToString}; pub use std::string::{String, ToString};
#[doc(no_inline)] #[doc(no_inline)]
pub use std::sync::atomic::{AtomicBool, Ordering}; pub use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
#[doc(no_inline)] #[doc(no_inline)]
pub use std::sync::{Arc, Weak}; pub use std::sync::{Arc, Weak};
#[doc(no_inline)] #[doc(no_inline)]
@ -116,6 +117,13 @@ pub use std::time::Duration;
#[doc(no_inline)] #[doc(no_inline)]
pub use std::vec::Vec; pub use std::vec::Vec;
#[doc(no_inline)]
pub use async_lock::RwLock as AsyncRwLock;
#[doc(no_inline)]
pub use async_lock::RwLockReadGuard as AsyncRwLockReadGuard;
#[doc(no_inline)]
pub use async_lock::RwLockWriteGuard as AsyncRwLockWriteGuard;
cfg_if! { cfg_if! {
if #[cfg(target_arch = "wasm32")] { if #[cfg(target_arch = "wasm32")] {
#[doc(no_inline)] #[doc(no_inline)]
@ -124,8 +132,10 @@ cfg_if! {
pub use async_lock::MutexGuard as AsyncMutexGuard; pub use async_lock::MutexGuard as AsyncMutexGuard;
#[doc(no_inline)] #[doc(no_inline)]
pub use async_lock::MutexGuardArc as AsyncMutexGuardArc; pub use async_lock::MutexGuardArc as AsyncMutexGuardArc;
#[doc(no_inline)] #[doc(no_inline)]
pub use async_executors::JoinHandle as LowLevelJoinHandle; pub use async_executors::JoinHandle as LowLevelJoinHandle;
} else { } else {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
@ -135,8 +145,17 @@ cfg_if! {
pub use async_std::sync::MutexGuard as AsyncMutexGuard; pub use async_std::sync::MutexGuard as AsyncMutexGuard;
#[doc(no_inline)] #[doc(no_inline)]
pub use async_std::sync::MutexGuardArc as AsyncMutexGuardArc; pub use async_std::sync::MutexGuardArc as AsyncMutexGuardArc;
// #[doc(no_inline)]
// pub use async_std::sync::RwLock as AsyncRwLock;
// #[doc(no_inline)]
// pub use async_std::sync::RwLockReadGuard as AsyncRwLockReadGuard;
// #[doc(no_inline)]
// pub use async_std::sync::RwLockWriteGuard as AsyncRwLockWriteGuard;
#[doc(no_inline)] #[doc(no_inline)]
pub use async_std::task::JoinHandle as LowLevelJoinHandle; pub use async_std::task::JoinHandle as LowLevelJoinHandle;
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
#[doc(no_inline)] #[doc(no_inline)]
pub use tokio::sync::Mutex as AsyncMutex; pub use tokio::sync::Mutex as AsyncMutex;
@ -144,6 +163,15 @@ cfg_if! {
pub use tokio::sync::MutexGuard as AsyncMutexGuard; pub use tokio::sync::MutexGuard as AsyncMutexGuard;
#[doc(no_inline)] #[doc(no_inline)]
pub use tokio::sync::OwnedMutexGuard as AsyncMutexGuardArc; pub use tokio::sync::OwnedMutexGuard as AsyncMutexGuardArc;
// #[doc(no_inline)]
// pub use tokio::sync::RwLock as AsyncRwLock;
// #[doc(no_inline)]
// pub use tokio::sync::RwLockReadGuard as AsyncRwLockReadGuard;
// #[doc(no_inline)]
// pub use tokio::sync::RwLockWriteGuard as AsyncRwLockWriteGuard;
#[doc(no_inline)] #[doc(no_inline)]
pub use tokio::task::JoinHandle as LowLevelJoinHandle; pub use tokio::task::JoinHandle as LowLevelJoinHandle;
} else { } else {
@ -202,6 +230,8 @@ pub use spawn::*;
#[doc(inline)] #[doc(inline)]
pub use split_url::*; pub use split_url::*;
#[doc(inline)] #[doc(inline)]
pub use startup_lock::*;
#[doc(inline)]
pub use tick_task::*; pub use tick_task::*;
#[doc(inline)] #[doc(inline)]
pub use timeout::*; pub use timeout::*;
@ -221,6 +251,7 @@ pub mod tests;
cfg_if! { cfg_if! {
if #[cfg(feature = "tracing")] { if #[cfg(feature = "tracing")] {
use tracing::*; use tracing::*;
#[macro_export] #[macro_export]
macro_rules! debug_target_enabled { macro_rules! debug_target_enabled {
($target:expr) => { enabled!(target: $target, Level::DEBUG) } ($target:expr) => { enabled!(target: $target, Level::DEBUG) }

8
veilid-tools/src/must_join_single_future.rs
View File

@ -64,6 +64,7 @@ where
} }
/// Check the result and take it if there is one /// Check the result and take it if there is one
#[cfg_attr(feature = "tracing", instrument(level = "trace", skip_all))]
pub async fn check(&self) -> Result<Option<T>, ()> { pub async fn check(&self) -> Result<Option<T>, ()> {
let mut out: Option<T> = None; let mut out: Option<T> = None;
@ -95,6 +96,7 @@ where
} }
/// Wait for the result and take it /// Wait for the result and take it
#[cfg_attr(feature = "tracing", instrument(level = "trace", skip_all))]
pub async fn join(&self) -> Result<Option<T>, ()> { pub async fn join(&self) -> Result<Option<T>, ()> {
let mut out: Option<T> = None; let mut out: Option<T> = None;
@ -124,6 +126,7 @@ where
// Possibly spawn the future possibly returning the value of the last execution // Possibly spawn the future possibly returning the value of the last execution
pub async fn single_spawn_local( pub async fn single_spawn_local(
&self, &self,
name: &str,
future: impl Future<Output = T> + 'static, future: impl Future<Output = T> + 'static,
) -> Result<(Option<T>, bool), ()> { ) -> Result<(Option<T>, bool), ()> {
let mut out: Option<T> = None; let mut out: Option<T> = None;
@ -152,7 +155,7 @@ where
// Run if we should do that // Run if we should do that
if run { if run {
self.unlock(Some(spawn_local(future))); self.unlock(Some(spawn_local(name, future)));
} }
// Return the prior result if we have one // Return the prior result if we have one
@ -166,6 +169,7 @@ where
{ {
pub async fn single_spawn( pub async fn single_spawn(
&self, &self,
name: &str,
future: impl Future<Output = T> + Send + 'static, future: impl Future<Output = T> + Send + 'static,
) -> Result<(Option<T>, bool), ()> { ) -> Result<(Option<T>, bool), ()> {
let mut out: Option<T> = None; let mut out: Option<T> = None;
@ -191,7 +195,7 @@ where
} }
// Run if we should do that // Run if we should do that
if run { if run {
self.unlock(Some(spawn(future))); self.unlock(Some(spawn(name, future)));
} }
// Return the prior result if we have one // Return the prior result if we have one
Ok((out, run)) Ok((out, run))

2
veilid-tools/src/network_interfaces/netlink.rs
View File

@ -317,7 +317,7 @@ impl PlatformSupportNetlink {
let (connection, handle, _) = new_connection_with_socket::<RTNetLinkSocket>()?; let (connection, handle, _) = new_connection_with_socket::<RTNetLinkSocket>()?;
// Spawn a connection handler // Spawn a connection handler
let connection_jh = spawn(connection); let connection_jh = spawn("rtnetlink connection", connection);
// Save the connection // Save the connection
self.connection_jh = Some(connection_jh); self.connection_jh = Some(connection_jh);

2
veilid-tools/src/network_result.rs
View File

@ -32,6 +32,7 @@ impl<T> IoNetworkResultExt<T> for io::Result<T> {
Err(e) => match e.kind() { Err(e) => match e.kind() {
io::ErrorKind::TimedOut => Ok(NetworkResult::Timeout), io::ErrorKind::TimedOut => Ok(NetworkResult::Timeout),
io::ErrorKind::UnexpectedEof io::ErrorKind::UnexpectedEof
| io::ErrorKind::BrokenPipe
| io::ErrorKind::ConnectionAborted | io::ErrorKind::ConnectionAborted
| io::ErrorKind::ConnectionRefused | io::ErrorKind::ConnectionRefused
| io::ErrorKind::ConnectionReset | io::ErrorKind::ConnectionReset
@ -51,6 +52,7 @@ impl<T> IoNetworkResultExt<T> for io::Result<T> {
match e.kind() { match e.kind() {
io::ErrorKind::TimedOut => Ok(NetworkResult::Timeout), io::ErrorKind::TimedOut => Ok(NetworkResult::Timeout),
io::ErrorKind::UnexpectedEof io::ErrorKind::UnexpectedEof
| io::ErrorKind::BrokenPipe
| io::ErrorKind::ConnectionAborted | io::ErrorKind::ConnectionAborted
| io::ErrorKind::ConnectionRefused | io::ErrorKind::ConnectionRefused
| io::ErrorKind::ConnectionReset => Ok(NetworkResult::NoConnection(e)), | io::ErrorKind::ConnectionReset => Ok(NetworkResult::NoConnection(e)),

38
veilid-tools/src/spawn.rs
View File

@ -4,7 +4,7 @@ cfg_if! {
if #[cfg(target_arch = "wasm32")] { if #[cfg(target_arch = "wasm32")] {
use async_executors::{Bindgen, LocalSpawnHandleExt, SpawnHandleExt}; use async_executors::{Bindgen, LocalSpawnHandleExt, SpawnHandleExt};
pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out> pub fn spawn<Out>(_name: &str, future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out>
where where
Out: Send + 'static, Out: Send + 'static,
{ {
@ -15,7 +15,7 @@ cfg_if! {
) )
} }
pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out> pub fn spawn_local<Out>(_name: &str, future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out>
where where
Out: 'static, Out: 'static,
{ {
@ -26,7 +26,7 @@ cfg_if! {
) )
} }
pub fn spawn_detached<Out>(future: impl Future<Output = Out> + Send + 'static) pub fn spawn_detached<Out>(_name: &str, future: impl Future<Output = Out> + Send + 'static)
where where
Out: Send + 'static, Out: Send + 'static,
{ {
@ -35,7 +35,7 @@ cfg_if! {
.expect("wasm-bindgen-futures spawn_handle_local should never error out") .expect("wasm-bindgen-futures spawn_handle_local should never error out")
.detach() .detach()
} }
pub fn spawn_detached_local<Out>(future: impl Future<Output = Out> + 'static) pub fn spawn_detached_local<Out>(_name: &str, future: impl Future<Output = Out> + 'static)
where where
Out: 'static, Out: 'static,
{ {
@ -47,60 +47,60 @@ cfg_if! {
} else { } else {
pub fn spawn<Out>(future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out> pub fn spawn<Out>(name: &str, future: impl Future<Output = Out> + Send + 'static) -> MustJoinHandle<Out>
where where
Out: Send + 'static, Out: Send + 'static,
{ {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
MustJoinHandle::new(async_std::task::spawn(future)) MustJoinHandle::new(async_std::task::Builder::new().name(name.to_string()).spawn(future).unwrap())
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
MustJoinHandle::new(tokio::task::spawn(future)) MustJoinHandle::new(tokio::task::Builder::new().name(name).spawn(future).unwrap())
} }
} }
} }
pub fn spawn_local<Out>(future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out> pub fn spawn_local<Out>(name: &str, future: impl Future<Output = Out> + 'static) -> MustJoinHandle<Out>
where where
Out: 'static, Out: 'static,
{ {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
MustJoinHandle::new(async_std::task::spawn_local(future)) MustJoinHandle::new(async_std::task::Builder::new().name(name.to_string()).local(future).unwrap())
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
MustJoinHandle::new(tokio::task::spawn_local(future)) MustJoinHandle::new(tokio::task::Builder::new().name(name).spawn_local(future).unwrap())
} }
} }
} }
pub fn spawn_detached<Out>(future: impl Future<Output = Out> + Send + 'static) pub fn spawn_detached<Out>(name: &str, future: impl Future<Output = Out> + Send + 'static)
where where
Out: Send + 'static, Out: Send + 'static,
{ {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
drop(async_std::task::spawn(future)); drop(async_std::task::Builder::new().name(name.to_string()).spawn(future).unwrap());
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
drop(tokio::task::spawn(future)); drop(tokio::task::Builder::new().name(name).spawn(future).unwrap());
} }
} }
} }
pub fn spawn_detached_local<Out>(future: impl Future<Output = Out> + 'static) pub fn spawn_detached_local<Out>(name: &str,future: impl Future<Output = Out> + 'static)
where where
Out: 'static, Out: 'static,
{ {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
drop(async_std::task::spawn_local(future)); drop(async_std::task::Builder::new().name(name.to_string()).local(future).unwrap());
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
drop(tokio::task::spawn_local(future)); drop(tokio::task::Builder::new().name(name).spawn_local(future).unwrap());
} }
} }
} }
#[allow(unused_variables)] #[allow(unused_variables)]
pub async fn blocking_wrapper<F, R>(blocking_task: F, err_result: R) -> R pub async fn blocking_wrapper<F, R>(name: &str, blocking_task: F, err_result: R) -> R
where where
F: FnOnce() -> R + Send + 'static, F: FnOnce() -> R + Send + 'static,
R: Send + 'static, R: Send + 'static,
@ -108,9 +108,11 @@ cfg_if! {
// run blocking stuff in blocking thread // run blocking stuff in blocking thread
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
let _name = name;
// async_std::task::Builder blocking doesn't work like spawn_blocking()
async_std::task::spawn_blocking(blocking_task).await async_std::task::spawn_blocking(blocking_task).await
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
tokio::task::spawn_blocking(blocking_task).await.unwrap_or(err_result) tokio::task::Builder::new().name(name).spawn_blocking(blocking_task).unwrap().await.unwrap_or(err_result)
} else { } else {
#[compile_error("must use an executor")] #[compile_error("must use an executor")]
} }

177
veilid-tools/src/startup_lock.rs Normal file
View File

@ -0,0 +1,177 @@
use super::*;
#[derive(ThisError, Debug, Copy, Clone, PartialEq, Eq)]
#[error("Already started")]
pub struct StartupLockAlreadyStartedError;
#[derive(ThisError, Debug, Copy, Clone, PartialEq, Eq)]
#[error("Already shut down")]
pub struct StartupLockAlreadyShutDownError;
#[derive(ThisError, Debug, Copy, Clone, PartialEq, Eq)]
#[error("Not started")]
pub struct StartupLockNotStartedError;
/// RAII-style lock for startup and shutdown operations
/// Must call 'success()' on this lock to report a successful startup or shutdown
/// Dropping this lock without calling 'success()' first indicates a failed
/// startup or shutdown operation
#[derive(Debug)]
pub struct StartupLockGuard<'a> {
guard: AsyncRwLockWriteGuard<'a, bool>,
success_value: bool,
}
impl<'a> StartupLockGuard<'a> {
/// Call this function at the end of a successful startup or shutdown
/// operation to switch the state of the StartupLock.
pub fn success(mut self) {
*self.guard = self.success_value;
}
}
/// RAII-style lock for entry operations on a started-up region of code.
#[derive(Debug)]
pub struct StartupLockEnterGuard<'a> {
_guard: AsyncRwLockReadGuard<'a, bool>,
#[cfg(feature = "debug-locks")]
id: usize,
#[cfg(feature = "debug-locks")]
active_guards: Arc<Mutex<HashMap<usize, backtrace::Backtrace>>>,
}
#[cfg(feature = "debug-locks")]
impl<'a> Drop for StartupLockEnterGuard<'a> {
fn drop(&mut self) {
self.active_guards.lock().remove(&self.id);
}
}
#[cfg(feature = "debug-locks")]
static GUARD_ID: AtomicUsize = AtomicUsize::new(0);
/// Synchronization mechanism that tracks the startup and shutdown of a region of code.
/// Guarantees that some code can only be started up once and shut down only if it is
/// already started.
/// Also tracks if the code is in-use and will wait for all 'entered' code to finish
/// before shutting down. Once a shutdown is requested, future calls to 'enter' will
/// fail, ensuring that nothing is 'entered' at the time of shutdown. This allows an
/// asynchronous shutdown to wait for operations to finish before proceeding.
#[derive(Debug)]
pub struct StartupLock {
startup_state: AsyncRwLock<bool>,
stop_source: Mutex<Option<StopSource>>,
#[cfg(feature = "debug-locks")]
active_guards: Arc<Mutex<HashMap<usize, backtrace::Backtrace>>>,
}
impl StartupLock {
pub fn new() -> Self {
Self {
startup_state: AsyncRwLock::new(false),
stop_source: Mutex::new(None),
#[cfg(feature = "debug-locks")]
active_guards: Arc::new(Mutex::new(HashMap::new())),
}
}
/// Start up if things are not already started up
/// One must call 'success()' on the returned startup lock guard if startup was successful
/// otherwise the startup lock will not shift to the 'started' state.
pub fn startup(&self) -> Result<StartupLockGuard, StartupLockAlreadyStartedError> {
let guard =
asyncrwlock_try_write!(self.startup_state).ok_or(StartupLockAlreadyStartedError)?;
if *guard {
return Err(StartupLockAlreadyStartedError);
}
*self.stop_source.lock() = Some(StopSource::new());
Ok(StartupLockGuard {
guard,
success_value: true,
})
}
/// Get a stop token for this lock
/// One can wait on this to timeout operations when a shutdown is requested
pub fn stop_token(&self) -> Option<StopToken> {
self.stop_source.lock().as_ref().map(|ss| ss.token())
}
/// Check if this StartupLock is currently in a started state
/// Returns false is the state is in transition
pub fn is_started(&self) -> bool {
let Some(guard) = asyncrwlock_try_read!(self.startup_state) else {
return false;
};
*guard
}
/// Check if this StartupLock is currently in a shut down state
/// Returns false is the state is in transition
pub fn is_shut_down(&self) -> bool {
let Some(guard) = asyncrwlock_try_read!(self.startup_state) else {
return false;
};
!*guard
}
/// Wait for all 'entered' operations to finish before shutting down
/// One must call 'success()' on the returned startup lock guard if shutdown was successful
/// otherwise the startup lock will not shift to the 'stopped' state.
pub async fn shutdown(&self) -> Result<StartupLockGuard, StartupLockAlreadyShutDownError> {
// Drop the stop source to ensure we can detect shutdown has been requested
*self.stop_source.lock() = None;
cfg_if! {
if #[cfg(feature = "debug-locks")] {
let guard = match timeout(30000, self.startup_state.write()).await {
Ok(v) => v,
Err(_) => {
eprintln!("active guards: {:#?}", self.active_guards.lock().values().collect::<Vec<_>>());
panic!("shutdown deadlock");
}
};
} else {
let guard = self.startup_state.write().await;
}
}
if !*guard {
return Err(StartupLockAlreadyShutDownError);
}
Ok(StartupLockGuard {
guard,
success_value: false,
})
}
/// Enter an operation in a started-up module.
/// If this module has not yet started up or is in the process of startup or shutdown
/// this will fail.
pub fn enter(&self) -> Result<StartupLockEnterGuard, StartupLockNotStartedError> {
let guard = asyncrwlock_try_read!(self.startup_state).ok_or(StartupLockNotStartedError)?;
if !*guard {
return Err(StartupLockNotStartedError);
}
let out = StartupLockEnterGuard {
_guard: guard,
#[cfg(feature = "debug-locks")]
id: GUARD_ID.fetch_add(1, Ordering::AcqRel),
#[cfg(feature = "debug-locks")]
active_guards: self.active_guards.clone(),
};
#[cfg(feature = "debug-locks")]
self.active_guards
.lock()
.insert(out.id, backtrace::Backtrace::new());
Ok(out)
}
}
impl Default for StartupLock {
fn default() -> Self {
Self::new()
}
}

1
veilid-tools/src/tests/common/mod.rs
View File

@ -1,5 +1,6 @@
pub mod test_async_tag_lock; pub mod test_async_tag_lock;
pub mod test_host_interface; pub mod test_host_interface;
pub mod test_startup_lock;
#[allow(dead_code)] #[allow(dead_code)]
pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [ pub static DEFAULT_LOG_IGNORE_LIST: [&str; 21] = [

12
veilid-tools/src/tests/common/test_async_tag_lock.rs
View File

@ -35,7 +35,7 @@ pub async fn test_simple_single_contention() {
let g1 = table.lock_tag(a1).await; let g1 = table.lock_tag(a1).await;
info!("locked"); info!("locked");
let t1 = spawn(async move { let t1 = spawn("t1", async move {
// move the guard into the task // move the guard into the task
let _g1_take = g1; let _g1_take = g1;
// hold the guard for a bit // hold the guard for a bit
@ -90,7 +90,7 @@ pub async fn test_simple_double_contention() {
let g2 = table.lock_tag(a2).await; let g2 = table.lock_tag(a2).await;
info!("locked"); info!("locked");
let t1 = spawn(async move { let t1 = spawn("t1", async move {
// move the guard into the tas // move the guard into the tas
let _g1_take = g1; let _g1_take = g1;
// hold the guard for a bit // hold the guard for a bit
@ -99,7 +99,7 @@ pub async fn test_simple_double_contention() {
// release the guard // release the guard
info!("released"); info!("released");
}); });
let t2 = spawn(async move { let t2 = spawn("t2", async move {
// move the guard into the task // move the guard into the task
let _g2_take = g2; let _g2_take = g2;
// hold the guard for a bit // hold the guard for a bit
@ -131,7 +131,7 @@ pub async fn test_parallel_single_contention() {
let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234); let a1 = SocketAddr::new("1.2.3.4".parse().unwrap(), 1234);
let table1 = table.clone(); let table1 = table.clone();
let t1 = spawn(async move { let t1 = spawn("t1", async move {
// lock the tag // lock the tag
let _g = table1.lock_tag(a1).await; let _g = table1.lock_tag(a1).await;
info!("locked t1"); info!("locked t1");
@ -143,7 +143,7 @@ pub async fn test_parallel_single_contention() {
}); });
let table2 = table.clone(); let table2 = table.clone();
let t2 = spawn(async move { let t2 = spawn("t2", async move {
// lock the tag // lock the tag
let _g = table2.lock_tag(a1).await; let _g = table2.lock_tag(a1).await;
info!("locked t2"); info!("locked t2");
@ -155,7 +155,7 @@ pub async fn test_parallel_single_contention() {
}); });
let table3 = table.clone(); let table3 = table.clone();
let t3 = spawn(async move { let t3 = spawn("t3", async move {
// lock the tag // lock the tag
let _g = table3.lock_tag(a1).await; let _g = table3.lock_tag(a1).await;
info!("locked t3"); info!("locked t3");

37
veilid-tools/src/tests/common/test_host_interface.rs
View File

@ -30,7 +30,7 @@ pub async fn test_eventual() {
let i4 = e1.instance_clone(4u32); let i4 = e1.instance_clone(4u32);
drop(i2); drop(i2);
let jh = spawn(async move { let jh = spawn("task", async move {
sleep(1000).await; sleep(1000).await;
e1.resolve(); e1.resolve();
}); });
@ -47,7 +47,7 @@ pub async fn test_eventual() {
let i3 = e1.instance_clone(3u32); let i3 = e1.instance_clone(3u32);
let i4 = e1.instance_clone(4u32); let i4 = e1.instance_clone(4u32);
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
let i5 = e1.instance_clone(5u32); let i5 = e1.instance_clone(5u32);
let i6 = e1.instance_clone(6u32); let i6 = e1.instance_clone(6u32);
assert_eq!(i1.await, 1u32); assert_eq!(i1.await, 1u32);
@ -67,7 +67,7 @@ pub async fn test_eventual() {
let i1 = e1.instance_clone(1u32); let i1 = e1.instance_clone(1u32);
let i2 = e1.instance_clone(2u32); let i2 = e1.instance_clone(2u32);
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
assert_eq!(i1.await, 1u32); assert_eq!(i1.await, 1u32);
assert_eq!(i2.await, 2u32); assert_eq!(i2.await, 2u32);
}); });
@ -80,7 +80,7 @@ pub async fn test_eventual() {
// //
let j1 = e1.instance_clone(1u32); let j1 = e1.instance_clone(1u32);
let j2 = e1.instance_clone(2u32); let j2 = e1.instance_clone(2u32);
let jh = spawn(async move { let jh = spawn("task", async move {
assert_eq!(j1.await, 1u32); assert_eq!(j1.await, 1u32);
assert_eq!(j2.await, 2u32); assert_eq!(j2.await, 2u32);
}); });
@ -105,7 +105,7 @@ pub async fn test_eventual_value() {
drop(i2); drop(i2);
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
sleep(1000).await; sleep(1000).await;
e1_c1.resolve(3u32); e1_c1.resolve(3u32);
}); });
@ -122,7 +122,7 @@ pub async fn test_eventual_value() {
let i3 = e1.instance(); let i3 = e1.instance();
let i4 = e1.instance(); let i4 = e1.instance();
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
let i5 = e1.instance(); let i5 = e1.instance();
let i6 = e1.instance(); let i6 = e1.instance();
i1.await; i1.await;
@ -144,7 +144,7 @@ pub async fn test_eventual_value() {
let i1 = e1.instance(); let i1 = e1.instance();
let i2 = e1.instance(); let i2 = e1.instance();
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
i1.await; i1.await;
i2.await; i2.await;
}); });
@ -157,7 +157,7 @@ pub async fn test_eventual_value() {
// //
let j1 = e1.instance(); let j1 = e1.instance();
let j2 = e1.instance(); let j2 = e1.instance();
let jh = spawn(async move { let jh = spawn("task", async move {
j1.await; j1.await;
j2.await; j2.await;
}); });
@ -181,7 +181,7 @@ pub async fn test_eventual_value_clone() {
let i4 = e1.instance(); let i4 = e1.instance();
drop(i2); drop(i2);
let jh = spawn(async move { let jh = spawn("task", async move {
sleep(1000).await; sleep(1000).await;
e1.resolve(3u32); e1.resolve(3u32);
}); });
@ -199,7 +199,7 @@ pub async fn test_eventual_value_clone() {
let i3 = e1.instance(); let i3 = e1.instance();
let i4 = e1.instance(); let i4 = e1.instance();
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
let i5 = e1.instance(); let i5 = e1.instance();
let i6 = e1.instance(); let i6 = e1.instance();
assert_eq!(i1.await, 4); assert_eq!(i1.await, 4);
@ -220,7 +220,7 @@ pub async fn test_eventual_value_clone() {
let i1 = e1.instance(); let i1 = e1.instance();
let i2 = e1.instance(); let i2 = e1.instance();
let e1_c1 = e1.clone(); let e1_c1 = e1.clone();
let jh = spawn(async move { let jh = spawn("task", async move {
assert_eq!(i1.await, 5); assert_eq!(i1.await, 5);
assert_eq!(i2.await, 5); assert_eq!(i2.await, 5);
}); });
@ -231,7 +231,7 @@ pub async fn test_eventual_value_clone() {
// //
let j1 = e1.instance(); let j1 = e1.instance();
let j2 = e1.instance(); let j2 = e1.instance();
let jh = spawn(async move { let jh = spawn("task", async move {
assert_eq!(j1.await, 6); assert_eq!(j1.await, 6);
assert_eq!(j2.await, 6); assert_eq!(j2.await, 6);
}); });
@ -245,7 +245,7 @@ pub async fn test_interval() {
info!("testing interval"); info!("testing interval");
let tick: Arc<Mutex<u32>> = Arc::new(Mutex::new(0u32)); let tick: Arc<Mutex<u32>> = Arc::new(Mutex::new(0u32));
let stopper = interval(1000, move || { let stopper = interval("interval", 1000, move || {
let tick = tick.clone(); let tick = tick.clone();
async move { async move {
let mut tick = tick.lock(); let mut tick = tick.lock();
@ -493,7 +493,7 @@ pub async fn test_must_join_single_future() {
let sf = MustJoinSingleFuture::<u32>::new(); let sf = MustJoinSingleFuture::<u32>::new();
assert_eq!(sf.check().await, Ok(None)); assert_eq!(sf.check().await, Ok(None));
assert_eq!( assert_eq!(
sf.single_spawn(async { sf.single_spawn("t1", async {
sleep(2000).await; sleep(2000).await;
69 69
}) })
@ -501,10 +501,13 @@ pub async fn test_must_join_single_future() {
Ok((None, true)) Ok((None, true))
); );
assert_eq!(sf.check().await, Ok(None)); assert_eq!(sf.check().await, Ok(None));
assert_eq!(sf.single_spawn(async { panic!() }).await, Ok((None, false))); assert_eq!(
sf.single_spawn("t2", async { panic!() }).await,
Ok((None, false))
);
assert_eq!(sf.join().await, Ok(Some(69))); assert_eq!(sf.join().await, Ok(Some(69)));
assert_eq!( assert_eq!(
sf.single_spawn(async { sf.single_spawn("t3", async {
sleep(1000).await; sleep(1000).await;
37 37
}) })
@ -513,7 +516,7 @@ pub async fn test_must_join_single_future() {
); );
sleep(2000).await; sleep(2000).await;
assert_eq!( assert_eq!(
sf.single_spawn(async { sf.single_spawn("t4", async {
sleep(1000).await; sleep(1000).await;
27 27
}) })

179
veilid-tools/src/tests/common/test_startup_lock.rs Normal file
View File

@ -0,0 +1,179 @@
use crate::*;
pub async fn test_startup_shutdown() {
info!("test_startup_shutdown");
let lock = StartupLock::new();
// Normal case
{
let guard = lock.startup().expect("should startup");
guard.success();
}
assert!(lock.is_started());
assert!(!lock.is_shut_down());
{
let guard = lock.shutdown().await.expect("should shutdown");
guard.success();
}
assert!(!lock.is_started());
assert!(lock.is_shut_down());
// Startup fail case
{
lock.startup().expect("should startup");
// Don't call success()
}
assert!(!lock.is_started());
{
lock.shutdown().await.expect_err("should not shutdown");
}
assert!(!lock.is_started());
// Shutdown fail case
{
let guard = lock.startup().expect("should startup");
guard.success();
}
assert!(lock.is_started());
{
lock.shutdown().await.expect("should shutdown");
// Don't call success()
}
assert!(lock.is_started());
{
let guard = lock.shutdown().await.expect("should shutdown");
guard.success();
}
assert!(!lock.is_started());
}
pub async fn test_contention() {
info!("test_contention");
let lock = Arc::new(StartupLock::new());
let val = Arc::new(AtomicBool::new(false));
{
let guard = lock.startup().expect("should startup");
guard.success();
}
assert!(lock.is_started());
let lock2 = lock.clone();
let val2 = val.clone();
let jh = spawn("task", async move {
let _guard = lock2.enter().expect("should enter");
sleep(2000).await;
val2.store(true, Ordering::Release);
});
sleep(1000).await;
{
let guard = lock.shutdown().await.expect("should shutdown");
assert!(
val.load(Ordering::Acquire),
"should have waited for enter to exit"
);
guard.success();
}
assert!(!lock.is_started());
jh.await;
}
pub async fn test_bad_enter() {
info!("test_bad_enter");
let lock = Arc::new(StartupLock::new());
lock.enter()
.expect_err("should not enter when not started up");
{
let guard = lock.startup().expect("should startup");
guard.success();
}
assert!(lock.is_started());
assert!(!lock.is_shut_down());
let lock2 = lock.clone();
let jh = spawn("task", async move {
let guard = lock2.shutdown().await.expect("should shutdown");
sleep(2000).await;
guard.success();
});
sleep(1000).await;
assert!(!lock.is_started());
assert!(!lock.is_shut_down());
lock.enter()
.expect_err("should not enter when shutting down");
jh.await;
assert!(!lock.is_started());
assert!(lock.is_shut_down());
lock.enter().expect_err("should not enter when shut down");
}
pub async fn test_multiple_enter() {
info!("test_multiple_enter");
let lock = Arc::new(StartupLock::new());
let s1 = lock.startup().expect("should startup");
s1.success();
{
let _e1 = lock.enter().expect("should enter 1");
{
let _e2 = lock.enter().expect("should enter 2");
{
let _e3 = lock.enter().expect("should enter 3");
}
}
}
let e4 = lock.enter().expect("should enter 4");
let e5 = lock.enter().expect("should enter 5");
let e6 = lock.enter().expect("should enter 6");
//eprintln!("1");
let lock2 = lock.clone();
let jh = spawn("task", async move {
//eprintln!("2");
let guard = lock2.shutdown().await.expect("should shutdown");
//eprintln!("7");
sleep(2000).await;
//eprintln!("8");
guard.success();
});
sleep(1000).await;
//eprintln!("3");
assert!(!lock.is_started());
assert!(!lock.is_shut_down());
// Now drop the enter created before shutdown
drop(e4);
//eprintln!("4");
drop(e5);
//eprintln!("5");
drop(e6);
//eprintln!("6");
// This should finally exit
jh.await;
//eprintln!("9");
assert!(!lock.is_started());
assert!(lock.is_shut_down());
lock.enter().expect_err("should not enter");
}
pub async fn test_all() {
test_startup_shutdown().await;
test_contention().await;
test_bad_enter().await;
test_multiple_enter().await;
}

11
veilid-tools/src/tests/native/mod.rs
View File

@ -14,6 +14,8 @@ use super::*;
pub async fn run_all_tests() { pub async fn run_all_tests() {
info!("TEST: exec_test_host_interface"); info!("TEST: exec_test_host_interface");
test_host_interface::test_all().await; test_host_interface::test_all().await;
info!("TEST: exec_test_startup_lock");
test_startup_lock::test_all().await;
info!("TEST: exec_test_network_interfaces"); info!("TEST: exec_test_network_interfaces");
test_network_interfaces::test_all().await; test_network_interfaces::test_all().await;
info!("TEST: exec_test_async_peek_stream"); info!("TEST: exec_test_async_peek_stream");
@ -87,6 +89,15 @@ cfg_if! {
}); });
} }
#[test]
#[serial]
fn run_test_startup_lock() {
setup();
block_on(async {
test_startup_lock::test_all().await;
});
}
#[test] #[test]
#[serial] #[serial]
fn run_test_network_interfaces() { fn run_test_network_interfaces() {

31
veilid-tools/src/tick_task.rs
View File

@ -10,6 +10,7 @@ type TickTaskRoutine<E> =
/// If the prior tick is still running, it will allow it to finish, and do another tick when the timer comes around again. /// If the prior tick is still running, it will allow it to finish, and do another tick when the timer comes around again.
/// One should attempt to make tasks short-lived things that run in less than the tick period if you want things to happen with regular periodicity. /// One should attempt to make tasks short-lived things that run in less than the tick period if you want things to happen with regular periodicity.
pub struct TickTask<E: Send + 'static> { pub struct TickTask<E: Send + 'static> {
name: String,
last_timestamp_us: AtomicU64, last_timestamp_us: AtomicU64,
tick_period_us: u64, tick_period_us: u64,
routine: OnceCell<Box<TickTaskRoutine<E>>>, routine: OnceCell<Box<TickTaskRoutine<E>>>,
@ -19,8 +20,9 @@ pub struct TickTask<E: Send + 'static> {
} }
impl<E: Send + 'static> TickTask<E> { impl<E: Send + 'static> TickTask<E> {
pub fn new_us(tick_period_us: u64) -> Self { pub fn new_us(name: &str, tick_period_us: u64) -> Self {
Self { Self {
name: name.to_string(),
last_timestamp_us: AtomicU64::new(0), last_timestamp_us: AtomicU64::new(0),
tick_period_us, tick_period_us,
routine: OnceCell::new(), routine: OnceCell::new(),
@ -29,8 +31,9 @@ impl<E: Send + 'static> TickTask<E> {
running: Arc::new(AtomicBool::new(false)), running: Arc::new(AtomicBool::new(false)),
} }
} }
pub fn new_ms(tick_period_ms: u32) -> Self { pub fn new_ms(name: &str, tick_period_ms: u32) -> Self {
Self { Self {
name: name.to_string(),
last_timestamp_us: AtomicU64::new(0), last_timestamp_us: AtomicU64::new(0),
tick_period_us: (tick_period_ms as u64) * 1000u64, tick_period_us: (tick_period_ms as u64) * 1000u64,
routine: OnceCell::new(), routine: OnceCell::new(),
@ -39,8 +42,9 @@ impl<E: Send + 'static> TickTask<E> {
running: Arc::new(AtomicBool::new(false)), running: Arc::new(AtomicBool::new(false)),
} }
} }
pub fn new(tick_period_sec: u32) -> Self { pub fn new(name: &str, tick_period_sec: u32) -> Self {
Self { Self {
name: name.to_string(),
last_timestamp_us: AtomicU64::new(0), last_timestamp_us: AtomicU64::new(0),
tick_period_us: (tick_period_sec as u64) * 1000000u64, tick_period_us: (tick_period_sec as u64) * 1000000u64,
routine: OnceCell::new(), routine: OnceCell::new(),
@ -100,19 +104,26 @@ impl<E: Send + 'static> TickTask<E> {
return Ok(()); return Ok(());
} }
self.internal_tick(now, last_timestamp_us).await.map(drop) let itick = self.internal_tick(now, last_timestamp_us);
itick.await.map(drop)
} }
pub async fn try_tick_now(&self) -> Result<bool, E> { pub async fn try_tick_now(&self) -> Result<bool, E> {
let now = get_timestamp(); let now = get_timestamp();
let last_timestamp_us = self.last_timestamp_us.load(Ordering::Acquire); let last_timestamp_us = self.last_timestamp_us.load(Ordering::Acquire);
self.internal_tick(now, last_timestamp_us).await let itick = self.internal_tick(now, last_timestamp_us);
itick.await
} }
async fn internal_tick(&self, now: u64, last_timestamp_us: u64) -> Result<bool, E> { async fn internal_tick(&self, now: u64, last_timestamp_us: u64) -> Result<bool, E> {
// Lock the stop source, tells us if we have ever started this future // Lock the stop source, tells us if we have ever started this future
let opt_stop_source = &mut *self.stop_source.lock().await; let opt_stop_source_fut = self.stop_source.lock();
let opt_stop_source = &mut *opt_stop_source_fut.await;
if opt_stop_source.is_some() { if opt_stop_source.is_some() {
// See if the previous execution finished with an error // See if the previous execution finished with an error
match self.single_future.check().await { match self.single_future.check().await {
@ -141,13 +152,19 @@ impl<E: Send + 'static> TickTask<E> {
let stop_token = stop_source.token(); let stop_token = stop_source.token();
let running = self.running.clone(); let running = self.running.clone();
let routine = self.routine.get().unwrap()(stop_token, last_timestamp_us, now); let routine = self.routine.get().unwrap()(stop_token, last_timestamp_us, now);
let wrapped_routine = Box::pin(async move { let wrapped_routine = Box::pin(async move {
running.store(true, core::sync::atomic::Ordering::Release); running.store(true, core::sync::atomic::Ordering::Release);
let out = routine.await; let out = routine.await;
running.store(false, core::sync::atomic::Ordering::Release); running.store(false, core::sync::atomic::Ordering::Release);
out out
}); });
match self.single_future.single_spawn(wrapped_routine).await {
match self
.single_future
.single_spawn(&self.name, wrapped_routine)
.await
{
// We should have already consumed the result of the last run, or there was none // We should have already consumed the result of the last run, or there was none
// and we should definitely have run, because the prior 'check()' operation // and we should definitely have run, because the prior 'check()' operation
// should have ensured the singlefuture was ready to run // should have ensured the singlefuture was ready to run

10
veilid-tools/src/timeout.rs
View File

@ -8,7 +8,9 @@ cfg_if! {
where where
F: Future<Output = T>, F: Future<Output = T>,
{ {
match select(Box::pin(sleep(dur_ms)), Box::pin(f)).await { let tout = select(Box::pin(sleep(dur_ms)), Box::pin(f));
match tout.await {
Either::Left((_x, _b)) => Err(TimeoutError()), Either::Left((_x, _b)) => Err(TimeoutError()),
Either::Right((y, _a)) => Ok(y), Either::Right((y, _a)) => Ok(y),
} }
@ -22,11 +24,13 @@ cfg_if! {
{ {
cfg_if! { cfg_if! {
if #[cfg(feature="rt-async-std")] { if #[cfg(feature="rt-async-std")] {
async_std::future::timeout(Duration::from_millis(dur_ms as u64), f).await.map_err(|e| e.into()) let tout = async_std::future::timeout(Duration::from_millis(dur_ms as u64), f);
} else if #[cfg(feature="rt-tokio")] { } else if #[cfg(feature="rt-tokio")] {
tokio::time::timeout(Duration::from_millis(dur_ms as u64), f).await.map_err(|e| e.into()) let tout = tokio::time::timeout(Duration::from_millis(dur_ms as u64), f);
} }
} }
tout.await.map_err(|e| e.into())
} }
} }

50
veilid-tools/src/tools.rs
View File

@ -54,6 +54,20 @@ cfg_if::cfg_if! {
$x.clone().try_lock_owned().ok() $x.clone().try_lock_owned().ok()
}; };
} }
// #[macro_export]
// macro_rules! asyncrwlock_try_read {
// ($x:expr) => {
// $x.try_read().ok()
// };
// }
// #[macro_export]
// macro_rules! asyncrwlock_try_write {
// ($x:expr) => {
// $x.try_write().ok()
// };
// }
} else { } else {
#[macro_export] #[macro_export]
macro_rules! asyncmutex_try_lock { macro_rules! asyncmutex_try_lock {
@ -73,9 +87,23 @@ cfg_if::cfg_if! {
$x.try_lock_arc() $x.try_lock_arc()
}; };
} }
} }
} }
#[macro_export]
macro_rules! asyncrwlock_try_read {
($x:expr) => {
$x.try_read()
};
}
#[macro_export]
macro_rules! asyncrwlock_try_write {
($x:expr) => {
$x.try_write()
};
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn system_boxed<'a, Out>( pub fn system_boxed<'a, Out>(
@ -474,3 +502,25 @@ pub fn type_name_of_val<T: ?Sized>(_val: &T) -> &'static str {
pub fn map_to_string<X: ToString>(arg: X) -> String { pub fn map_to_string<X: ToString>(arg: X) -> String {
arg.to_string() arg.to_string()
} }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub struct DebugGuard {
name: &'static str,
counter: &'static AtomicUsize,
}
impl DebugGuard {
pub fn new(name: &'static str, counter: &'static AtomicUsize) -> Self {
let c = counter.fetch_add(1, Ordering::SeqCst);
eprintln!("{} entered: {}", name, c + 1);
Self { name, counter }
}
}
impl Drop for DebugGuard {
fn drop(&mut self) {
let c = self.counter.fetch_sub(1, Ordering::SeqCst);
eprintln!("{} exited: {}", self.name, c - 1);
}
}

7
veilid-tools/tests/web.rs
View File

@ -44,3 +44,10 @@ async fn run_test_async_tag_lock() {
test_async_tag_lock::test_all().await; test_async_tag_lock::test_all().await;
} }
#[wasm_bindgen_test]
async fn run_test_startup_lock() {
setup();
test_startup_lock::test_all().await;
}