update

scripts/jaeger.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# ensure adb is installed
+if command -v docker &> /dev/null; then 
+    echo '[X] docker is available in the path'
+else
+    echo 'docker is not available in the path'
+    exit 1
+fi
+
+# pull jaeger
+echo pulling Jaeger
+docker pull jaegertracing/all-in-one:1.35
+
+# run jaeger
+echo running Jaeger
+docker run -d --name jaeger \
+  -p 16686:16686 \
+  -p 4317:4317 \
+  jaegertracing/all-in-one:1.35 --collector.otlp.enabled=true $@
+
+read -p "Press [Enter] key to stop jaeger"
+
+docker stop jaeger
+docker rm jaeger

veilid-core/src/core_context.rs

@@ -10,6 +10,7 @@ cfg_if! {
     if #[cfg(target_arch = "wasm32")] {
         pub type UpdateCallback = Arc<dyn Fn(VeilidUpdate)>;
     } else {
+
         pub type UpdateCallback = Arc<dyn Fn(VeilidUpdate) + Send + Sync>;
     }
 }
@@ -195,7 +196,7 @@ impl VeilidCoreContext {
         Self::new_common(update_callback, config).await
     }
 
-    #[instrument(err, skip(update_callback))]
+    #[instrument(err, skip_all)]
     async fn new_with_config_json(
         update_callback: UpdateCallback,
         config_json: String,
@@ -209,7 +210,7 @@ impl VeilidCoreContext {
         Self::new_common(update_callback, config).await
     }
 
-    #[instrument(err, skip(update_callback))]
+    #[instrument(err, skip_all)]
     async fn new_common(
         update_callback: UpdateCallback,
         config: VeilidConfig,

veilid-core/src/dht/crypto.rs

@@ -2,7 +2,7 @@ use super::key::*;
 use crate::intf::*;
 use crate::xx::*;
 use crate::*;
-use chacha20::cipher::{NewCipher, StreamCipher};
+use chacha20::cipher::{KeyIvInit, StreamCipher};
 use chacha20::XChaCha20;
 use chacha20poly1305 as ch;
 use chacha20poly1305::aead::{AeadInPlace, NewAead};

veilid-core/src/network_manager/native/mod.rs

@@ -481,7 +481,7 @@ impl Network {
         self.inner.lock().network_started = true;
 
         // Inform routing table entries that our dial info has changed
-        self.routing_table().send_node_info_updates();
+        self.routing_table().send_node_info_updates().await;
 
         Ok(())
     }

veilid-core/src/network_manager/native/network_class_discovery.rs

@@ -618,7 +618,7 @@ impl Network {
             log_net!(debug "network class changed to {:?}", network_class);
 
             // Send updates to everyone
-            routing_table.send_node_info_updates();
+            routing_table.send_node_info_updates().await;
         }
 
         Ok(())

veilid-core/src/routing_table/mod.rs

@@ -399,23 +399,23 @@ impl RoutingTable {
     }
 
     // Inform routing table entries that our dial info has changed
-    pub fn send_node_info_updates(&self) {
+    pub async fn send_node_info_updates(&self) {
         let this = self.clone();
-        // Run in background
-        intf::spawn(async move {
         // Run in background only once
-            this.clone()
+        let _ = self
+            .clone()
             .unlocked_inner
             .node_info_update_single_future
             .single_spawn(async move {
-
                 // Only update if we actually have a valid network class
                 let netman = this.network_manager();
                 if matches!(
                     netman.get_network_class().unwrap_or(NetworkClass::Invalid),
                     NetworkClass::Invalid
                 ) {
-                        trace!("not sending node info update because our network class is not yet valid");
+                    trace!(
+                        "not sending node info update because our network class is not yet valid"
+                    );
                     return;
                 }
 
@@ -424,18 +424,18 @@ impl RoutingTable {
                     let mut inner = this.inner.lock();
                     let mut node_refs = Vec::<NodeRef>::with_capacity(inner.bucket_entry_count);
                     let cur_ts = intf::get_timestamp();
-                        Self::with_entries(&mut *inner, cur_ts, BucketEntryState::Unreliable, |k, e| {
+                    Self::with_entries(
+                        &mut *inner,
+                        cur_ts,
+                        BucketEntryState::Unreliable,
+                        |k, e| {
                             // Only update nodes that haven't seen our node info yet
                             if !e.has_seen_our_node_info() {
-                                node_refs.push(NodeRef::new(
+                                node_refs.push(NodeRef::new(this.clone(), *k, e, None));
-                                    this.clone(),
-                                    *k,
-                                    e,
-                                    None,
-                                ));
                             }
                             Option::<()>::None
-                        });
+                        },
+                    );
                     node_refs
                 };
 
@@ -465,9 +465,7 @@ impl RoutingTable {
 
                 log_rtab!("Finished sending node updates");
             })
-                .await
+            .await;
-        })
-        .detach()
     }
 
     // Attempt to empty the routing table
@@ -1134,7 +1132,10 @@ impl RoutingTable {
         let netman = self.network_manager();
         let relay_node_id = netman.relay_node().map(|nr| nr.node_id());
 
+        let mut unord = FuturesUnordered::new();
+        {
             let mut inner = self.inner.lock();
+
             Self::with_entries(&mut *inner, cur_ts, BucketEntryState::Unreliable, |k, e| {
                 if e.needs_ping(k, cur_ts, relay_node_id) {
                     let nr = NodeRef::new(self.clone(), *k, e, None);
@@ -1143,10 +1144,15 @@ impl RoutingTable {
                         nr,
                         e.state_debug_info(cur_ts)
                     );
-                intf::spawn_local(rpc.clone().rpc_call_status(nr)).detach();
+                    unord.push(intf::spawn_local(rpc.clone().rpc_call_status(nr)));
                 }
                 Option::<()>::None
             });
+        }
+
+        // Wait for futures to complete
+        while unord.next().await.is_some() {}
+
         Ok(())
     }
 

veilid-server/Cargo.toml

@@ -14,6 +14,9 @@ path = "src/main.rs"
 tracing = { version = "^0", features = ["log", "attributes"] }
 tracing-subscriber = { version = "^0", features = ["env-filter"] }
 tracing-appender = "^0"
+tracing-opentelemetry = "^0"
+opentelemetry = { version = "^0", features = ["rt-async-std"] }
+opentelemetry-otlp = { version = "^0", features = ["grpc-sys"] }
 clap = "^3"
 async-std = { version = "^1", features = ["unstable"] }
 async-tungstenite = { version = "^0", features = ["async-std-runtime", "async-tls"] }

veilid-server/src/cmdline.rs

@@ -61,6 +61,14 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result<clap::ArgMatches, clap
                 .conflicts_with("debug")
                 .help("Turn on trace logging on the terminal"),
         )
+        .arg(
+            Arg::new("otlp")
+                .long("otlp")
+                .takes_value(true)
+                .value_name("endpoint")
+                .default_missing_value("http://localhost:4317")
+                .help("Turn on OpenTelemetry tracing"),
+        )
         .arg(
             Arg::new("subnode-index")
                 .long("subnode-index")
@@ -197,6 +205,19 @@ pub fn process_command_line() -> Result<(Settings, ArgMatches), String> {
         settingsrw.logging.terminal.enabled = true;
         settingsrw.logging.terminal.level = LogLevel::Trace;
     }
+    if matches.occurrences_of("otlp") != 0 {
+        settingsrw.logging.otlp.enabled = true;
+        settingsrw.logging.otlp.grpc_endpoint = Some(
+            ParsedUrl::from_str(
+                &matches
+                    .value_of("otlp")
+                    .expect("should not be null because of default missing value")
+                    .to_string(),
+            )
+            .map_err(|e| format!("failed to parse OTLP url: {}", e))?,
+        );
+        settingsrw.logging.otlp.level = LogLevel::Trace;
+    }
     if matches.is_present("attach") {
         settingsrw.auto_attach = !matches!(matches.value_of("attach"), Some("true"));
     }

veilid-server/src/main.rs

@@ -58,10 +58,13 @@ fn main() -> Result<(), String> {
 
     // Handle non-normal server modes
     if !matches!(server_mode, ServerMode::Normal) {
+        // run the server to set the node id and quit
+        return task::block_on(async {
             // Init combined console/file logger
             let _logs = VeilidLogs::setup(settings.clone())?;
-        // run the server to set the node id and quit
+
-        return task::block_on(async { run_veilid_server(settings, server_mode).await })
+            run_veilid_server(settings, server_mode).await
+        })
         .map(|v| {
             println!("{}", success);
             v
@@ -83,9 +86,6 @@ fn main() -> Result<(), String> {
         }
     }
 
-    // Init combined console/file logger
-    let _logs = VeilidLogs::setup(settings.clone())?;
-
     // --- Normal Startup ---
     ctrlc::set_handler(move || {
         shutdown();
@@ -93,5 +93,10 @@ fn main() -> Result<(), String> {
     .expect("Error setting Ctrl-C handler");
 
     // Run the server loop
-    task::block_on(async { run_veilid_server(settings, server_mode).await })
+    task::block_on(async {
+        // Init combined console/file logger
+        let _logs = VeilidLogs::setup(settings.clone())?;
+
+        run_veilid_server(settings, server_mode).await
+    })
 }

veilid-server/src/settings.rs

@@ -36,6 +36,10 @@ logging:
     api:
         enabled: false
         level: 'info'
+    otlp:
+        enabled: false
+        level: 'trace'
+        grpc_endpoint: 'http://localhost:4317'
 testing:
     subnode_index: 0
 core:
@@ -427,6 +431,13 @@ pub struct Api {
     pub level: LogLevel,
 }
 
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Otlp {
+    pub enabled: bool,
+    pub level: LogLevel,
+    pub grpc_endpoint: Option<ParsedUrl>,
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct ClientApi {
     pub enabled: bool,
@@ -439,6 +450,7 @@ pub struct Logging {
     pub terminal: Terminal,
     pub file: File,
     pub api: Api,
+    pub otlp: Otlp,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -889,6 +901,9 @@ impl Settings {
         set_config_value!(inner.logging.file.level, value);
         set_config_value!(inner.logging.api.enabled, value);
         set_config_value!(inner.logging.api.level, value);
+        set_config_value!(inner.logging.otlp.enabled, value);
+        set_config_value!(inner.logging.otlp.level, value);
+        set_config_value!(inner.logging.otlp.grpc_endpoint, value);
         set_config_value!(inner.testing.subnode_index, value);
         set_config_value!(inner.core.protected_store.allow_insecure_fallback, value);
         set_config_value!(
@@ -1404,6 +1419,12 @@ mod tests {
         assert_eq!(s.logging.file.level, LogLevel::Info);
         assert_eq!(s.logging.api.enabled, false);
         assert_eq!(s.logging.api.level, LogLevel::Info);
+        assert_eq!(s.logging.otlp.enabled, false);
+        assert_eq!(s.logging.otlp.level, LogLevel::Trace);
+        assert_eq!(
+            s.logging.otlp.grpc_endpoint,
+            Some(ParsedUrl::from_str("http://127.0.0.1:4317").unwrap())
+        );
         assert_eq!(s.testing.subnode_index, 0);
 
         assert_eq!(

veilid-server/src/unix.rs

@@ -95,6 +95,8 @@ pub fn run_daemon(settings: Settings, _matches: ArgMatches) -> Result<(), String
         daemon
     };
 
+    // Now, run the server
+    task::block_on(async {
         // Init combined console/file logger
         let _logs = VeilidLogs::setup(settings.clone())?;
 
@@ -103,8 +105,6 @@ pub fn run_daemon(settings: Settings, _matches: ArgMatches) -> Result<(), String
             .start()
             .map_err(|e| format!("Failed to daemonize: {}", e))?;
 
-    // Now, run the server
-    task::block_on(async {
         // Catch signals
         let signals = Signals::new(&[SIGHUP, SIGTERM, SIGINT, SIGQUIT])
             .map_err(|e| format!("failed to init signals: {}", e))?;

veilid-server/src/veilid_logs.rs

@@ -1,5 +1,6 @@
 use crate::settings::*;
 use cfg_if::*;
+use opentelemetry_otlp::WithExportConfig;
 use std::path::*;
 use tracing::*;
 use tracing_appender::*;
@@ -34,6 +35,7 @@ impl VeilidLogs {
 
         let subscriber = Registry::default();
 
+        // Terminal logger
         let subscriber = subscriber.with(if settingsr.logging.terminal.enabled {
             let terminal_max_log_level: level_filters::LevelFilter =
                 convert_loglevel(settingsr.logging.terminal.level)
@@ -53,6 +55,46 @@ impl VeilidLogs {
             None
         });
 
+        // OpenTelemetry logger
+        let subscriber = subscriber.with(if settingsr.logging.otlp.enabled {
+            let otlp_max_log_level: level_filters::LevelFilter =
+                convert_loglevel(settingsr.logging.otlp.level)
+                    .to_tracing_level()
+                    .into();
+            let grpc_endpoint = match &settingsr.logging.otlp.grpc_endpoint {
+                Some(v) => &v.urlstring,
+                None => {
+                    return Err("missing OTLP GRPC endpoint url".to_owned());
+                }
+            };
+
+            // Required for GRPC dns resolution to work
+            std::env::set_var("GRPC_DNS_RESOLVER", "native");
+
+            let tracer = opentelemetry_otlp::new_pipeline()
+                .tracing()
+                .with_exporter(
+                    opentelemetry_otlp::new_exporter()
+                        .grpcio()
+                        .with_endpoint(grpc_endpoint),
+                )
+                .install_batch(opentelemetry::runtime::AsyncStd)
+                .map_err(|e| format!("failed to install OpenTelemetry tracer: {}", e))?;
+
+            let ignore_list = ignore_list.clone();
+            Some(
+                tracing_opentelemetry::layer()
+                    .with_tracer(tracer)
+                    .with_filter(otlp_max_log_level)
+                    .with_filter(filter::FilterFn::new(move |metadata| {
+                        logfilter(metadata, &ignore_list)
+                    })),
+            )
+        } else {
+            None
+        });
+
+        // File logger
         let mut guard = None;
         let subscriber = subscriber.with(if settingsr.logging.file.enabled {
             let file_max_log_level: level_filters::LevelFilter =
@@ -98,6 +140,7 @@ impl VeilidLogs {
             None
         });
 
+        // API logger
         let subscriber = subscriber.with(if settingsr.logging.api.enabled {
             // Get layer from veilid core, filtering is done by ApiTracingLayer automatically
             Some(veilid_core::ApiTracingLayer::get())
@@ -105,6 +148,7 @@ impl VeilidLogs {
             None
         });
 
+        // Systemd Journal logger
         cfg_if! {
             if #[cfg(target_os = "linux")] {
                 let subscriber = subscriber.with(if settingsr.logging.system.enabled {