flamegraph and instrumentation work

veilid-server/Cargo.toml

@@ -81,6 +81,7 @@ hostname = "^0"
 stop-token = { version = "^0", default-features = false }
 sysinfo = { version = "^0.30.6" }
 wg = { version = "^0.9.1", features = ["future"] }
+tracing-flame = "0.2.0"
 
 [target.'cfg(windows)'.dependencies]
 windows-service = "^0"

veilid-server/profile.json

@@ -0,0 +1 @@
+{"meta":{"categories":[{"name":"Other","color":"grey","subcategories":["Other"]},{"name":"User","color":"yellow","subcategories":["Other"]}],"debug":false,"extensions":{"baseURL":[],"id":[],"length":0,"name":[]},"interval":0.1,"preprocessedProfileVersion":46,"processType":0,"product":"../target/profiling/veilid-server","sampleUnits":{"eventDelay":"ms","threadCPUDelta":"µs","time":"ms"},"startTime":1720014457589.647,"symbolicated":false,"pausedRanges":[],"version":24,"usesOnlyOneStackType":true,"doesNotUseFrameImplementation":true,"sourceCodeIsNotOnSearchfox":true,"markerSchema":[]},"libs":[{"name":"dyld","path":"/usr/lib/dyld","debugName":"dyld","debugPath":"/usr/lib/dyld","breakpadId":"37BBC384075531C7A8080ED49E44DD8E0","codeId":"37BBC384075531C7A8080ED49E44DD8E","arch":"arm64e"},{"name":"veilid-server","path":"/Users/dildog/code/veilid/target/profiling/veilid-server","debugName":"veilid-server","debugPath":"/Users/dildog/code/veilid/target/profiling/veilid-server","breakpadId":"0C900E21ABF83DD5848F0728125792EA0","codeId":"0C900E21ABF83DD5848F0728125792EA","arch":"arm64"},{"name":"libsystem_malloc.dylib","path":"/usr/lib/system/libsystem_malloc.dylib","debugName":"libsystem_malloc.dylib","debugPath":"/usr/lib/system/libsystem_malloc.dylib","breakpadId":"C6337A382B5C380595E8CF1786E2F4E70","codeId":"C6337A382B5C380595E8CF1786E2F4E7","arch":"arm64e"},{"name":"libsystem_kernel.dylib","path":"/usr/lib/system/libsystem_kernel.dylib","debugName":"libsystem_kernel.dylib","debugPath":"/usr/lib/system/libsystem_kernel.dylib","breakpadId":"9B8B53F9E2B636DF98E928D8FCA732F20","codeId":"9B8B53F9E2B636DF98E928D8FCA732F2","arch":"arm64e"}],"threads":[{"frameTable":{"length":18,"address":[24799,9886639,9760907,9887711,9893535,9924447,9927607,1245483,1245699,1229056,1245627,145256,9893543,1397243,1399123,4045067,4040447,19732],"inlineDepth":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"category":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"subcategory":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"func":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17],"nativeSymbol":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"innerWindowID":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"implementation":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"line":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"column":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"optimizations":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]},"funcTable":{"length":18,"name":[1,3,4,5,6,7,8,9,10,11,12,14,15,16,17,18,19,21],"isJS":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"relevantForJS":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"resource":[0,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,3],"fileName":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"lineNumber":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"columnNumber":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]},"markers":{"length":0,"category":[],"data":[],"endTime":[],"name":[],"phase":[],"startTime":[]},"name":"veilid-server","isMainThread":true,"nativeSymbols":{"length":0,"address":[],"functionSize":[],"libIndex":[],"name":[]},"pausedRanges":[],"pid":"12087","processName":"veilid-server","processShutdownTime":73.451958,"processStartupTime":70.423167,"processType":"default","registerTime":70.423167,"resourceTable":{"length":4,"lib":[0,1,2,3],"name":[0,2,13,20],"host":[null,null,null,null],"type":[1,1,1,1]},"samples":{"length":4,"stack":[9,11,17,17],"time":[72.541208,72.654958,72.786542,72.872625],"weight":[1,1,1,1],"weightType":"samples","threadCPUDelta":[6619,7,56,11]},"stackTable":{"length":18,"prefix":[null,0,1,2,3,4,5,6,7,8,7,10,3,12,13,14,15,16],"frame":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17],"category":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"subcategory":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"stringArray":["dyld","0x60df","veilid-server","0x96dbaf","0x94f08b","0x96dfdf","0x96f69f","0x976f5f","0x977bb7","0x13012b","0x130203","0x12c100","0x1301bb","libsystem_malloc.dylib","0x23768","0x96f6a7","0x1551fb","0x155953","0x3db90b","0x3da6ff","libsystem_kernel.dylib","0x4d14"],"tid":"1220572","unregisterTime":73.451958}],"pages":[],"profilerOverhead":[],"counters":[]}

veilid-server/src/main.rs

@@ -84,6 +84,10 @@ pub struct CmdlineArgs {
     #[arg(long, value_name = "endpoint")]
     otlp: Option<String>,
 
+    /// Turn on flamegraph tracing (experimental, isn't terribly useful)
+    #[arg(long, hide = true, value_name = "PATH", num_args=0..=1, require_equals=true, default_missing_value = "")]
+    flame: Option<OsString>,
+
     /// Run as an extra daemon on the same machine for testing purposes, specify a number greater than zero to offset the listening ports
     #[arg(long)]
     subnode_index: Option<u16>,
@@ -206,6 +210,19 @@ fn main() -> EyreResult<()> {
         .wrap_err("failed to parse OTLP address")?;
         settingsrw.logging.otlp.level = LogLevel::Trace;
     }
+    if let Some(flame) = args.flame {
+        let flame = if flame.is_empty() {
+            Settings::get_default_flame_path(settingsrw.testing.subnode_index)
+                .to_string_lossy()
+                .to_string()
+        } else {
+            flame.to_string_lossy().to_string()
+        };
+        println!("Enabling flamegraph output to {}", flame);
+        settingsrw.logging.flame.enabled = true;
+        settingsrw.logging.flame.path = flame;
+    }
+
     if args.no_attach {
         settingsrw.auto_attach = false;
     }

veilid-server/src/settings.rs

@@ -66,6 +66,9 @@ logging:
         level: 'trace'
         grpc_endpoint: 'localhost:4317'
         ignore_log_targets: []
+    flame:
+        enabled: false
+        path: ''
     console:
         enabled: false
 testing:
@@ -442,6 +445,12 @@ pub struct Terminal {
     pub ignore_log_targets: Vec<String>,
 }
 
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Flame {
+    pub enabled: bool,
+    pub path: String,
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct Console {
     pub enabled: bool,
@@ -493,6 +502,7 @@ pub struct Logging {
     pub file: File,
     pub api: Api,
     pub otlp: Otlp,
+    pub flame: Flame,
     pub console: Console,
 }
 
@@ -854,6 +864,15 @@ impl Settings {
             .unwrap_or_else(|| PathBuf::from("./veilid-server.conf"))
     }
 
+    /// Determine default flamegraph output path
+    pub fn get_default_flame_path(subnode_index: u16) -> PathBuf {
+        std::env::temp_dir().join(if subnode_index == 0 {
+            "veilid-server.folded".to_owned()
+        } else {
+            format!("veilid-server-{}.folded", subnode_index)
+        })
+    }
+
     #[allow(dead_code)]
     fn get_or_create_private_directory<P: AsRef<Path>>(path: P, group_read: bool) -> bool {
         let path = path.as_ref();
@@ -975,6 +994,8 @@ impl Settings {
         set_config_value!(inner.logging.otlp.level, value);
         set_config_value!(inner.logging.otlp.grpc_endpoint, value);
         set_config_value!(inner.logging.otlp.ignore_log_targets, value);
+        set_config_value!(inner.logging.flame.enabled, value);
+        set_config_value!(inner.logging.flame.path, value);
         set_config_value!(inner.logging.console.enabled, value);
         set_config_value!(inner.testing.subnode_index, value);
         set_config_value!(inner.core.capabilities.disable, value);
@@ -1542,6 +1563,8 @@ mod tests {
             s.logging.otlp.grpc_endpoint,
             NamedSocketAddrs::from_str("localhost:4317").unwrap()
         );
+        assert!(!s.logging.flame.enabled);
+        assert_eq!(s.logging.flame.path, "");
         assert!(!s.logging.console.enabled);
         assert_eq!(s.testing.subnode_index, 0);
 

veilid-server/src/veilid_logs.rs

@@ -17,11 +17,13 @@ use std::collections::BTreeMap;
 use std::path::*;
 use std::sync::Arc;
 use tracing_appender::*;
+use tracing_flame::FlameLayer;
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::*;
 
 struct VeilidLogsInner {
-    _guard: Option<non_blocking::WorkerGuard>,
+    _file_guard: Option<non_blocking::WorkerGuard>,
+    _flame_guard: Option<tracing_flame::FlushGuard<std::io::BufWriter<std::fs::File>>>,
     filters: BTreeMap<&'static str, veilid_core::VeilidLayerFilter>,
 }
 
@@ -71,6 +73,25 @@ impl VeilidLogs {
             layers.push(layer.boxed());
         }
 
+        // Flamegraph logger
+        let mut flame_guard = None;
+        if settingsr.logging.flame.enabled {
+            let filter = veilid_core::VeilidLayerFilter::new(
+                convert_loglevel(LogLevel::Trace),
+                &[], //&settingsr.logging.terminal.ignore_log_targets,
+            );
+            let (flame_layer, guard) = FlameLayer::with_file(&settingsr.logging.flame.path)?;
+            flame_guard = Some(guard);
+            filters.insert("flame", filter.clone());
+            layers.push(
+                flame_layer
+                    .with_threads_collapsed(true)
+                    .with_empty_samples(false)
+                    .with_filter(filter)
+                    .boxed(),
+            );
+        }
+
         // OpenTelemetry logger
         #[cfg(feature = "opentelemetry-otlp")]
         if settingsr.logging.otlp.enabled {
@@ -121,7 +142,7 @@ impl VeilidLogs {
         }
 
         // File logger
-        let mut guard = None;
+        let mut file_guard = None;
         if settingsr.logging.file.enabled {
             let log_path = Path::new(&settingsr.logging.file.path);
             let full_path = std::env::current_dir()
@@ -143,7 +164,7 @@ impl VeilidLogs {
             let appender = tracing_appender::rolling::never(log_parent, Path::new(log_filename));
             let (non_blocking_appender, non_blocking_guard) =
                 tracing_appender::non_blocking(appender);
-            guard = Some(non_blocking_guard);
+            file_guard = Some(non_blocking_guard);
 
             let filter = veilid_core::VeilidLayerFilter::new(
                 convert_loglevel(settingsr.logging.file.level),
@@ -192,7 +213,8 @@ impl VeilidLogs {
 
         Ok(VeilidLogs {
             inner: Arc::new(Mutex::new(VeilidLogsInner {
-                _guard: guard,
+                _file_guard: file_guard,
+                _flame_guard: flame_guard,
                 filters,
             })),
         })