From 6228c1df241bb2dc30d1ad6c27751258df330bbf Mon Sep 17 00:00:00 2001 From: neequ57 Date: Tue, 12 Nov 2024 17:11:50 +0000 Subject: [PATCH] IP geolocation, extend BucketEntry --- veilid-core/src/logging/facilities.rs | 12 ++- .../src/logging/veilid_layer_filter.rs | 10 +- veilid-core/src/routing_table/bucket_entry.rs | 24 +++++ veilid-core/src/routing_table/debug.rs | 101 +++++++++++++----- veilid-core/src/routing_table/geolocation.rs | 29 ++++- veilid-core/src/routing_table/mod.rs | 9 +- .../routing_table/types/geolocation_info.rs | 31 ++++++ veilid-core/src/routing_table/types/mod.rs | 4 + .../routing_table/types/signed_node_info.rs | 48 +++++++++ veilid-flutter/rust/src/dart_ffi.rs | 5 +- veilid-server/src/veilid_logs.rs | 10 +- 11 files changed, 240 insertions(+), 43 deletions(-) create mode 100644 veilid-core/src/routing_table/types/geolocation_info.rs diff --git a/veilid-core/src/logging/facilities.rs b/veilid-core/src/logging/facilities.rs index bf984edf..5adc608f 100644 --- a/veilid-core/src/logging/facilities.rs +++ b/veilid-core/src/logging/facilities.rs @@ -1,4 +1,4 @@ -pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [ +pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: &[&str] = &[ "mio", "h2", "hyper", @@ -28,9 +28,11 @@ pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [ "fanout", "receipt", "rpc_message", + #[cfg(feature = "geolocation")] + "maxminddb", ]; -pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [ +pub static FLAME_LOG_FACILITIES_IGNORE_LIST: &[&str] = &[ "mio", "h2", "hyper", @@ -53,9 +55,11 @@ pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [ "hickory_proto", "attohttpc", "ws_stream_wasm", + #[cfg(feature = "geolocation")] + "maxminddb", ]; -pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: [&str; 8] = [ +pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: &[&str] = &[ "net", "rpc", "rtab", @@ -66,7 +70,7 @@ pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: [&str; 8] = [ "crypto", ]; -pub static DURATION_LOG_FACILITIES: [&str; 1] = ["veilid_api"]; +pub static DURATION_LOG_FACILITIES: &[&str] = &["veilid_api"]; #[macro_export] macro_rules! fn_string { diff --git a/veilid-core/src/logging/veilid_layer_filter.rs b/veilid-core/src/logging/veilid_layer_filter.rs index eaaf98c0..84f135af 100644 --- a/veilid-core/src/logging/veilid_layer_filter.rs +++ b/veilid-core/src/logging/veilid_layer_filter.rs @@ -19,8 +19,9 @@ impl VeilidLayerFilter { ignore_change_list: &[String], ) -> VeilidLayerFilter { let mut ignore_list = DEFAULT_LOG_FACILITIES_IGNORE_LIST - .map(|x| x.to_owned()) - .to_vec(); + .iter() + .map(|&x| x.to_owned()) + .collect::>(); Self::apply_ignore_change_list(&mut ignore_list, ignore_change_list); Self { inner: Arc::new(RwLock::new(VeilidLayerFilterInner { @@ -64,8 +65,9 @@ impl VeilidLayerFilter { let mut inner = self.inner.write(); inner.ignore_list = ignore_list.unwrap_or_else(|| { DEFAULT_LOG_FACILITIES_IGNORE_LIST - .map(|x| x.to_owned()) - .to_vec() + .iter() + .map(|&x| x.to_owned()) + .collect::>() }); } callsite::rebuild_interest_cache(); diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 6b717e87..c9a3628b 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -174,6 +174,10 @@ pub(crate) struct BucketEntryInner { last_sender_info: HashMap, /// The node info for this entry on the publicinternet routing domain public_internet: BucketEntryPublicInternet, + /// Node location + #[cfg(feature = "geolocation")] + #[serde(skip)] + geolocation_info: GeolocationInfo, /// The node info for this entry on the localnetwork routing domain local_network: BucketEntryLocalNetwork, /// Statistics gathered for the peer @@ -461,6 +465,12 @@ impl BucketEntryInner { self.updated_since_last_network_change = true; self.make_not_dead(Timestamp::now()); + // Update geolocation info + #[cfg(feature = "geolocation")] + { + self.geolocation_info = signed_node_info.get_geolocation_info(routing_domain); + } + // If we're updating an entry's node info, purge all // but the last connection in our last connections list // because the dial info could have changed and it's safer to just reconnect. @@ -473,6 +483,13 @@ impl BucketEntryInner { node_info_changed } + #[cfg(feature = "geolocation")] + pub(super) fn update_geolocation_info(&mut self) { + if let Some(ref sni) = self.public_internet.signed_node_info { + self.geolocation_info = sni.get_geolocation_info(RoutingDomain::PublicInternet); + } + } + pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool { for routing_domain in routing_domain_set { // Get the correct signed_node_info for the chosen routing domain @@ -726,6 +743,11 @@ impl BucketEntryInner { self.state_reason(cur_ts).into() } + #[cfg(feature = "geolocation")] + pub fn geolocation_info(&self) -> &GeolocationInfo { + &self.geolocation_info + } + pub fn set_punished(&mut self, punished: Option) { self.punishment = punished; if punished.is_some() { @@ -1099,6 +1121,8 @@ impl BucketEntry { signed_node_info: None, node_status: None, }, + #[cfg(feature = "geolocation")] + geolocation_info: Default::default(), peer_stats: PeerStats { time_added: now, rpc_stats: RPCStats::default(), diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 2db4e8c6..bbc3758b 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -144,45 +144,88 @@ impl RoutingTable { e: &BucketEntryInner, relay_tag: &str, ) -> String { - format!( + let state_reason = Self::format_state_reason(e.state_reason(cur_ts)); + + let average_latency = e + .peer_stats() + .latency + .as_ref() + .map(|l| l.to_string()) + .unwrap_or_else(|| "???".to_string()); + + let capabilities = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) { + ni.capabilities() + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(",") + } else { + "???".to_owned() + }; + + let since_last_question = e + .peer_stats() + .rpc_stats + .last_question_ts + .as_ref() + .map(|l| cur_ts.saturating_sub(*l).to_string()) + .unwrap_or_else(|| "???".to_string()); + + let since_last_seen = e + .peer_stats() + .rpc_stats + .last_seen_ts + .as_ref() + .map(|l| cur_ts.saturating_sub(*l).to_string()) + .unwrap_or_else(|| "???".to_string()); + + #[allow(unused_mut)] + let mut result = format!( " {} [{}][{}] {} [{}] lastq@{} seen@{}", // node id node, // state reason - Self::format_state_reason(e.state_reason(cur_ts)), + state_reason, // Relay tag relay_tag, // average latency - e.peer_stats() - .latency - .as_ref() - .map(|l| l.to_string()) - .unwrap_or_else(|| "???".to_string()), + average_latency, // capabilities - if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) { - ni.capabilities() - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(",") - } else { - "???".to_owned() - }, + capabilities, // duration since last question - e.peer_stats() - .rpc_stats - .last_question_ts - .as_ref() - .map(|l| cur_ts.saturating_sub(*l).to_string()) - .unwrap_or_else(|| "???".to_string()), + since_last_question, // duration since last seen - e.peer_stats() - .rpc_stats - .last_seen_ts - .as_ref() - .map(|l| cur_ts.saturating_sub(*l).to_string()) - .unwrap_or_else(|| "???".to_string()), - ) + since_last_seen, + ); + + #[cfg(feature = "geolocation")] + { + let geolocation_info = e.geolocation_info(); + + if let Some(cc) = geolocation_info.country_code() { + result += &format!(" {cc}"); + } else { + result += " ??"; + } + + if !geolocation_info.relay_country_codes().is_empty() { + result += "/"; + } + + for (i, cc) in geolocation_info.relay_country_codes().iter().enumerate() { + if i > 0 { + result += ","; + } + + if let Some(cc) = cc { + result += &format!("{cc}"); + } else { + result += "??"; + } + } + } + + result } pub fn debug_info_entries( diff --git a/veilid-core/src/routing_table/geolocation.rs b/veilid-core/src/routing_table/geolocation.rs index d8fb0067..119d6292 100644 --- a/veilid-core/src/routing_table/geolocation.rs +++ b/veilid-core/src/routing_table/geolocation.rs @@ -1,5 +1,3 @@ -#![allow(unused)] - use crate::CountryCode; use maxminddb::MaxMindDBError; use once_cell::sync::Lazy; @@ -63,6 +61,7 @@ pub fn query_country_code(addr: IpAddr) -> Option { mod tests { use crate::CountryCode; use core::str::FromStr; + use maxminddb::WithinItem; #[test] fn test_query_country_code() { @@ -91,4 +90,30 @@ mod tests { assert!(super::query_country_code("10.0.0.1".parse().unwrap()).is_none()); assert!(super::query_country_code("::1".parse().unwrap()).is_none()); } + + #[test] + fn test_iter_over_ipv4_mmdb() { + let db = super::IPV4.as_ref().unwrap(); + + let count = db + .within("0.0.0.0/0".parse().unwrap()) + .unwrap() + .map(|item: Result, _>| item.unwrap()) + .count(); + + assert!(count > 100, "Expecting some IPv4 subnets in IPv4 MMDB"); + } + + #[test] + fn test_iter_over_ipv6_mmdb() { + let db = super::IPV6.as_ref().unwrap(); + + let count = db + .within("::/0".parse().unwrap()) + .unwrap() + .map(|item: Result, _>| item.unwrap()) + .count(); + + assert!(count > 100, "Expecting some IPv6 subnets in IPv6 MMDB"); + } } diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 5fe68495..e032fbd1 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -493,8 +493,15 @@ impl RoutingTable { ) -> EyreResult<()> { let mut all_entries: Vec> = Vec::with_capacity(all_entry_bytes.len()); for entry_bytes in all_entry_bytes { - let entryinner = deserialize_json_bytes(&entry_bytes) + #[allow(unused_mut)] + let mut entryinner: BucketEntryInner = deserialize_json_bytes(&entry_bytes) .wrap_err("failed to deserialize bucket entry")?; + + #[cfg(feature = "geolocation")] + { + entryinner.update_geolocation_info(); + } + let entry = Arc::new(BucketEntry::new_with_inner(entryinner)); // Keep strong reference in table diff --git a/veilid-core/src/routing_table/types/geolocation_info.rs b/veilid-core/src/routing_table/types/geolocation_info.rs new file mode 100644 index 00000000..a44624a0 --- /dev/null +++ b/veilid-core/src/routing_table/types/geolocation_info.rs @@ -0,0 +1,31 @@ +use super::*; + +#[derive(Debug, Default)] +pub struct GeolocationInfo { + country_code: Option, + relay_country_codes: Vec>, +} + +impl GeolocationInfo { + pub fn new( + country_code: Option, + relay_country_codes: Vec>, + ) -> Self { + GeolocationInfo { + country_code, + relay_country_codes, + } + } + + /// Get node country code. Might be `None` if unable to determine. + pub fn country_code(&self) -> Option { + self.country_code + } + + /// Get country codes of relays used by the node. + /// There will be exactly one entry for each relay. + /// Empty if no relays are used. + pub fn relay_country_codes(&self) -> &[Option] { + &self.relay_country_codes + } +} diff --git a/veilid-core/src/routing_table/types/mod.rs b/veilid-core/src/routing_table/types/mod.rs index c26a573b..d195273a 100644 --- a/veilid-core/src/routing_table/types/mod.rs +++ b/veilid-core/src/routing_table/types/mod.rs @@ -1,6 +1,8 @@ mod contact_method; mod dial_info_detail; mod direction; +#[cfg(feature = "geolocation")] +mod geolocation_info; mod node_info; mod node_status; mod peer_info; @@ -14,6 +16,8 @@ use super::*; pub use contact_method::*; pub use dial_info_detail::*; pub use direction::*; +#[cfg(feature = "geolocation")] +pub use geolocation_info::*; pub use node_info::*; pub use node_status::*; pub use peer_info::*; diff --git a/veilid-core/src/routing_table/types/signed_node_info.rs b/veilid-core/src/routing_table/types/signed_node_info.rs index d30b1fc7..e4adfc1e 100644 --- a/veilid-core/src/routing_table/types/signed_node_info.rs +++ b/veilid-core/src/routing_table/types/signed_node_info.rs @@ -115,6 +115,54 @@ impl SignedNodeInfo { .unwrap_or_default(); } + #[cfg(feature = "geolocation")] + /// Get geolocation info of node and its relays. + pub fn get_geolocation_info(&self, routing_domain: RoutingDomain) -> GeolocationInfo { + if routing_domain != RoutingDomain::PublicInternet { + // Country code is irrelevant for local network + return GeolocationInfo::new(None, vec![]); + } + + let get_node_country_code = |node_info: &NodeInfo| { + let country_codes = node_info + .dial_info_detail_list() + .iter() + .map(|did| match &did.dial_info { + DialInfo::UDP(di) => di.socket_address.ip_addr(), + DialInfo::TCP(di) => di.socket_address.ip_addr(), + DialInfo::WS(di) => di.socket_address.ip_addr(), + DialInfo::WSS(di) => di.socket_address.ip_addr(), + }) + .map(geolocation::query_country_code) + .collect::>(); + + if country_codes.is_empty() { + return None; + } + + // Indexing cannot panic, guarded by a check above + let cc0 = country_codes[0]; + + if !country_codes.iter().all(|cc| cc.is_some() && *cc == cc0) { + // Lookup failed for some address or results are different + return None; + } + + cc0 + }; + + match self { + SignedNodeInfo::Direct(sni) => { + GeolocationInfo::new(get_node_country_code(sni.node_info()), vec![]) + } + SignedNodeInfo::Relayed(sni) => { + let relay_cc = get_node_country_code(sni.relay_info().node_info()); + + GeolocationInfo::new(get_node_country_code(sni.node_info()), vec![relay_cc]) + } + } + } + /// Compare this SignedNodeInfo to another one /// Exclude the signature and timestamp and any other fields that are not /// semantically valuable diff --git a/veilid-flutter/rust/src/dart_ffi.rs b/veilid-flutter/rust/src/dart_ffi.rs index 1c96f928..80501daf 100644 --- a/veilid-flutter/rust/src/dart_ffi.rs +++ b/veilid-flutter/rust/src/dart_ffi.rs @@ -321,7 +321,10 @@ pub extern "C" fn initialize_veilid_core(platform_config: FfiStr) { if platform_config.logging.flame.enabled { let filter = veilid_core::VeilidLayerFilter::new_no_default( veilid_core::VeilidConfigLogLevel::Trace, - &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()), + &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST + .iter() + .map(|&x| x.to_string()) + .collect::>(), ); let (flame_layer, guard) = FlameLayer::with_file(&platform_config.logging.flame.path).unwrap(); diff --git a/veilid-server/src/veilid_logs.rs b/veilid-server/src/veilid_logs.rs index a0e912e5..c1a72ae9 100644 --- a/veilid-server/src/veilid_logs.rs +++ b/veilid-server/src/veilid_logs.rs @@ -95,7 +95,10 @@ impl VeilidLogs { if settingsr.logging.flame.enabled { let filter = veilid_core::VeilidLayerFilter::new_no_default( veilid_core::VeilidConfigLogLevel::Trace, - &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()), + &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST + .iter() + .map(|&x| x.to_string()) + .collect::>(), ); let (flame_layer, guard) = FlameLayer::with_file(&settingsr.logging.flame.path)?; flame_guard = Some(guard); @@ -115,7 +118,10 @@ impl VeilidLogs { if settingsr.logging.perfetto.enabled { let filter = veilid_core::VeilidLayerFilter::new_no_default( veilid_core::VeilidConfigLogLevel::Trace, - &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()), + &veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST + .iter() + .map(|&x| x.to_string()) + .collect::>(), ); let perfetto_layer = PerfettoLayer::new(std::sync::Mutex::new(std::fs::File::create( &settingsr.logging.perfetto.path,