From 9eaac095d37f59c32e3873145515cbb6153ee80e Mon Sep 17 00:00:00 2001 From: neequ57 Date: Sun, 3 Nov 2024 18:24:08 +0000 Subject: [PATCH] IP geolocation --- Cargo.lock | 227 +++++++++++++++++- veilid-core/Cargo.toml | 5 + veilid-core/build.rs | 47 ++++ veilid-core/src/routing_table/geolocation.rs | 94 ++++++++ veilid-core/src/routing_table/mod.rs | 2 + .../src/veilid_api/types/country_code.rs | 116 +++++++++ veilid-core/src/veilid_api/types/mod.rs | 4 + 7 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 veilid-core/src/routing_table/geolocation.rs create mode 100644 veilid-core/src/veilid_api/types/country_code.rs diff --git a/Cargo.lock b/Cargo.lock index a622b39e..f406beab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1254,9 +1254,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" dependencies = [ "generic-array", "subtle", @@ -1603,6 +1603,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "enum-as-inner" version = "0.6.0" @@ -1886,6 +1895,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2441,6 +2465,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2579,6 +2616,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2885,6 +2931,18 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maxminddb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6087e5d8ea14861bb7c7f573afbc7be3798d3ef0fae87ec4fd9a4de9a127c3c" +dependencies = [ + "ipnetwork", + "log", + "memchr", + "serde", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2977,6 +3035,23 @@ dependencies = [ "getrandom", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nb-connect" version = "1.2.0" @@ -3426,6 +3501,50 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.20.0" @@ -4169,6 +4288,46 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -4383,6 +4542,15 @@ dependencies = [ "sdd", ] +[[package]] +name = "schannel" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "schemars" version = "0.8.21" @@ -4623,6 +4791,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -4913,9 +5093,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -4959,6 +5139,27 @@ dependencies = [ "windows 0.52.0", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.12.0" @@ -5138,6 +5339,16 @@ dependencies = [ "syn 2.0.74", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -5719,7 +5930,7 @@ dependencies = [ "cfg-if 1.0.0", "chrono", "clap 4.5.15", - "config 0.13.4", + "config 0.14.0", "console", "crossbeam-channel", "cursive", @@ -5736,7 +5947,7 @@ dependencies = [ "log", "lru", "owning_ref", - "parking_lot 0.11.2", + "parking_lot 0.12.3", "rustyline-async", "serde", "serde_derive", @@ -5797,6 +6008,7 @@ dependencies = [ "libc", "lock_api", "lz4_flex", + "maxminddb", "ndk", "ndk-glue", "nix 0.27.1", @@ -5806,6 +6018,7 @@ dependencies = [ "parking_lot 0.12.3", "paste", "range-set-blaze", + "reqwest", "rustls", "rustls-pemfile", "sanitize-filename", @@ -6037,7 +6250,7 @@ dependencies = [ "js-sys", "lazy_static", "parking_lot 0.12.3", - "send_wrapper 0.4.0", + "send_wrapper 0.6.0", "serde", "serde-wasm-bindgen 0.6.5", "serde_bytes", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 32848ab1..5d17bff1 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -57,6 +57,9 @@ debug-locks = ["veilid-tools/debug-locks"] unstable-blockstore = [] unstable-tunnels = [] +# GeoIP +geolocation = ["maxminddb", "reqwest"] + ### DEPENDENCIES [dependencies] @@ -153,6 +156,7 @@ bugsalot = { package = "veilid-bugsalot", version = "0.2.0" } chrono = "0.4.38" libc = "0.2.155" nix = "0.27.1" +maxminddb = { version = "0.24.0", optional = true } # System async-std = { version = "1.12.0", features = ["unstable"], optional = true } @@ -278,6 +282,7 @@ glob = "0.3.1" filetime = "0.2.23" sha2 = "0.10.8" hex = "0.4.3" +reqwest = { version = "0.11", features = ["blocking"], optional = true } [package.metadata.wasm-pack.profile.release] wasm-opt = ["-O", "--enable-mutable-globals"] diff --git a/veilid-core/build.rs b/veilid-core/build.rs index b4cdfdde..c55a1aba 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -190,6 +190,50 @@ fn fix_android_emulator() { } } +#[cfg(feature = "geolocation")] +fn download_file(url: &str, filename: impl AsRef) -> Result<(), Box> { + let content = reqwest::blocking::get(url)?.bytes()?; + std::fs::write(filename, content)?; + Ok(()) +} + +#[cfg(feature = "geolocation")] +fn download_geoip_database_files() -> Result<(), Box> { + let manifest_dir = env::var("CARGO_MANIFEST_DIR")?; + let target_dir = std::path::PathBuf::from(manifest_dir).join("../target"); + + // Source: https://github.com/sapics/ip-location-db + let files = [ + ( + "https://cdn.jsdelivr.net/npm/@ip-location-db/asn-country-mmdb/asn-country-ipv4.mmdb", + Path::new(&target_dir).join("ipv4.mmdb"), + ), + ( + "https://cdn.jsdelivr.net/npm/@ip-location-db/asn-country-mmdb/asn-country-ipv6.mmdb", + Path::new(&target_dir).join("ipv6.mmdb"), + ), + ]; + + for (url, filename) in files { + if !filename.exists() { + println!("Downloading {url}"); + download_file(url, &filename)?; + continue; + } + + let modified = std::fs::metadata(&filename)?.modified()?; + let now = std::time::SystemTime::now(); + let time_diff = now.duration_since(modified)?; + + if time_diff > std::time::Duration::from_secs(60 * 60 * 24) { + println!("Downloading {url}"); + download_file(url, &filename)?; + } + } + + Ok(()) +} + fn main() { if std::env::var("DOCS_RS").is_ok() || std::env::var("CARGO_CFG_DOC").is_ok() @@ -204,4 +248,7 @@ fn main() { } fix_android_emulator(); + + #[cfg(feature = "geolocation")] + download_geoip_database_files().expect("failed to download geoip database files"); } diff --git a/veilid-core/src/routing_table/geolocation.rs b/veilid-core/src/routing_table/geolocation.rs new file mode 100644 index 00000000..d8fb0067 --- /dev/null +++ b/veilid-core/src/routing_table/geolocation.rs @@ -0,0 +1,94 @@ +#![allow(unused)] + +use crate::CountryCode; +use maxminddb::MaxMindDBError; +use once_cell::sync::Lazy; +use serde::Deserialize; +use std::net::IpAddr; +use tracing::error; + +const IPV4_MMDB: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../target/ipv4.mmdb")); +const IPV6_MMDB: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../target/ipv6.mmdb")); + +static IPV4: Lazy>> = + Lazy::new(|| match maxminddb::Reader::from_source(IPV4_MMDB) { + Ok(reader) => Some(reader), + Err(err) => { + error!("Unable to open embedded IPv4 geolocation database: {}", err); + None + } + }); + +static IPV6: Lazy>> = + Lazy::new(|| match maxminddb::Reader::from_source(IPV6_MMDB) { + Ok(reader) => Some(reader), + Err(err) => { + error!("Unable to open embedded IPv6 geolocation database: {}", err); + None + } + }); + +#[derive(Deserialize)] +struct Country { + country_code: CountryCode, +} + +pub fn query_country_code(addr: IpAddr) -> Option { + let db = match addr { + IpAddr::V4(_) => &*IPV4, + IpAddr::V6(_) => &*IPV6, + }; + + let Some(db) = db else { + return None; + }; + + let result: Country = match db.lookup(addr) { + Ok(result) => result, + Err(MaxMindDBError::AddressNotFoundError(_)) => return None, + Err(err) => { + // We only expect AddressNotFoundError as possible error, + // anything else means there's a problem + error!("Unable to query country code: {}", err); + return None; + } + }; + + Some(result.country_code) +} + +#[cfg(test)] +mod tests { + use crate::CountryCode; + use core::str::FromStr; + + #[test] + fn test_query_country_code() { + let test_cases = [ + ("1.2.3.4", "AU"), + ("18.103.1.1", "US"), + ("100.128.1.1", "US"), + ("198.3.123.4", "US"), + ("2001:2a0::1", "JP"), + ]; + + for (ip_str, expected_country) in test_cases { + let ip = ip_str.parse().unwrap(); + let expected_country_code = CountryCode::from_str(expected_country).unwrap(); + + let country_code = super::query_country_code(ip).unwrap(); + assert_eq!( + country_code, expected_country_code, + "Wrong country for {ip_str}", + ); + + eprintln!("{ip_str} -> {country_code}"); + } + + assert!(super::query_country_code("127.0.0.1".parse().unwrap()).is_none()); + assert!(super::query_country_code("10.0.0.1".parse().unwrap()).is_none()); + assert!(super::query_country_code("::1".parse().unwrap()).is_none()); + } +} diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 05d473df..5fe68495 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -2,6 +2,8 @@ mod bucket; mod bucket_entry; mod debug; mod find_peers; +#[cfg(feature = "geolocation")] +mod geolocation; mod node_ref; mod privacy; mod route_spec_store; diff --git a/veilid-core/src/veilid_api/types/country_code.rs b/veilid-core/src/veilid_api/types/country_code.rs new file mode 100644 index 00000000..db6f739b --- /dev/null +++ b/veilid-core/src/veilid_api/types/country_code.rs @@ -0,0 +1,116 @@ +use super::*; +use std::hash::{Hash, Hasher}; + +/// Two-letter country code. Case-insensitive when comparing. +#[derive(Copy, Default, Clone, Ord, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(try_from = "String")] +#[serde(into = "String")] +pub struct CountryCode(pub [u8; 2]); + +impl From<[u8; 2]> for CountryCode { + fn from(b: [u8; 2]) -> Self { + Self(b) + } +} + +impl From for String { + fn from(u: CountryCode) -> Self { + String::from_utf8_lossy(&u.0).to_string() + } +} + +impl TryFrom<&[u8]> for CountryCode { + type Error = VeilidAPIError; + fn try_from(b: &[u8]) -> Result { + Ok(Self(b.try_into().map_err(VeilidAPIError::generic)?)) + } +} + +impl TryFrom for CountryCode { + type Error = VeilidAPIError; + fn try_from(s: String) -> Result { + Self::from_str(s.as_str()) + } +} + +impl fmt::Display for CountryCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "{}", String::from_utf8_lossy(&self.0)) + } +} + +impl fmt::Debug for CountryCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "{}", String::from_utf8_lossy(&self.0)) + } +} + +impl FromStr for CountryCode { + type Err = VeilidAPIError; + fn from_str(s: &str) -> Result { + Ok(Self( + s.as_bytes().try_into().map_err(VeilidAPIError::generic)?, + )) + } +} + +impl Hash for CountryCode { + fn hash(&self, state: &mut H) { + let this = [ + self.0[0].to_ascii_uppercase(), + self.0[1].to_ascii_uppercase(), + ]; + + state.write(&this[..]); + } +} + +impl PartialEq for CountryCode { + fn eq(&self, other: &Self) -> bool { + self.0[0].to_ascii_uppercase() == other.0[0].to_ascii_uppercase() + && self.0[1].to_ascii_uppercase() == other.0[1].to_ascii_uppercase() + } +} + +impl PartialOrd for CountryCode { + fn partial_cmp(&self, other: &Self) -> Option { + let this = [ + self.0[0].to_ascii_uppercase(), + self.0[1].to_ascii_uppercase(), + ]; + let other = [ + other.0[0].to_ascii_uppercase(), + other.0[1].to_ascii_uppercase(), + ]; + + this.partial_cmp(&other) + } +} + +#[cfg(test)] +mod tests { + use crate::CountryCode; + use core::str::FromStr; + use std::collections::HashSet; + + #[test] + fn test_hash_country_code() { + let mut set = HashSet::new(); + + set.insert(CountryCode::from_str("aa").unwrap()); + + assert!(set.get(&CountryCode::from_str("AA").unwrap()).is_some()); + } + + #[test] + fn test_compare_country_code() { + assert_eq!( + CountryCode::from_str("aa").unwrap(), + CountryCode::from_str("AA").unwrap(), + ); + + assert!(CountryCode::from_str("aa").unwrap() < CountryCode::from_str("Ab").unwrap()); + + assert!(CountryCode::from_str("Bc").unwrap() > CountryCode::from_str("bb").unwrap()); + } +} diff --git a/veilid-core/src/veilid_api/types/mod.rs b/veilid-core/src/veilid_api/types/mod.rs index 4cecc66d..c20c9897 100644 --- a/veilid-core/src/veilid_api/types/mod.rs +++ b/veilid-core/src/veilid_api/types/mod.rs @@ -1,6 +1,8 @@ #[macro_use] mod aligned_u64; mod app_message_call; +#[cfg(feature = "geolocation")] +mod country_code; mod dht; mod fourcc; mod safety; @@ -16,6 +18,8 @@ use super::*; pub use aligned_u64::*; pub use app_message_call::*; +#[cfg(feature = "geolocation")] +pub use country_code::*; pub use dht::*; pub use fourcc::*; pub use safety::*;