Merge branch 'geolocation' into 'main'

IP geolocation

See merge request veilid/veilid!330

Cargo.lock

@@ -1254,9 +1254,9 @@ dependencies = [
 
 [[package]]
 name = "crypto-mac"
-version = "0.11.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e"
 dependencies = [
  "generic-array",
  "subtle",
@@ -1603,6 +1603,15 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "enum-as-inner"
 version = "0.6.0"
@@ -1886,6 +1895,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -2441,6 +2465,19 @@ dependencies = [
  "tokio-io-timeout",
 ]
 
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.60"
@@ -2579,6 +2616,15 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "ipnetwork"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
@@ -2885,6 +2931,18 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
+[[package]]
+name = "maxminddb"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6087e5d8ea14861bb7c7f573afbc7be3798d3ef0fae87ec4fd9a4de9a127c3c"
+dependencies = [
+ "ipnetwork",
+ "log",
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.4"
@@ -2977,6 +3035,23 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nb-connect"
 version = "1.2.0"
@@ -3426,6 +3501,50 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
+[[package]]
+name = "openssl"
+version = "0.10.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if 1.0.0",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.74",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.20.0"
@@ -4169,6 +4288,46 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
 
+[[package]]
+name = "reqwest"
+version = "0.11.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
+dependencies = [
+ "base64",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http 0.2.12",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls-pemfile",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "resolv-conf"
 version = "0.7.0"
@@ -4383,6 +4542,15 @@ dependencies = [
  "sdd",
 ]
 
+[[package]]
+name = "schannel"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "schemars"
 version = "0.8.21"
@@ -4623,6 +4791,18 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.9.34+deprecated"
@@ -4913,9 +5093,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "subtle"
-version = "2.4.1"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
@@ -4959,6 +5139,27 @@ dependencies = [
  "windows 0.52.0",
 ]
 
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.12.0"
@@ -5138,6 +5339,16 @@ dependencies = [
  "syn 2.0.74",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-stream"
 version = "0.1.15"
@@ -5719,7 +5930,7 @@ dependencies = [
  "cfg-if 1.0.0",
  "chrono",
  "clap 4.5.15",
- "config 0.13.4",
+ "config 0.14.0",
  "console",
  "crossbeam-channel",
  "cursive",
@@ -5736,7 +5947,7 @@ dependencies = [
  "log",
  "lru",
  "owning_ref",
- "parking_lot 0.11.2",
+ "parking_lot 0.12.3",
  "rustyline-async",
  "serde",
  "serde_derive",
@@ -5797,6 +6008,7 @@ dependencies = [
  "libc",
  "lock_api",
  "lz4_flex",
+ "maxminddb",
  "ndk",
  "ndk-glue",
  "nix 0.27.1",
@@ -5806,6 +6018,7 @@ dependencies = [
  "parking_lot 0.12.3",
  "paste",
  "range-set-blaze",
+ "reqwest",
  "rustls",
  "rustls-pemfile",
  "sanitize-filename",
@@ -6037,7 +6250,7 @@ dependencies = [
  "js-sys",
  "lazy_static",
  "parking_lot 0.12.3",
- "send_wrapper 0.4.0",
+ "send_wrapper 0.6.0",
  "serde",
  "serde-wasm-bindgen 0.6.5",
  "serde_bytes",

veilid-core/Cargo.toml

@@ -57,6 +57,9 @@ debug-locks = ["veilid-tools/debug-locks"]
 unstable-blockstore = []
 unstable-tunnels = []
 
+# GeoIP
+geolocation = ["maxminddb", "reqwest"]
+
 ### DEPENDENCIES
 
 [dependencies]
@@ -153,6 +156,7 @@ bugsalot = { package = "veilid-bugsalot", version = "0.2.0" }
 chrono = "0.4.38"
 libc = "0.2.155"
 nix = "0.27.1"
+maxminddb = { version = "0.24.0", optional = true }
 
 # System
 async-std = { version = "1.12.0", features = ["unstable"], optional = true }
@@ -278,6 +282,7 @@ glob = "0.3.1"
 filetime = "0.2.23"
 sha2 = "0.10.8"
 hex = "0.4.3"
+reqwest = { version = "0.11", features = ["blocking"], optional = true }
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ["-O", "--enable-mutable-globals"]

veilid-core/build.rs

@@ -190,6 +190,50 @@ fn fix_android_emulator() {
     }
 }
 
+#[cfg(feature = "geolocation")]
+fn download_file(url: &str, filename: impl AsRef<Path>) -> Result<(), Box<dyn std::error::Error>> {
+    let content = reqwest::blocking::get(url)?.bytes()?;
+    std::fs::write(filename, content)?;
+    Ok(())
+}
+
+#[cfg(feature = "geolocation")]
+fn download_geoip_database_files() -> Result<(), Box<dyn std::error::Error>> {
+    let manifest_dir = env::var("CARGO_MANIFEST_DIR")?;
+    let target_dir = std::path::PathBuf::from(manifest_dir).join("../target");
+
+    // Source: https://github.com/sapics/ip-location-db
+    let files = [
+        (
+            "https://cdn.jsdelivr.net/npm/@ip-location-db/asn-country-mmdb/asn-country-ipv4.mmdb",
+            Path::new(&target_dir).join("ipv4.mmdb"),
+        ),
+        (
+            "https://cdn.jsdelivr.net/npm/@ip-location-db/asn-country-mmdb/asn-country-ipv6.mmdb",
+            Path::new(&target_dir).join("ipv6.mmdb"),
+        ),
+    ];
+
+    for (url, filename) in files {
+        if !filename.exists() {
+            println!("Downloading {url}");
+            download_file(url, &filename)?;
+            continue;
+        }
+
+        let modified = std::fs::metadata(&filename)?.modified()?;
+        let now = std::time::SystemTime::now();
+        let time_diff = now.duration_since(modified)?;
+
+        if time_diff > std::time::Duration::from_secs(60 * 60 * 24) {
+            println!("Downloading {url}");
+            download_file(url, &filename)?;
+        }
+    }
+
+    Ok(())
+}
+
 fn main() {
     if std::env::var("DOCS_RS").is_ok()
         || std::env::var("CARGO_CFG_DOC").is_ok()
@@ -204,4 +248,7 @@ fn main() {
     }
 
     fix_android_emulator();
+
+    #[cfg(feature = "geolocation")]
+    download_geoip_database_files().expect("failed to download geoip database files");
 }

veilid-core/src/routing_table/geolocation.rs

@@ -0,0 +1,94 @@
+#![allow(unused)]
+
+use crate::CountryCode;
+use maxminddb::MaxMindDBError;
+use once_cell::sync::Lazy;
+use serde::Deserialize;
+use std::net::IpAddr;
+use tracing::error;
+
+const IPV4_MMDB: &[u8] =
+    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../target/ipv4.mmdb"));
+const IPV6_MMDB: &[u8] =
+    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../target/ipv6.mmdb"));
+
+static IPV4: Lazy<Option<maxminddb::Reader<&'static [u8]>>> =
+    Lazy::new(|| match maxminddb::Reader::from_source(IPV4_MMDB) {
+        Ok(reader) => Some(reader),
+        Err(err) => {
+            error!("Unable to open embedded IPv4 geolocation database: {}", err);
+            None
+        }
+    });
+
+static IPV6: Lazy<Option<maxminddb::Reader<&'static [u8]>>> =
+    Lazy::new(|| match maxminddb::Reader::from_source(IPV6_MMDB) {
+        Ok(reader) => Some(reader),
+        Err(err) => {
+            error!("Unable to open embedded IPv6 geolocation database: {}", err);
+            None
+        }
+    });
+
+#[derive(Deserialize)]
+struct Country {
+    country_code: CountryCode,
+}
+
+pub fn query_country_code(addr: IpAddr) -> Option<CountryCode> {
+    let db = match addr {
+        IpAddr::V4(_) => &*IPV4,
+        IpAddr::V6(_) => &*IPV6,
+    };
+
+    let Some(db) = db else {
+        return None;
+    };
+
+    let result: Country = match db.lookup(addr) {
+        Ok(result) => result,
+        Err(MaxMindDBError::AddressNotFoundError(_)) => return None,
+        Err(err) => {
+            // We only expect AddressNotFoundError as possible error,
+            // anything else means there's a problem
+            error!("Unable to query country code: {}", err);
+            return None;
+        }
+    };
+
+    Some(result.country_code)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::CountryCode;
+    use core::str::FromStr;
+
+    #[test]
+    fn test_query_country_code() {
+        let test_cases = [
+            ("1.2.3.4", "AU"),
+            ("18.103.1.1", "US"),
+            ("100.128.1.1", "US"),
+            ("198.3.123.4", "US"),
+            ("2001:2a0::1", "JP"),
+        ];
+
+        for (ip_str, expected_country) in test_cases {
+            let ip = ip_str.parse().unwrap();
+            let expected_country_code = CountryCode::from_str(expected_country).unwrap();
+
+            let country_code = super::query_country_code(ip).unwrap();
+            assert_eq!(
+                country_code, expected_country_code,
+                "Wrong country for {ip_str}",
+            );
+
+            eprintln!("{ip_str} -> {country_code}");
+        }
+
+        assert!(super::query_country_code("127.0.0.1".parse().unwrap()).is_none());
+        assert!(super::query_country_code("10.0.0.1".parse().unwrap()).is_none());
+        assert!(super::query_country_code("::1".parse().unwrap()).is_none());
+    }
+}

veilid-core/src/routing_table/mod.rs

@@ -2,6 +2,8 @@ mod bucket;
 mod bucket_entry;
 mod debug;
 mod find_peers;
+#[cfg(feature = "geolocation")]
+mod geolocation;
 mod node_ref;
 mod privacy;
 mod route_spec_store;

veilid-core/src/veilid_api/types/country_code.rs

@@ -0,0 +1,116 @@
+use super::*;
+use std::hash::{Hash, Hasher};
+
+/// Two-letter country code. Case-insensitive when comparing.
+#[derive(Copy, Default, Clone, Ord, Eq, Serialize, Deserialize, JsonSchema)]
+#[serde(try_from = "String")]
+#[serde(into = "String")]
+pub struct CountryCode(pub [u8; 2]);
+
+impl From<[u8; 2]> for CountryCode {
+    fn from(b: [u8; 2]) -> Self {
+        Self(b)
+    }
+}
+
+impl From<CountryCode> for String {
+    fn from(u: CountryCode) -> Self {
+        String::from_utf8_lossy(&u.0).to_string()
+    }
+}
+
+impl TryFrom<&[u8]> for CountryCode {
+    type Error = VeilidAPIError;
+    fn try_from(b: &[u8]) -> Result<Self, Self::Error> {
+        Ok(Self(b.try_into().map_err(VeilidAPIError::generic)?))
+    }
+}
+
+impl TryFrom<String> for CountryCode {
+    type Error = VeilidAPIError;
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        Self::from_str(s.as_str())
+    }
+}
+
+impl fmt::Display for CountryCode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        write!(f, "{}", String::from_utf8_lossy(&self.0))
+    }
+}
+
+impl fmt::Debug for CountryCode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        write!(f, "{}", String::from_utf8_lossy(&self.0))
+    }
+}
+
+impl FromStr for CountryCode {
+    type Err = VeilidAPIError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            s.as_bytes().try_into().map_err(VeilidAPIError::generic)?,
+        ))
+    }
+}
+
+impl Hash for CountryCode {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        let this = [
+            self.0[0].to_ascii_uppercase(),
+            self.0[1].to_ascii_uppercase(),
+        ];
+
+        state.write(&this[..]);
+    }
+}
+
+impl PartialEq for CountryCode {
+    fn eq(&self, other: &Self) -> bool {
+        self.0[0].to_ascii_uppercase() == other.0[0].to_ascii_uppercase()
+            && self.0[1].to_ascii_uppercase() == other.0[1].to_ascii_uppercase()
+    }
+}
+
+impl PartialOrd for CountryCode {
+    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
+        let this = [
+            self.0[0].to_ascii_uppercase(),
+            self.0[1].to_ascii_uppercase(),
+        ];
+        let other = [
+            other.0[0].to_ascii_uppercase(),
+            other.0[1].to_ascii_uppercase(),
+        ];
+
+        this.partial_cmp(&other)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::CountryCode;
+    use core::str::FromStr;
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_hash_country_code() {
+        let mut set = HashSet::new();
+
+        set.insert(CountryCode::from_str("aa").unwrap());
+
+        assert!(set.get(&CountryCode::from_str("AA").unwrap()).is_some());
+    }
+
+    #[test]
+    fn test_compare_country_code() {
+        assert_eq!(
+            CountryCode::from_str("aa").unwrap(),
+            CountryCode::from_str("AA").unwrap(),
+        );
+
+        assert!(CountryCode::from_str("aa").unwrap() < CountryCode::from_str("Ab").unwrap());
+
+        assert!(CountryCode::from_str("Bc").unwrap() > CountryCode::from_str("bb").unwrap());
+    }
+}

veilid-core/src/veilid_api/types/mod.rs

@@ -1,6 +1,8 @@
 #[macro_use]
 mod aligned_u64;
 mod app_message_call;
+#[cfg(feature = "geolocation")]
+mod country_code;
 mod dht;
 mod fourcc;
 mod safety;
@@ -16,6 +18,8 @@ use super::*;
 
 pub use aligned_u64::*;
 pub use app_message_call::*;
+#[cfg(feature = "geolocation")]
+pub use country_code::*;
 pub use dht::*;
 pub use fourcc::*;
 pub use safety::*;