Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-10-01 01:26:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint

Browse Source
This commit is contained in:
John Smith 2023-04-15 21:19:29 -04:00
parent 3b687aed50
commit 0769d032fc
5 changed files with 120 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Cargo.lock generated
View File

@ -2827,7 +2827,7 @@ dependencies = [
"parking_lot 0.12.1",
"rand 0.8.5",
"rusqlite",
"sysinfo",
"sysinfo 0.27.7",
"tempfile",
"tokio 1.24.2",
]
@ -5107,6 +5107,20 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "sysinfo"
version = "0.28.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys 0.8.3",
"libc",
"ntapi 0.4.0",
"once_cell",
"winapi 0.3.9",
]
[[package]]
name = "tap"
version = "1.0.1"
@ -6082,6 +6096,7 @@ dependencies = [
"signal-hook",
"signal-hook-async-std",
"stop-token",
"sysinfo 0.28.4",
"tokio 1.24.2",
"tokio-stream",
"tokio-util",

18
veilid-core/src/storage_manager/mod.rs
View File

@ -65,24 +65,30 @@ impl StorageManager {
}
fn local_limits_from_config(config: VeilidConfig) -> RecordStoreLimits {
let c = config.get();
RecordStoreLimits {
subkey_cache_size: todo!(),
subkey_cache_size: c.network.dht.local_subkey_cache_size as usize,
max_subkey_size: MAX_SUBKEY_SIZE,
max_record_total_size: MAX_RECORD_DATA_SIZE,
max_records: None,
max_subkey_cache_memory_mb: Some(xxx),
max_subkey_cache_memory_mb: Some(
c.network.dht.local_max_subkey_cache_memory_mb as usize,
),
max_storage_space_mb: None,
}
}
fn remote_limits_from_config(config: VeilidConfig) -> RecordStoreLimits {
let c = config.get();
RecordStoreLimits {
subkey_cache_size: todo!(),
subkey_cache_size: c.network.dht.remote_subkey_cache_size as usize,
max_subkey_size: MAX_SUBKEY_SIZE,
max_record_total_size: MAX_RECORD_DATA_SIZE,
max_records: Some(xxx),
max_subkey_cache_memory_mb: Some(xxx),
max_storage_space_mb: Some(xxx),
max_records: Some(c.network.dht.remote_max_records as usize),
max_subkey_cache_memory_mb: Some(
c.network.dht.remote_max_subkey_cache_memory_mb as usize,
),
max_storage_space_mb: Some(c.network.dht.remote_max_storage_space_mb as usize),
}
}

41
veilid-core/src/veilid_config.rs
View File

@ -277,42 +277,25 @@ pub struct VeilidConfigTLS {
RkyvDeserialize,
)]
pub struct VeilidConfigDHT {
pub resolve_node_timeout_ms: Option<u32>,
pub resolve_node_timeout_ms: u32,
pub resolve_node_count: u32,
pub resolve_node_fanout: u32,
pub max_find_node_count: u32,
pub get_value_timeout_ms: Option<u32>,
pub get_value_timeout_ms: u32,
pub get_value_count: u32,
pub get_value_fanout: u32,
pub set_value_timeout_ms: Option<u32>,
pub set_value_timeout_ms: u32,
pub set_value_count: u32,
pub set_value_fanout: u32,
pub min_peer_count: u32,
pub min_peer_refresh_time_ms: u32,
pub validate_dial_info_receipt_time_ms: u32,
pub local_subkey_cache_size: fn local_limits_from_config(config: VeilidConfig) -> RecordStoreLimits {
RecordStoreLimits {
subkey_cache_size: todo!(),
max_records: None,
max_subkey_cache_memory_mb: Some(xxx),
max_storage_space_mb: None,
max_subkey_size: MAX_SUBKEY_SIZE,
max_record_total_size: MAX_RECORD_DATA_SIZE,
}
}
fn remote_limits_from_config(config: VeilidConfig) -> RecordStoreLimits {
RecordStoreLimits {
subkey_cache_size: todo!(),
max_records: Some(xxx),
max_subkey_cache_memory_mb: Some(xxx),
max_storage_space_mb: Some(xxx),
max_subkey_size: MAX_SUBKEY_SIZE,
max_record_total_size: MAX_RECORD_DATA_SIZE,
}
}
pub local_subkey_cache_size: u32,
pub local_max_subkey_cache_memory_mb: u32,
pub remote_subkey_cache_size: u32,
pub remote_max_records: u32,
pub remote_max_subkey_cache_memory_mb: u32,
pub remote_max_storage_space_mb: u32,
}
/// Configure RPC
@ -685,6 +668,12 @@ impl VeilidConfig {
get_config!(inner.network.dht.min_peer_count);
get_config!(inner.network.dht.min_peer_refresh_time_ms);
get_config!(inner.network.dht.validate_dial_info_receipt_time_ms);
get_config!(inner.network.dht.local_subkey_cache_size);
get_config!(inner.network.dht.local_max_subkey_cache_memory_mb);
get_config!(inner.network.dht.remote_subkey_cache_size);
get_config!(inner.network.dht.remote_max_records);
get_config!(inner.network.dht.remote_max_subkey_cache_memory_mb);
get_config!(inner.network.dht.remote_max_storage_space_mb);
get_config!(inner.network.rpc.concurrency);
get_config!(inner.network.rpc.queue_size);
get_config!(inner.network.rpc.max_timestamp_behind_ms);

1
veilid-server/Cargo.toml
View File

@ -57,6 +57,7 @@ flume = { version = "^0", features = ["async"] }
rpassword = "^6"
hostname = "^0"
stop-token = { version = "^0", default-features = false }
sysinfo = { version = "^0.28.4", default-features = false }
[target.'cfg(windows)'.dependencies]
windows-service = "^0"

77
veilid-server/src/settings.rs
View File

@ -6,6 +6,7 @@ use serde_derive::*;
use std::ffi::OsStr;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use sysinfo::{DiskExt, System, SystemExt};
use url::Url;
use veilid_core::tools::*;
use veilid_core::*;
@ -95,6 +96,12 @@ core:
min_peer_count: 20
min_peer_refresh_time_ms: 2000
validate_dial_info_receipt_time_ms: 2000
local_subkey_cache_size: 128
local_max_subkey_cache_memory_mb: 256
remote_subkey_cache_size: 1024
remote_max_records: 65536
remote_max_subkey_cache_memory_mb: %REMOTE_MAX_SUBKEY_CACHE_MEMORY_MB%
remote_max_storage_space_mb: 0
upnp: true
detect_address_changes: true
restricted_nat_retries: 0
@ -164,7 +171,11 @@ core:
&Settings::get_default_private_key_directory()
.join("server.key")
.to_string_lossy(),
);
)
.replace(
"%REMOTE_MAX_SUBKEY_CACHE_MEMORY_MB%",
&Settings::get_default_remote_max_subkey_cache_memory_mb().to_string_lossy(),
)
config::Config::builder()
.add_source(config::File::from_str(
&default_config,
@ -512,6 +523,12 @@ pub struct Dht {
pub min_peer_count: u32,
pub min_peer_refresh_time_ms: u32,
pub validate_dial_info_receipt_time_ms: u32,
pub local_subkey_cache_size: u32,
pub local_max_subkey_cache_memory_mb: u32,
pub remote_subkey_cache_size: u32,
pub remote_max_records: u32,
pub remote_max_subkey_cache_memory_mb: u32,
pub remote_max_storage_space_mb: u32,
}
#[derive(Debug, Deserialize, Serialize)]
@ -623,6 +640,11 @@ impl Settings {
// Generate config
let inner: SettingsInner = cfg.try_deserialize()?;
// Fill in missing defaults
if inner.core.network.dht.remote_max_storage_space_mb == 0 {
inner.core.network.dht.remote_max_storage_space_mb = Self::get_default_remote_max_storage_space_mb(&inner);
}
//
Ok(Self {
inner: Arc::new(RwLock::new(inner)),
@ -833,6 +855,34 @@ impl Settings {
pk_path
}
pub fn get_default_remote_max_subkey_cache_memory_mb() -> usize {
let mut sys = System::new_with_specifics(sysinfo::RefreshKind::new().with_memory());
sys.free_memory() as usize / 8
}
pub fn get_default_remote_max_storage_space_mb(inner: &SettingsInner) -> usize {
let mut sys = System::new_with_specifics(sysinfo::RefreshKind::new().with_disks());
let dht_storage_path = inner.core.table_store.directory.clone();
let mut available_mb = 0usize;
// Sort longer mount point paths first since we want the mount point closest to our table store directory
sys.sort_disks_by(|a,b| {
b.mount_point().to_string_lossy().len().cmp(&a.mount_point().to_string_lossy().len())
});
for disk in sys.disks() {
if dht_storage_path.starts_with(disk.mount_point()) {
let available_mb = disk.available_space() / 1_000_000usize;
if available_mb > 40_000 {
// Default to 10GB if more than 40GB is available
return 10_000;
}
// Default to 1/4 of the available space, if less than 40GB is available
return available_mb;
}
}
// If we can't figure out our storage path go with 1GB of space and pray
1_000
}
pub fn set(&self, key: &str, value: &str) -> EyreResult<()> {
let mut inner = self.inner.write();
@ -937,6 +987,12 @@ impl Settings {
inner.core.network.dht.validate_dial_info_receipt_time_ms,
value
);
set_config_value!(inner.core.network.dht.local_subkey_cache_size, value);
set_config_value!(inner.core.network.dht.local_max_subkey_cache_memory_mb, value);
set_config_value!(inner.core.network.dht.remote_subkey_cache_size, value);
set_config_value!(inner.core.network.dht.remote_max_records, value);
set_config_value!(inner.core.network.dht.remote_max_subkey_cache_memory_mb, value);
set_config_value!(inner.core.network.dht.remote_max_storage_space_mb, value);
set_config_value!(inner.core.network.upnp, value);
set_config_value!(inner.core.network.detect_address_changes, value);
set_config_value!(inner.core.network.restricted_nat_retries, value);
@ -1145,6 +1201,25 @@ impl Settings {
"network.dht.validate_dial_info_receipt_time_ms" => Ok(Box::new(
inner.core.network.dht.validate_dial_info_receipt_time_ms,
)),
"network.dht.local_subkey_cache_size" => Ok(Box::new(
inner.core.network.dht.local_subkey_cache_size,
)),
"network.dht.local_max_subkey_cache_memory_mb" => Ok(Box::new(
inner.core.network.dht.local_max_subkey_cache_memory_mb,
)),
"network.dht.remote_subkey_cache_size" => Ok(Box::new(
inner.core.network.dht.remote_subkey_cache_size
)),
"network.dht.remote_max_records" => Ok(Box::new(
inner.core.network.dht.remote_max_records,
)),
"network.dht.remote_max_subkey_cache_memory_mb" => Ok(Box::new(
inner.core.network.dht.remote_max_subkey_cache_memory_mb,
)),
"network.dht.remote_max_storage_space_mb" => Ok(Box::new(
inner.core.network.dht.remote_max_storage_space_mb,
)),
"network.upnp" => Ok(Box::new(inner.core.network.upnp)),
"network.detect_address_changes" => {
Ok(Box::new(inner.core.network.detect_address_changes))