From cb899b44eab1b02210ee3034d2138c6fca10c567 Mon Sep 17 00:00:00 2001 From: John Smith Date: Fri, 12 May 2023 20:13:04 -0400 Subject: [PATCH] api work --- veilid-cli/src/client_api_connection.rs | 2 +- veilid-cli/src/command_processor.rs | 2 +- veilid-core/src/routing_table/debug.rs | 48 ++- .../route_spec_store/route_spec_store.rs | 2 +- veilid-core/src/storage_manager/debug.rs | 18 + veilid-core/src/storage_manager/mod.rs | 1 + .../src/storage_manager/record_store.rs | 31 ++ veilid-core/src/veilid_api/debug.rs | 36 ++ .../src/veilid_api/types/veilid_state.rs | 4 +- veilid-flutter/lib/veilid.dart | 345 ++++++++++++++++-- veilid-server/src/cmdline.rs | 2 +- veilid-server/src/main.rs | 25 +- 12 files changed, 448 insertions(+), 68 deletions(-) create mode 100644 veilid-core/src/storage_manager/debug.rs diff --git a/veilid-cli/src/client_api_connection.rs b/veilid-cli/src/client_api_connection.rs index 0bb89122..7a79dcd4 100644 --- a/veilid-cli/src/client_api_connection.rs +++ b/veilid-cli/src/client_api_connection.rs @@ -92,7 +92,7 @@ impl veilid_client::Server for VeilidClientImpl { VeilidUpdate::Config(config) => { self.comproc.update_config(config); } - VeilidUpdate::Route(route) => { + VeilidUpdate::RouteChange(route) => { self.comproc.update_route(route); } VeilidUpdate::Shutdown => self.comproc.update_shutdown(), diff --git a/veilid-cli/src/command_processor.rs b/veilid-cli/src/command_processor.rs index 5832fa21..60fe90d7 100644 --- a/veilid-cli/src/command_processor.rs +++ b/veilid-cli/src/command_processor.rs @@ -406,7 +406,7 @@ reply - reply to an AppCall not handled directly by the server pub fn update_config(&mut self, config: veilid_core::VeilidStateConfig) { self.inner_mut().ui.set_config(config.config) } - pub fn update_route(&mut self, route: veilid_core::VeilidStateRoute) { + pub fn update_route(&mut self, route: veilid_core::VeilidRouteChange) { let mut out = String::new(); if !route.dead_routes.is_empty() { out.push_str(&format!("Dead routes: {:?}", route.dead_routes)); diff --git a/veilid-core/src/routing_table/debug.rs b/veilid-core/src/routing_table/debug.rs index 6b4772e9..7e1cf476 100644 --- a/veilid-core/src/routing_table/debug.rs +++ b/veilid-core/src/routing_table/debug.rs @@ -2,28 +2,6 @@ use super::*; use routing_table::tasks::bootstrap::BOOTSTRAP_TXT_VERSION_0; impl RoutingTable { - pub(crate) fn debug_info_nodeinfo(&self) -> String { - let mut out = String::new(); - let inner = self.inner.read(); - out += "Routing Table Info:\n"; - - out += &format!(" Node Ids: {}\n", self.unlocked_inner.node_ids()); - out += &format!( - " Self Latency Stats Accounting: {:#?}\n\n", - inner.self_latency_stats_accounting - ); - out += &format!( - " Self Transfer Stats Accounting: {:#?}\n\n", - inner.self_transfer_stats_accounting - ); - out += &format!( - " Self Transfer Stats: {:#?}\n\n", - inner.self_transfer_stats - ); - - out - } - pub(crate) async fn debug_info_txtrecord(&self) -> String { let mut out = String::new(); @@ -71,14 +49,34 @@ impl RoutingTable { node_ids, some_hostname.unwrap() ); - for short_url in short_urls { - out += &format!(",{}", short_url); - } + out += &short_urls.join(","); out += "\n"; } out } + pub(crate) fn debug_info_nodeinfo(&self) -> String { + let mut out = String::new(); + let inner = self.inner.read(); + out += "Routing Table Info:\n"; + + out += &format!(" Node Ids: {}\n", self.unlocked_inner.node_ids()); + out += &format!( + " Self Latency Stats Accounting: {:#?}\n\n", + inner.self_latency_stats_accounting + ); + out += &format!( + " Self Transfer Stats Accounting: {:#?}\n\n", + inner.self_transfer_stats_accounting + ); + out += &format!( + " Self Transfer Stats: {:#?}\n\n", + inner.self_transfer_stats + ); + + out + } + pub(crate) fn debug_info_dialinfo(&self) -> String { let ldis = self.dial_info_details(RoutingDomain::LocalNetwork); let gdis = self.dial_info_details(RoutingDomain::PublicInternet); diff --git a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs index 134de46c..3f29ba0d 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_spec_store.rs @@ -115,7 +115,7 @@ impl RouteSpecStore { dr }; - let update = VeilidUpdate::Route(VeilidStateRoute { + let update = VeilidUpdate::RouteChange(VeilidRouteChange { dead_routes, dead_remote_routes, }); diff --git a/veilid-core/src/storage_manager/debug.rs b/veilid-core/src/storage_manager/debug.rs new file mode 100644 index 00000000..0c47b56a --- /dev/null +++ b/veilid-core/src/storage_manager/debug.rs @@ -0,0 +1,18 @@ +use super::*; + +impl StorageManager { + pub(crate) async fn debug_local_records(&self) -> String { + let inner = self.inner.lock().await; + let Some(local_record_store) = &inner.local_record_store else { + return "not initialized".to_owned(); + }; + local_record_store.debug_records() + } + pub(crate) async fn debug_remote_records(&self) -> String { + let inner = self.inner.lock().await; + let Some(remote_record_store) = &inner.remote_record_store else { + return "not initialized".to_owned(); + }; + remote_record_store.debug_records() + } +} diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index 5e8b4fb5..b37142b3 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -1,3 +1,4 @@ +mod debug; mod get_value; mod keys; mod record_store; diff --git a/veilid-core/src/storage_manager/record_store.rs b/veilid-core/src/storage_manager/record_store.rs index d37e1d3b..d23eb14d 100644 --- a/veilid-core/src/storage_manager/record_store.rs +++ b/veilid-core/src/storage_manager/record_store.rs @@ -516,4 +516,35 @@ where } self.purge_dead_records(false).await; } + + pub(super) fn debug_records(&self) -> String { + // Dump fields in an abbreviated way + let mut out = String::new(); + + out += "Record Index:\n"; + for (rik, rec) in &self.record_index { + out += &format!( + " {} @ {} len={}\n", + rik.key.to_string(), + rec.last_touched().as_u64(), + rec.record_data_size() + ); + } + out += &format!("Subkey Cache Count: {}\n", self.subkey_cache.len()); + out += &format!( + "Subkey Cache Total Size: {}\n", + self.subkey_cache_total_size + ); + out += &format!("Total Storage Space: {}\n", self.total_storage_space); + out += &format!("Dead Records: {}\n", self.dead_records.len()); + for dr in &self.dead_records { + out += &format!(" {}\n", dr.0.key.to_string()); + } + out += &format!("Changed Records: {}\n", self.changed_records.len()); + for cr in &self.changed_records { + out += &format!(" {}\n", cr.key.to_string()); + } + + out + } } diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 2b7c3c49..f54471e2 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -874,6 +874,39 @@ impl VeilidAPI { } } + async fn debug_record_list(&self, args: Vec) -> Result { + // + let storage_manager = self.storage_manager()?; + + let scope = get_debug_argument_at(&args, 1, "debug_record_list", "scope", get_string)?; + let out = match scope.as_str() { + "local" => { + let mut out = format!("Local Records:\n"); + out += &storage_manager.debug_local_records().await; + out + } + "remote" => { + let mut out = format!("Remote Records:\n"); + out += &storage_manager.debug_remote_records().await; + out + } + _ => "Invalid scope\n".to_owned(), + }; + return Ok(out); + } + + async fn debug_record(&self, args: String) -> Result { + let args: Vec = args.split_whitespace().map(|s| s.to_owned()).collect(); + + let command = get_debug_argument_at(&args, 0, "debug_record", "command", get_string)?; + + if command == "list" { + self.debug_record_list(args).await + } else { + Ok(">>> Unknown command\n".to_owned()) + } + } + pub async fn debug_help(&self, _args: String) -> Result { Ok(r#">>> Debug commands: help @@ -897,6 +930,7 @@ impl VeilidAPI { list import test + record list is: * direct: [+][] @@ -953,6 +987,8 @@ impl VeilidAPI { self.debug_restart(rest).await } else if arg == "route" { self.debug_route(rest).await + } else if arg == "record" { + self.debug_record(rest).await } else { Err(VeilidAPIError::generic("Unknown debug command")) } diff --git a/veilid-core/src/veilid_api/types/veilid_state.rs b/veilid-core/src/veilid_api/types/veilid_state.rs index d2bb50b2..739cbee3 100644 --- a/veilid-core/src/veilid_api/types/veilid_state.rs +++ b/veilid-core/src/veilid_api/types/veilid_state.rs @@ -96,7 +96,7 @@ pub struct VeilidStateNetwork { Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, )] #[archive_attr(repr(C), derive(CheckBytes))] -pub struct VeilidStateRoute { +pub struct VeilidRouteChange { pub dead_routes: Vec, pub dead_remote_routes: Vec, } @@ -130,7 +130,7 @@ pub enum VeilidUpdate { Attachment(VeilidStateAttachment), Network(VeilidStateNetwork), Config(VeilidStateConfig), - Route(VeilidStateRoute), + RouteChange(VeilidRouteChange), ValueChange(VeilidValueChange), Shutdown, } diff --git a/veilid-flutter/lib/veilid.dart b/veilid-flutter/lib/veilid.dart index 387d0bba..c6087efe 100644 --- a/veilid-flutter/lib/veilid.dart +++ b/veilid-flutter/lib/veilid.dart @@ -234,6 +234,201 @@ Object? veilidApiToEncodable(Object? value) { throw UnsupportedError('Cannot convert to JSON: $value'); } +////////////////////////////////////// +/// Crypto + +typedef CryptoKind = String; +const cryptoKindVLD0 = "VLD0"; +const cryptoKindNONE = "NONE"; + +////////////////////////////////////// +/// DHT Schema + +abstract class DHTSchema { + factory DHTSchema.fromJson(dynamic json) { + switch (json["kind"]) { + case "DFLT": + { + return DHTSchemaDFLT(oCnt: json["o_cnt"]); + } + case "SMPL": + { + return DHTSchemaSMPL( + oCnt: json["o_cnt"], + members: List.from( + json['members'].map((j) => DHTSchemaMember.fromJson(j)))); + } + default: + { + throw VeilidAPIExceptionInternal( + "Invalid VeilidAPIException type: ${json['kind']}"); + } + } + } + Map get json; +} + +class DHTSchemaDFLT implements DHTSchema { + final int oCnt; + // + DHTSchemaDFLT({ + required this.oCnt, + }) { + if (oCnt < 0 || oCnt > 65535) { + throw VeilidAPIExceptionInvalidArgument( + "value out of range", "oCnt", oCnt.toString()); + } + } + + @override + Map get json { + return { + 'kind': "DFLT", + 'o_cnt': oCnt, + }; + } +} + +class DHTSchemaMember { + String mKey; + int mCnt; + + DHTSchemaMember({ + required this.mKey, + required this.mCnt, + }) { + if (mCnt < 0 || mCnt > 65535) { + throw VeilidAPIExceptionInvalidArgument( + "value out of range", "mCnt", mCnt.toString()); + } + } + + Map get json { + return { + 'm_key': mKey, + 'm_cnt': mCnt, + }; + } + + DHTSchemaMember.fromJson(dynamic json) + : mKey = json['m_key'], + mCnt = json['m_cnt']; +} + +class DHTSchemaSMPL implements DHTSchema { + final int oCnt; + final List members; + // + DHTSchemaSMPL({ + required this.oCnt, + required this.members, + }) { + if (oCnt < 0 || oCnt > 65535) { + throw VeilidAPIExceptionInvalidArgument( + "value out of range", "oCnt", oCnt.toString()); + } + } + @override + Map get json { + return { + 'kind': "SMPL", + 'o_cnt': oCnt, + 'members': members.map((p) => p.json).toList(), + }; + } +} + +////////////////////////////////////// +/// DHTRecordDescriptor + +class DHTRecordDescriptor { + String key; + String owner; + String? ownerSecret; + DHTSchema schema; + + DHTRecordDescriptor({ + required this.key, + required this.owner, + this.ownerSecret, + required this.schema, + }); + + Map get json { + return { + 'key': key, + 'owner': owner, + 'owner_secret': ownerSecret, + 'schema': schema.json, + }; + } + + DHTRecordDescriptor.fromJson(dynamic json) + : key = json['key'], + owner = json['owner'], + ownerSecret = json['owner_secret'], + schema = DHTSchema.fromJson(json['schema']); +} + +////////////////////////////////////// +/// ValueSubkeyRange + +class ValueSubkeyRange { + final int low; + final int high; + + ValueSubkeyRange({ + required this.low, + required this.high, + }) { + if (low < 0 || low > high) { + throw VeilidAPIExceptionInvalidArgument( + "invalid range", "low", low.toString()); + } + if (high < 0) { + throw VeilidAPIExceptionInvalidArgument( + "invalid range", "high", high.toString()); + } + } + + ValueSubkeyRange.fromJson(dynamic json) + : low = json[0], + high = json[1] { + if ((json as List).length != 2) { + throw VeilidAPIExceptionInvalidArgument( + "not a pair of integers", "json", json.toString()); + } + } + + List get json { + return [low, high]; + } +} + +////////////////////////////////////// +/// ValueData + +class ValueData { + final int seq; + final Uint8List data; + final String writer; + + ValueData({ + required this.seq, + required this.data, + required this.writer, + }); + + ValueData.fromJson(dynamic json) + : seq = json['seq'], + data = base64UrlNoPadDecode(json['data']), + writer = json['writer']; + + Map get json { + return {'seq': seq, 'data': base64UrlNoPadEncode(data), 'writer': writer}; + } +} + ////////////////////////////////////// /// AttachmentState @@ -1287,9 +1482,21 @@ abstract class VeilidUpdate { { return VeilidUpdateConfig(state: VeilidStateConfig.fromJson(json)); } - case "Route": + case "RouteChange": { - return VeilidUpdateRoute(state: VeilidStateRoute.fromJson(json)); + return VeilidUpdateRouteChange( + deadRoutes: List.from(json['dead_routes'].map((j) => j)), + deadRemoteRoutes: + List.from(json['dead_remote_routes'].map((j) => j))); + } + case "ValueChange": + { + return VeilidUpdateValueChange( + key: json['key'], + subkeys: List.from( + json['subkeys'].map((j) => ValueSubkeyRange.fromJson(j))), + count: json['count'], + valueData: ValueData.fromJson(json['value_data'])); } default: { @@ -1405,16 +1612,46 @@ class VeilidUpdateConfig implements VeilidUpdate { } } -class VeilidUpdateRoute implements VeilidUpdate { - final VeilidStateRoute state; +class VeilidUpdateRouteChange implements VeilidUpdate { + final List deadRoutes; + final List deadRemoteRoutes; // - VeilidUpdateRoute({required this.state}); + VeilidUpdateRouteChange({ + required this.deadRoutes, + required this.deadRemoteRoutes, + }); @override Map get json { - var jsonRep = state.json; - jsonRep['kind'] = "Route"; - return jsonRep; + return { + 'dead_routes': deadRoutes.map((p) => p).toList(), + 'dead_remote_routes': deadRemoteRoutes.map((p) => p).toList() + }; + } +} + +class VeilidUpdateValueChange implements VeilidUpdate { + final String key; + final List subkeys; + final int count; + final ValueData valueData; + + // + VeilidUpdateValueChange({ + required this.key, + required this.subkeys, + required this.count, + required this.valueData, + }); + + @override + Map get json { + return { + 'key': key, + 'subkeys': subkeys.map((p) => p.json).toList(), + 'count': count, + 'value_data': valueData.json, + }; } } @@ -1492,31 +1729,6 @@ class VeilidStateConfig { } } -////////////////////////////////////// -/// VeilidStateRoute - -class VeilidStateRoute { - final List deadRoutes; - final List deadRemoteRoutes; - - VeilidStateRoute({ - required this.deadRoutes, - required this.deadRemoteRoutes, - }); - - VeilidStateRoute.fromJson(dynamic json) - : deadRoutes = List.from(json['dead_routes'].map((j) => j)), - deadRemoteRoutes = - List.from(json['dead_remote_routes'].map((j) => j)); - - Map get json { - return { - 'dead_routes': deadRoutes.map((p) => p).toList(), - 'dead_remote_routes': deadRemoteRoutes.map((p) => p).toList() - }; - } -} - ////////////////////////////////////// /// VeilidState @@ -1893,12 +2105,79 @@ class RouteBlob { ////////////////////////////////////// /// VeilidRoutingContext + abstract class VeilidRoutingContext { VeilidRoutingContext withPrivacy(); VeilidRoutingContext withCustomPrivacy(Stability stability); VeilidRoutingContext withSequencing(Sequencing sequencing); Future appCall(String target, Uint8List request); Future appMessage(String target, Uint8List message); + + Future createDHTRecord( + CryptoKind kind, DHTSchema schema); +xxx continue here + // pub async fn open_dht_record( + // &self, + // key: TypedKey, + // writer: Option, + // ) -> Result { + // let storage_manager = self.api.storage_manager()?; + // storage_manager + // .open_record(key, writer, self.unlocked_inner.safety_selection) + // .await + // } + + // pub async fn close_dht_record(&self, key: TypedKey) -> Result<(), VeilidAPIError> { + // let storage_manager = self.api.storage_manager()?; + // storage_manager.close_record(key).await + // } + + // pub async fn delete_dht_record(&self, key: TypedKey) -> Result<(), VeilidAPIError> { + // let storage_manager = self.api.storage_manager()?; + // storage_manager.delete_record(key).await + // } + + // pub async fn get_dht_value( + // &self, + // key: TypedKey, + // subkey: ValueSubkey, + // force_refresh: bool, + // ) -> Result, VeilidAPIError> { + // let storage_manager = self.api.storage_manager()?; + // storage_manager.get_value(key, subkey, force_refresh).await + // } + + // pub async fn set_dht_value( + // &self, + // key: TypedKey, + // subkey: ValueSubkey, + // data: Vec, + // ) -> Result, VeilidAPIError> { + // let storage_manager = self.api.storage_manager()?; + // storage_manager.set_value(key, subkey, data).await + // } + + // pub async fn watch_dht_values( + // &self, + // key: TypedKey, + // subkeys: &[ValueSubkeyRange], + // expiration: Timestamp, + // count: u32, + // ) -> Result { + // let storage_manager = self.api.storage_manager()?; + // storage_manager + // .watch_values(key, subkeys, expiration, count) + // .await + // } + + // pub async fn cancel_dht_watch( + // &self, + // key: TypedKey, + // subkeys: &[ValueSubkeyRange], + // ) -> Result { + // let storage_manager = self.api.storage_manager()?; + // storage_manager.cancel_watch_values(key, subkeys).await + // } } ///////////////////////////////////// diff --git a/veilid-server/src/cmdline.rs b/veilid-server/src/cmdline.rs index 9f37de05..f2e7906a 100644 --- a/veilid-server/src/cmdline.rs +++ b/veilid-server/src/cmdline.rs @@ -82,7 +82,7 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result EyreResult<()> { // --- Generate DHT Key --- if matches.occurrences_of("generate-key-pair") != 0 { if let Some(ckstr) = matches.get_one::("generate-key-pair") { - let ck: veilid_core::CryptoKind = - veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?; - let tkp = veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?; - println!("{}", tkp.to_string()); + if ckstr == "" { + let mut tks = veilid_core::TypedKeySet::new(); + let mut tss = veilid_core::TypedSecretSet::new(); + for ck in veilid_core::VALID_CRYPTO_KINDS { + let tkp = veilid_core::Crypto::generate_keypair(ck) + .wrap_err("invalid crypto kind")?; + tks.add(veilid_core::TypedKey::new(tkp.kind, tkp.value.key)); + tss.add(veilid_core::TypedSecret::new(tkp.kind, tkp.value.secret)); + } + println!( + "Public Keys:\n{}\nSecret Keys:\n{}\n", + tks.to_string(), + tss.to_string() + ); + } else { + let ck: veilid_core::CryptoKind = + veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?; + let tkp = + veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?; + println!("{}", tkp.to_string()); + } return Ok(()); } else { bail!("missing crypto kind");