From 4a190a685301385338e0772394b223484e62c522 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 13 Apr 2024 14:15:09 -0400 Subject: [PATCH] add offline subkeys to dht record report --- veilid-core/src/storage_manager/mod.rs | 10 ++++++ .../veilid_api/types/dht/dht_record_report.rs | 10 +++++- veilid-flutter/lib/routing_context.dart | 1 + .../lib/routing_context.freezed.dart | 32 ++++++++++++++++++- veilid-flutter/lib/routing_context.g.dart | 5 +++ veilid-python/veilid/schema/RecvMessage.json | 22 +++++++++++++ veilid-python/veilid/types.py | 6 +++- 7 files changed, 83 insertions(+), 3 deletions(-) diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index 761010b4..d09d771e 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -795,10 +795,19 @@ impl StorageManager { "more subkeys returned locally than requested" ); + // Get the offline subkeys for this record still only returning the ones we're inspecting + let offline_subkey_writes = inner + .offline_subkey_writes + .get(&key) + .map(|o| o.subkeys.clone()) + .unwrap_or_default() + .intersect(&subkeys); + // If this is the maximum scope we're interested in, return the report if matches!(scope, DHTReportScope::Local) { return Ok(DHTRecordReport::new( local_inspect_result.subkeys, + offline_subkey_writes, local_inspect_result.seqs, vec![], )); @@ -864,6 +873,7 @@ impl StorageManager { Ok(DHTRecordReport::new( result.inspect_result.subkeys, + offline_subkey_writes, local_inspect_result.seqs, result.inspect_result.seqs, )) diff --git a/veilid-core/src/veilid_api/types/dht/dht_record_report.rs b/veilid-core/src/veilid_api/types/dht/dht_record_report.rs index 3fa74ffa..f1fce660 100644 --- a/veilid-core/src/veilid_api/types/dht/dht_record_report.rs +++ b/veilid-core/src/veilid_api/types/dht/dht_record_report.rs @@ -12,6 +12,8 @@ pub struct DHTRecordReport { /// This may be a subset of the requested range if it exceeds the schema limits /// or has more than 512 subkeys subkeys: ValueSubkeyRangeSet, + /// The subkeys that have been writen offline that still need to be flushed + offline_subkeys: ValueSubkeyRangeSet, /// The sequence numbers of each subkey requested from a locally stored DHT Record local_seqs: Vec, /// The sequence numbers of each subkey requested from the DHT over the network @@ -22,11 +24,13 @@ from_impl_to_jsvalue!(DHTRecordReport); impl DHTRecordReport { pub fn new( subkeys: ValueSubkeyRangeSet, + offline_subkeys: ValueSubkeyRangeSet, local_seqs: Vec, network_seqs: Vec, ) -> Self { Self { subkeys, + offline_subkeys, local_seqs, network_seqs, } @@ -35,6 +39,9 @@ impl DHTRecordReport { pub fn subkeys(&self) -> &ValueSubkeyRangeSet { &self.subkeys } + pub fn offline_subkeys(&self) -> &ValueSubkeyRangeSet { + &self.offline_subkeys + } pub fn local_seqs(&self) -> &[ValueSeqNum] { &self.local_seqs } @@ -47,8 +54,9 @@ impl fmt::Debug for DHTRecordReport { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "DHTRecordReport {{\n subkeys: {:?}\n local_seqs:\n{}\n remote_seqs:\n{}\n}}\n", + "DHTRecordReport {{\n subkeys: {:?}\n offline_subkeys: {:?}\n local_seqs:\n{}\n remote_seqs:\n{}\n}}\n", &self.subkeys, + &self.offline_subkeys, &debug_seqs(&self.local_seqs), &debug_seqs(&self.network_seqs) ) diff --git a/veilid-flutter/lib/routing_context.dart b/veilid-flutter/lib/routing_context.dart index 276e5518..c097dbbe 100644 --- a/veilid-flutter/lib/routing_context.dart +++ b/veilid-flutter/lib/routing_context.dart @@ -246,6 +246,7 @@ class RouteBlob with _$RouteBlob { class DHTRecordReport with _$DHTRecordReport { const factory DHTRecordReport({ required List subkeys, + required List offlineSubkeys, required List localSeqs, required List networkSeqs, }) = _DHTRecordReport; diff --git a/veilid-flutter/lib/routing_context.freezed.dart b/veilid-flutter/lib/routing_context.freezed.dart index 43add7ad..529a9315 100644 --- a/veilid-flutter/lib/routing_context.freezed.dart +++ b/veilid-flutter/lib/routing_context.freezed.dart @@ -1363,6 +1363,8 @@ DHTRecordReport _$DHTRecordReportFromJson(Map json) { /// @nodoc mixin _$DHTRecordReport { List get subkeys => throw _privateConstructorUsedError; + List get offlineSubkeys => + throw _privateConstructorUsedError; List get localSeqs => throw _privateConstructorUsedError; List get networkSeqs => throw _privateConstructorUsedError; @@ -1380,6 +1382,7 @@ abstract class $DHTRecordReportCopyWith<$Res> { @useResult $Res call( {List subkeys, + List offlineSubkeys, List localSeqs, List networkSeqs}); } @@ -1398,6 +1401,7 @@ class _$DHTRecordReportCopyWithImpl<$Res, $Val extends DHTRecordReport> @override $Res call({ Object? subkeys = null, + Object? offlineSubkeys = null, Object? localSeqs = null, Object? networkSeqs = null, }) { @@ -1406,6 +1410,10 @@ class _$DHTRecordReportCopyWithImpl<$Res, $Val extends DHTRecordReport> ? _value.subkeys : subkeys // ignore: cast_nullable_to_non_nullable as List, + offlineSubkeys: null == offlineSubkeys + ? _value.offlineSubkeys + : offlineSubkeys // ignore: cast_nullable_to_non_nullable + as List, localSeqs: null == localSeqs ? _value.localSeqs : localSeqs // ignore: cast_nullable_to_non_nullable @@ -1428,6 +1436,7 @@ abstract class _$$DHTRecordReportImplCopyWith<$Res> @useResult $Res call( {List subkeys, + List offlineSubkeys, List localSeqs, List networkSeqs}); } @@ -1444,6 +1453,7 @@ class __$$DHTRecordReportImplCopyWithImpl<$Res> @override $Res call({ Object? subkeys = null, + Object? offlineSubkeys = null, Object? localSeqs = null, Object? networkSeqs = null, }) { @@ -1452,6 +1462,10 @@ class __$$DHTRecordReportImplCopyWithImpl<$Res> ? _value._subkeys : subkeys // ignore: cast_nullable_to_non_nullable as List, + offlineSubkeys: null == offlineSubkeys + ? _value._offlineSubkeys + : offlineSubkeys // ignore: cast_nullable_to_non_nullable + as List, localSeqs: null == localSeqs ? _value._localSeqs : localSeqs // ignore: cast_nullable_to_non_nullable @@ -1469,9 +1483,11 @@ class __$$DHTRecordReportImplCopyWithImpl<$Res> class _$DHTRecordReportImpl implements _DHTRecordReport { const _$DHTRecordReportImpl( {required final List subkeys, + required final List offlineSubkeys, required final List localSeqs, required final List networkSeqs}) : _subkeys = subkeys, + _offlineSubkeys = offlineSubkeys, _localSeqs = localSeqs, _networkSeqs = networkSeqs; @@ -1486,6 +1502,14 @@ class _$DHTRecordReportImpl implements _DHTRecordReport { return EqualUnmodifiableListView(_subkeys); } + final List _offlineSubkeys; + @override + List get offlineSubkeys { + if (_offlineSubkeys is EqualUnmodifiableListView) return _offlineSubkeys; + // ignore: implicit_dynamic_type + return EqualUnmodifiableListView(_offlineSubkeys); + } + final List _localSeqs; @override List get localSeqs { @@ -1504,7 +1528,7 @@ class _$DHTRecordReportImpl implements _DHTRecordReport { @override String toString() { - return 'DHTRecordReport(subkeys: $subkeys, localSeqs: $localSeqs, networkSeqs: $networkSeqs)'; + return 'DHTRecordReport(subkeys: $subkeys, offlineSubkeys: $offlineSubkeys, localSeqs: $localSeqs, networkSeqs: $networkSeqs)'; } @override @@ -1513,6 +1537,8 @@ class _$DHTRecordReportImpl implements _DHTRecordReport { (other.runtimeType == runtimeType && other is _$DHTRecordReportImpl && const DeepCollectionEquality().equals(other._subkeys, _subkeys) && + const DeepCollectionEquality() + .equals(other._offlineSubkeys, _offlineSubkeys) && const DeepCollectionEquality() .equals(other._localSeqs, _localSeqs) && const DeepCollectionEquality() @@ -1524,6 +1550,7 @@ class _$DHTRecordReportImpl implements _DHTRecordReport { int get hashCode => Object.hash( runtimeType, const DeepCollectionEquality().hash(_subkeys), + const DeepCollectionEquality().hash(_offlineSubkeys), const DeepCollectionEquality().hash(_localSeqs), const DeepCollectionEquality().hash(_networkSeqs)); @@ -1545,6 +1572,7 @@ class _$DHTRecordReportImpl implements _DHTRecordReport { abstract class _DHTRecordReport implements DHTRecordReport { const factory _DHTRecordReport( {required final List subkeys, + required final List offlineSubkeys, required final List localSeqs, required final List networkSeqs}) = _$DHTRecordReportImpl; @@ -1554,6 +1582,8 @@ abstract class _DHTRecordReport implements DHTRecordReport { @override List get subkeys; @override + List get offlineSubkeys; + @override List get localSeqs; @override List get networkSeqs; diff --git a/veilid-flutter/lib/routing_context.g.dart b/veilid-flutter/lib/routing_context.g.dart index e0991f99..922f429f 100644 --- a/veilid-flutter/lib/routing_context.g.dart +++ b/veilid-flutter/lib/routing_context.g.dart @@ -116,6 +116,9 @@ _$DHTRecordReportImpl _$$DHTRecordReportImplFromJson( subkeys: (json['subkeys'] as List) .map(ValueSubkeyRange.fromJson) .toList(), + offlineSubkeys: (json['offline_subkeys'] as List) + .map(ValueSubkeyRange.fromJson) + .toList(), localSeqs: (json['local_seqs'] as List).map((e) => e as int).toList(), networkSeqs: @@ -126,6 +129,8 @@ Map _$$DHTRecordReportImplToJson( _$DHTRecordReportImpl instance) => { 'subkeys': instance.subkeys.map((e) => e.toJson()).toList(), + 'offline_subkeys': + instance.offlineSubkeys.map((e) => e.toJson()).toList(), 'local_seqs': instance.localSeqs, 'network_seqs': instance.networkSeqs, }; diff --git a/veilid-python/veilid/schema/RecvMessage.json b/veilid-python/veilid/schema/RecvMessage.json index 1ee1ea81..21192210 100644 --- a/veilid-python/veilid/schema/RecvMessage.json +++ b/veilid-python/veilid/schema/RecvMessage.json @@ -2805,6 +2805,7 @@ "required": [ "local_seqs", "network_seqs", + "offline_subkeys", "subkeys" ], "properties": { @@ -2826,6 +2827,27 @@ "minimum": 0.0 } }, + "offline_subkeys": { + "description": "The subkeys that have been writen offline that still need to be flushed", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + ], + "maxItems": 2, + "minItems": 2 + } + }, "subkeys": { "description": "The actual subkey range within the schema being reported on This may be a subset of the requested range if it exceeds the schema limits or has more than 512 subkeys", "type": "array", diff --git a/veilid-python/veilid/types.py b/veilid-python/veilid/types.py index 1ba74dff..630c1bc4 100644 --- a/veilid-python/veilid/types.py +++ b/veilid-python/veilid/types.py @@ -382,26 +382,30 @@ class DHTRecordDescriptor: class DHTRecordReport: subkeys: list[tuple[ValueSubkey, ValueSubkey]] + offline_subkeys: list[tuple[ValueSubkey, ValueSubkey]] local_seqs: list[ValueSeqNum] network_seqs: list[ValueSeqNum] def __init__( self, subkeys: list[tuple[ValueSubkey, ValueSubkey]], + offline_subkeys: list[tuple[ValueSubkey, ValueSubkey]], local_seqs: list[ValueSeqNum], network_seqs: list[ValueSeqNum], ): self.subkeys = subkeys + self.offline_subkey = offline_subkeys self.local_seqs = local_seqs self.network_seqs = network_seqs def __repr__(self) -> str: - return f"<{self.__class__.__name__}(subkeys={self.subkeys!r}, local_seqs={self.local_seqs!r}, network_seqs={self.network_seqs!r})>" + return f"<{self.__class__.__name__}(subkeys={self.subkeys!r}, offline_subkeys={self.offline_subkeys!r}, local_seqs={self.local_seqs!r}, network_seqs={self.network_seqs!r})>" @classmethod def from_json(cls, j: dict) -> Self: return cls( [[p[0], p[1]] for p in j["subkeys"]], + [[p[0], p[1]] for p in j["offline_subkeys"]], [ValueSeqNum(s) for s in j["local_seqs"]], [ValueSeqNum(s) for s in j["network_seqs"]], )