Git-Mirrors/veilidchat
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilidchat.git synced 2024-10-01 06:55:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

better handling of subkeys for spine

Browse Source
This commit is contained in:
Christien Rioux 2024-06-08 23:18:54 -04:00
parent 0b9835b23d
commit 2c3d4dce93
9 changed files with 203 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

294
packages/veilid_support/example/integration_test/app_test.dart
View File

@ -36,161 +36,161 @@ void main() {
setUpAll(veilidFixture.attach); setUpAll(veilidFixture.attach);
tearDownAll(veilidFixture.detach); tearDownAll(veilidFixture.detach);
// group('TableDB Tests', () { group('TableDB Tests', () {
// group('TableDBArray Tests', () { group('TableDBArray Tests', () {
// // test('create/delete TableDBArray', testTableDBArrayCreateDelete); // test('create/delete TableDBArray', testTableDBArrayCreateDelete);
// group('TableDBArray Add/Get Tests', () { group('TableDBArray Add/Get Tests', () {
// for (final params in [ for (final params in [
// // //
// (99, 3, 15), (99, 3, 15),
// (100, 4, 16), (100, 4, 16),
// (101, 5, 17), (101, 5, 17),
// // //
// (511, 3, 127), (511, 3, 127),
// (512, 4, 128), (512, 4, 128),
// (513, 5, 129), (513, 5, 129),
// // //
// (4095, 3, 1023), (4095, 3, 1023),
// (4096, 4, 1024), (4096, 4, 1024),
// (4097, 5, 1025), (4097, 5, 1025),
// // //
// (65535, 3, 16383), (65535, 3, 16383),
// (65536, 4, 16384), (65536, 4, 16384),
// (65537, 5, 16385), (65537, 5, 16385),
// ]) { ]) {
// final count = params.$1; final count = params.$1;
// final singles = params.$2; final singles = params.$2;
// final batchSize = params.$3; final batchSize = params.$3;
// test( test(
// timeout: const Timeout(Duration(seconds: 480)), timeout: const Timeout(Duration(seconds: 480)),
// 'add/remove TableDBArray count = $count batchSize=$batchSize', 'add/remove TableDBArray count = $count batchSize=$batchSize',
// makeTestTableDBArrayAddGetClear( makeTestTableDBArrayAddGetClear(
// count: count, count: count,
// singles: singles, singles: singles,
// batchSize: batchSize, batchSize: batchSize,
// crypto: const VeilidCryptoPublic()), crypto: const VeilidCryptoPublic()),
// ); );
// } }
// }); });
// group('TableDBArray Insert Tests', () { group('TableDBArray Insert Tests', () {
// for (final params in [ for (final params in [
// // //
// (99, 3, 15), (99, 3, 15),
// (100, 4, 16), (100, 4, 16),
// (101, 5, 17), (101, 5, 17),
// // //
// (511, 3, 127), (511, 3, 127),
// (512, 4, 128), (512, 4, 128),
// (513, 5, 129), (513, 5, 129),
// // //
// (4095, 3, 1023), (4095, 3, 1023),
// (4096, 4, 1024), (4096, 4, 1024),
// (4097, 5, 1025), (4097, 5, 1025),
// // //
// (65535, 3, 16383), (65535, 3, 16383),
// (65536, 4, 16384), (65536, 4, 16384),
// (65537, 5, 16385), (65537, 5, 16385),
// ]) { ]) {
// final count = params.$1; final count = params.$1;
// final singles = params.$2; final singles = params.$2;
// final batchSize = params.$3; final batchSize = params.$3;
// test( test(
// timeout: const Timeout(Duration(seconds: 480)), timeout: const Timeout(Duration(seconds: 480)),
// 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize', 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
// makeTestTableDBArrayInsert( makeTestTableDBArrayInsert(
// count: count, count: count,
// singles: singles, singles: singles,
// batchSize: batchSize, batchSize: batchSize,
// crypto: const VeilidCryptoPublic()), crypto: const VeilidCryptoPublic()),
// ); );
// } }
// }); });
// group('TableDBArray Remove Tests', () { group('TableDBArray Remove Tests', () {
// for (final params in [ for (final params in [
// // //
// (99, 3, 15), (99, 3, 15),
// (100, 4, 16), (100, 4, 16),
// (101, 5, 17), (101, 5, 17),
// // //
// (511, 3, 127), (511, 3, 127),
// (512, 4, 128), (512, 4, 128),
// (513, 5, 129), (513, 5, 129),
// // //
// (4095, 3, 1023), (4095, 3, 1023),
// (4096, 4, 1024), (4096, 4, 1024),
// (4097, 5, 1025), (4097, 5, 1025),
// // //
// (16383, 3, 4095), (16383, 3, 4095),
// (16384, 4, 4096), (16384, 4, 4096),
// (16385, 5, 4097), (16385, 5, 4097),
// ]) { ]) {
// final count = params.$1; final count = params.$1;
// final singles = params.$2; final singles = params.$2;
// final batchSize = params.$3; final batchSize = params.$3;
// test( test(
// timeout: const Timeout(Duration(seconds: 480)), timeout: const Timeout(Duration(seconds: 480)),
// 'remove TableDBArray count=$count singles=$singles batchSize=$batchSize', 'remove TableDBArray count=$count singles=$singles batchSize=$batchSize',
// makeTestTableDBArrayRemove( makeTestTableDBArrayRemove(
// count: count, count: count,
// singles: singles, singles: singles,
// batchSize: batchSize, batchSize: batchSize,
// crypto: const VeilidCryptoPublic()), crypto: const VeilidCryptoPublic()),
// ); );
// } }
// }); });
// });
// });
group('DHT Support Tests', () {
setUpAll(updateProcessorFixture.setUp);
setUpAll(tickerFixture.setUp);
tearDownAll(tickerFixture.tearDown);
tearDownAll(updateProcessorFixture.tearDown);
test('create pool', testDHTRecordPoolCreate);
group('DHTRecordPool Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
test('create/delete record', testDHTRecordCreateDelete);
test('record scopes', testDHTRecordScopes);
test('create/delete deep record', testDHTRecordDeepCreateDelete);
});
group('DHTShortArray Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create shortarray stride=$stride',
makeTestDHTShortArrayCreateDelete(stride: stride));
test('add shortarray stride=$stride',
makeTestDHTShortArrayAdd(stride: stride));
}
});
group('DHTLog Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create log stride=$stride',
makeTestDHTLogCreateDelete(stride: stride));
test(
timeout: const Timeout(Duration(seconds: 480)),
'add/truncate log stride=$stride',
makeTestDHTLogAddTruncate(stride: stride),
);
}
}); });
}); });
// group('DHT Support Tests', () {
// setUpAll(updateProcessorFixture.setUp);
// setUpAll(tickerFixture.setUp);
// tearDownAll(tickerFixture.tearDown);
// tearDownAll(updateProcessorFixture.tearDown);
// test('create pool', testDHTRecordPoolCreate);
// group('DHTRecordPool Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
// test('create/delete record', testDHTRecordCreateDelete);
// test('record scopes', testDHTRecordScopes);
// test('create/delete deep record', testDHTRecordDeepCreateDelete);
// });
// group('DHTShortArray Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create shortarray stride=$stride',
// makeTestDHTShortArrayCreateDelete(stride: stride));
// test('add shortarray stride=$stride',
// makeTestDHTShortArrayAdd(stride: stride));
// }
// });
// group('DHTLog Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create log stride=$stride',
// makeTestDHTLogCreateDelete(stride: stride));
// test(
// timeout: const Timeout(Duration(seconds: 480)),
// 'add/truncate log stride=$stride',
// makeTestDHTLogAddTruncate(stride: stride),
// );
// }
// });
// });
}); });
}); });
} }

3
packages/veilid_support/lib/dht_support/src/dht_log/dht_log.dart
View File

@ -204,6 +204,9 @@ class DHTLog implements DHTDeleteable<DHTLog> {
/// Get the record key for this log /// Get the record key for this log
TypedKey get recordKey => _spine.recordKey; TypedKey get recordKey => _spine.recordKey;
/// Get the writer for the log
KeyPair? get writer => _spine._spineRecord.writer;
/// Get the record pointer foir this log /// Get the record pointer foir this log
OwnedDHTRecordPointer get recordPointer => _spine.recordPointer; OwnedDHTRecordPointer get recordPointer => _spine.recordPointer;

13
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_cubit.dart
View File

@ -126,13 +126,22 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
final end = ((tail - 1) % length) + 1; final end = ((tail - 1) % length) + 1;
final start = (count < end) ? end - count : 0; final start = (count < end) ? end - count : 0;
final offlinePositions = await reader.getOfflinePositions(); // If this is writeable get the offline positions
Set<int>? offlinePositions;
if (_log.writer != null) {
offlinePositions = await reader.getOfflinePositions();
if (offlinePositions == null) {
return const AsyncValue.loading();
}
}
// Get the items
final allItems = (await reader.getRange(start, final allItems = (await reader.getRange(start,
length: end - start, forceRefresh: forceRefresh)) length: end - start, forceRefresh: forceRefresh))
?.indexed ?.indexed
.map((x) => OnlineElementState( .map((x) => OnlineElementState(
value: _decodeElement(x.$2), value: _decodeElement(x.$2),
isOffline: offlinePositions.contains(x.$1))) isOffline: offlinePositions?.contains(x.$1) ?? false))
.toIList(); .toIList();
if (allItems == null) { if (allItems == null) {
return const AsyncValue.loading(); return const AsyncValue.loading();

14
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_read.dart
View File

@ -61,20 +61,23 @@ class _DHTLogRead implements DHTLogReadOperations {
} }
@override @override
Future<Set<int>> getOfflinePositions() async { Future<Set<int>?> getOfflinePositions() async {
final positionOffline = <int>{}; final positionOffline = <int>{};
// Iterate positions backward from most recent // Iterate positions backward from most recent
for (var pos = _spine.length - 1; pos >= 0; pos--) { for (var pos = _spine.length - 1; pos >= 0; pos--) {
final lookup = await _spine.lookupPosition(pos); final lookup = await _spine.lookupPosition(pos);
if (lookup == null) { if (lookup == null) {
throw StateError('Unable to look up position'); return null;
} }
// Check each segment for offline positions // Check each segment for offline positions
var foundOffline = false; var foundOffline = false;
await lookup.scope((sa) => sa.operate((read) async { final success = await lookup.scope((sa) => sa.operate((read) async {
final segmentOffline = await read.getOfflinePositions(); final segmentOffline = await read.getOfflinePositions();
if (segmentOffline == null) {
return false;
}
// For each shortarray segment go through their segment positions // For each shortarray segment go through their segment positions
// in reverse order and see if they are offline // in reverse order and see if they are offline
@ -88,8 +91,11 @@ class _DHTLogRead implements DHTLogReadOperations {
foundOffline = true; foundOffline = true;
} }
} }
return true;
})); }));
if (!success) {
return null;
}
// If we found nothing offline in this segment then we can stop // If we found nothing offline in this segment then we can stop
if (!foundOffline) { if (!foundOffline) {
break; break;

21
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_spine.dart
View File

@ -354,13 +354,24 @@ class _DHTLogSpine {
final subkey = l.subkey; final subkey = l.subkey;
final segment = l.segment; final segment = l.segment;
final subkeyData = await _spineRecord.get(subkey: subkey); // See if we have the segment key locally
if (subkeyData == null) { TypedKey? segmentKey;
return null; var subkeyData = await _spineRecord.get(
subkey: subkey, refreshMode: DHTRecordRefreshMode.local);
if (subkeyData != null) {
segmentKey = _getSegmentKey(subkeyData, segment);
} }
final segmentKey = _getSegmentKey(subkeyData, segment);
if (segmentKey == null) { if (segmentKey == null) {
return null; // If not, try from the network
subkeyData = await _spineRecord.get(
subkey: subkey, refreshMode: DHTRecordRefreshMode.network);
if (subkeyData == null) {
return null;
}
segmentKey = _getSegmentKey(subkeyData, segment);
if (segmentKey == null) {
return null;
}
} }
// Open a shortarray segment // Open a shortarray segment

2
packages/veilid_support/lib/dht_support/src/dht_record/dht_record.dart
View File

@ -16,7 +16,7 @@ class DHTRecordWatchChange extends Equatable {
/// Refresh mode for DHT record 'get' /// Refresh mode for DHT record 'get'
enum DHTRecordRefreshMode { enum DHTRecordRefreshMode {
/// Return existing subkey values if they exist locally already /// Return existing subkey values if they exist locally already
/// And then check the network for a newer value /// If not, check the network for a value
/// This is the default refresh mode /// This is the default refresh mode
cached, cached,

3
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array.dart
View File

@ -182,6 +182,9 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray> {
/// Get the record key for this shortarray /// Get the record key for this shortarray
TypedKey get recordKey => _head.recordKey; TypedKey get recordKey => _head.recordKey;
/// Get the writer for the log
KeyPair? get writer => _head._headRecord.writer;
/// Get the record pointer foir this shortarray /// Get the record pointer foir this shortarray
OwnedDHTRecordPointer get recordPointer => _head.recordPointer; OwnedDHTRecordPointer get recordPointer => _head.recordPointer;

13
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_cubit.dart
View File

@ -53,12 +53,21 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
{bool forceRefresh = false}) async { {bool forceRefresh = false}) async {
try { try {
final newState = await _shortArray.operate((reader) async { final newState = await _shortArray.operate((reader) async {
final offlinePositions = await reader.getOfflinePositions(); // If this is writeable get the offline positions
Set<int>? offlinePositions;
if (_shortArray.writer != null) {
offlinePositions = await reader.getOfflinePositions();
if (offlinePositions == null) {
return null;
}
}
// Get the items
final allItems = (await reader.getRange(0, forceRefresh: forceRefresh)) final allItems = (await reader.getRange(0, forceRefresh: forceRefresh))
?.indexed ?.indexed
.map((x) => DHTShortArrayElementState( .map((x) => DHTShortArrayElementState(
value: _decodeElement(x.$2), value: _decodeElement(x.$2),
isOffline: offlinePositions.contains(x.$1))) isOffline: offlinePositions?.contains(x.$1) ?? false))
.toIList(); .toIList();
return allItems; return allItems;
}); });

2
packages/veilid_support/lib/dht_support/src/interfaces/dht_random_read.dart
View File

@ -26,7 +26,7 @@ abstract class DHTRandomRead {
{int? length, bool forceRefresh = false}); {int? length, bool forceRefresh = false});
/// Get a list of the positions that were written offline and not flushed yet /// Get a list of the positions that were written offline and not flushed yet
Future<Set<int>> getOfflinePositions(); Future<Set<int>?> getOfflinePositions();
} }
extension DHTRandomReadExt on DHTRandomRead { extension DHTRandomReadExt on DHTRandomRead {