Git-Mirrors/veilidchat
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilidchat.git synced 2024-10-01 06:55:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dht-log' into 'main'

Browse Source 
DHT Log

See merge request veilid/veilidchat!26
This commit is contained in:
Christien Rioux 2024-05-21 23:49:33 +00:00
commit 8b64fbadc5
49 changed files with 3091 additions and 785 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/chat/cubits/single_contact_messages_cubit.dart
View File

@ -227,7 +227,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
}
Future<void> _reconcileMessagesInner(
{required DHTShortArrayWrite reconciledMessagesWriter,
{required DHTRandomReadWrite reconciledMessagesWriter,
required IList<proto.Message> messages}) async {
// Ensure remoteMessages is sorted by timestamp
final newMessages = messages
@ -236,7 +236,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
// Existing messages will always be sorted by timestamp so merging is easy
final existingMessages = await reconciledMessagesWriter
.getAllItemsProtobuf(proto.Message.fromBuffer);
.getItemRangeProtobuf(proto.Message.fromBuffer, 0);
if (existingMessages == null) {
throw Exception(
'Could not load existing reconciled messages at this time');

14
lib/chat_list/cubits/chat_list_cubit.dart
View File

@ -92,31 +92,29 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
// Remove Chat from account's list
// if this fails, don't keep retrying, user can try again later
final (deletedItem, success) =
final deletedItem =
// Ensure followers get their changes before we return
await syncFollowers(() => operateWrite((writer) async {
if (activeChatCubit.state == remoteConversationRecordKey) {
activeChatCubit.setActiveChat(null);
}
for (var i = 0; i < writer.length; i++) {
final cbuf = await writer.getItem(i);
if (cbuf == null) {
final c =
await writer.getItemProtobuf(proto.Chat.fromBuffer, i);
if (c == null) {
throw Exception('Failed to get chat');
}
final c = proto.Chat.fromBuffer(cbuf);
if (c.remoteConversationRecordKey == remoteConversationKey) {
// Found the right chat
if (await writer.tryRemoveItem(i) != null) {
await writer.removeItem(i);
return c;
}
return null;
}
}
return null;
}));
// Since followers are synced, we can safetly remove the reconciled
// chat record now
if (success && deletedItem != null) {
if (deletedItem != null) {
try {
await DHTRecordPool.instance.deleteRecord(
deletedItem.reconciledChatRecord.toVeilid().recordKey);

2
lib/chat_list/views/chat_single_contact_list_widget.dart
View File

@ -34,7 +34,7 @@ class ChatSingleContactListWidget extends StatelessWidget {
? const EmptyChatListWidget()
: SearchableList<proto.Chat>(
initialList: chatList.map((x) => x.value).toList(),
builder: (l, i, c) {
itemBuilder: (c) {
final contact =
contactMap[c.remoteConversationRecordKey];
if (contact == null) {

8
lib/contact_invitation/cubits/contact_invitation_list_cubit.dart
View File

@ -177,7 +177,7 @@ class ContactInvitationListCubit
_activeAccountInfo.userLogin.accountRecordInfo.accountRecord.recordKey;
// Remove ContactInvitationRecord from account's list
final (deletedItem, success) = await operateWrite((writer) async {
final deletedItem = await operateWrite((writer) async {
for (var i = 0; i < writer.length; i++) {
final item = await writer.getItemProtobuf(
proto.ContactInvitationRecord.fromBuffer, i);
@ -186,16 +186,14 @@ class ContactInvitationListCubit
}
if (item.contactRequestInbox.recordKey.toVeilid() ==
contactRequestInboxRecordKey) {
if (await writer.tryRemoveItem(i) != null) {
await writer.removeItem(i);
return item;
}
return null;
}
}
return null;
});
if (success && deletedItem != null) {
if (deletedItem != null) {
// Delete the contact request inbox
final contactRequestInbox = deletedItem.contactRequestInbox.toVeilid();
await (await pool.openRecordOwned(contactRequestInbox,

117
lib/contact_invitation/views/scan_invitation_dialog.dart
View File

@ -1,5 +1,4 @@
import 'dart:async';
import 'dart:io';
import 'dart:typed_data';
import 'package:awesome_extensions/awesome_extensions.dart';
@ -16,65 +15,64 @@ import '../../theme/theme.dart';
import '../../tools/tools.dart';
import 'invitation_dialog.dart';
class BarcodeOverlay extends CustomPainter {
BarcodeOverlay({
required this.barcode,
required this.arguments,
required this.boxFit,
required this.capture,
});
// class BarcodeOverlay extends CustomPainter {
// BarcodeOverlay({
// required this.barcode,
// required this.boxFit,
// required this.capture,
// required this.size,
// });
final BarcodeCapture capture;
final Barcode barcode;
final MobileScannerArguments arguments;
final BoxFit boxFit;
// final BarcodeCapture capture;
// final Barcode barcode;
// final BoxFit boxFit;
// final Size size;
@override
void paint(Canvas canvas, Size size) {
final adjustedSize = applyBoxFit(boxFit, arguments.size, size);
// @override
// void paint(Canvas canvas, Size size) {
// final adjustedSize = applyBoxFit(boxFit, size, size);
var verticalPadding = size.height - adjustedSize.destination.height;
var horizontalPadding = size.width - adjustedSize.destination.width;
if (verticalPadding > 0) {
verticalPadding = verticalPadding / 2;
} else {
verticalPadding = 0;
}
// var verticalPadding = size.height - adjustedSize.destination.height;
// var horizontalPadding = size.width - adjustedSize.destination.width;
// if (verticalPadding > 0) {
// verticalPadding = verticalPadding / 2;
// } else {
// verticalPadding = 0;
// }
if (horizontalPadding > 0) {
horizontalPadding = horizontalPadding / 2;
} else {
horizontalPadding = 0;
}
// if (horizontalPadding > 0) {
// horizontalPadding = horizontalPadding / 2;
// } else {
// horizontalPadding = 0;
// }
final ratioWidth = (Platform.isIOS ? capture.width : arguments.size.width) /
adjustedSize.destination.width;
final ratioHeight =
(Platform.isIOS ? capture.height : arguments.size.height) /
adjustedSize.destination.height;
// final ratioWidth = (Platform.isIOS ? capture.size.width : size.width) /
// adjustedSize.destination.width;
// final ratioHeight = (Platform.isIOS ? capture.size.height : size.height) /
// adjustedSize.destination.height;
final adjustedOffset = <Offset>[];
for (final offset in barcode.corners) {
adjustedOffset.add(
Offset(
offset.dx / ratioWidth + horizontalPadding,
offset.dy / ratioHeight + verticalPadding,
),
);
}
final cutoutPath = Path()..addPolygon(adjustedOffset, true);
// final adjustedOffset = <Offset>[];
// for (final offset in barcode.corners) {
// adjustedOffset.add(
// Offset(
// offset.dx / ratioWidth + horizontalPadding,
// offset.dy / ratioHeight + verticalPadding,
// ),
// );
// }
// final cutoutPath = Path()..addPolygon(adjustedOffset, true);
final backgroundPaint = Paint()
..color = Colors.red.withOpacity(0.3)
..style = PaintingStyle.fill
..blendMode = BlendMode.dstOut;
// final backgroundPaint = Paint()
// ..color = Colors.red.withOpacity(0.3)
// ..style = PaintingStyle.fill
// ..blendMode = BlendMode.dstOut;
canvas.drawPath(cutoutPath, backgroundPaint);
}
// canvas.drawPath(cutoutPath, backgroundPaint);
// }
@override
bool shouldRepaint(covariant CustomPainter oldDelegate) => false;
}
// @override
// bool shouldRepaint(covariant CustomPainter oldDelegate) => false;
// }
class ScannerOverlay extends CustomPainter {
ScannerOverlay(this.scanWindow);
@ -202,9 +200,9 @@ class ScanInvitationDialogState extends State<ScanInvitationDialog> {
IconButton(
color: Colors.white,
icon: ValueListenableBuilder(
valueListenable: cameraController.torchState,
valueListenable: cameraController,
builder: (context, state, child) {
switch (state) {
switch (state.torchState) {
case TorchState.off:
return Icon(Icons.flash_off,
color:
@ -212,6 +210,12 @@ class ScanInvitationDialogState extends State<ScanInvitationDialog> {
case TorchState.on:
return Icon(Icons.flash_on,
color: scale.primaryScale.primary);
case TorchState.auto:
return Icon(Icons.flash_auto,
color: scale.primaryScale.primary);
case TorchState.unavailable:
return Icon(Icons.no_flash,
color: scale.primaryScale.primary);
}
},
),
@ -236,10 +240,9 @@ class ScanInvitationDialogState extends State<ScanInvitationDialog> {
IconButton(
color: Colors.white,
icon: ValueListenableBuilder(
valueListenable:
cameraController.cameraFacingState,
valueListenable: cameraController,
builder: (context, state, child) {
switch (state) {
switch (state.cameraDirection) {
case CameraFacing.front:
return const Icon(Icons.camera_front);
case CameraFacing.back:
@ -265,7 +268,7 @@ class ScanInvitationDialogState extends State<ScanInvitationDialog> {
SchedulerBinding.instance
.addPostFrameCallback((_) {
cameraController.dispose();
Navigator.pop(context, null);
Navigator.pop(context);
})
})),
],

8
lib/contacts/cubits/contact_list_cubit.dart
View File

@ -70,7 +70,7 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
contact.remoteConversationRecordKey.toVeilid();
// Remove Contact from account's list
final (deletedItem, success) = await operateWrite((writer) async {
final deletedItem = await operateWrite((writer) async {
for (var i = 0; i < writer.length; i++) {
final item = await writer.getItemProtobuf(proto.Contact.fromBuffer, i);
if (item == null) {
@ -78,16 +78,14 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
}
if (item.remoteConversationRecordKey ==
contact.remoteConversationRecordKey) {
if (await writer.tryRemoveItem(i) != null) {
await writer.removeItem(i);
return item;
}
return null;
}
}
return null;
});
if (success && deletedItem != null) {
if (deletedItem != null) {
try {
// Make a conversation cubit to manipulate the conversation
final conversationCubit = ConversationCubit(

2
lib/contacts/cubits/conversation_cubit.dart
View File

@ -295,7 +295,7 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
debugName: 'ConversationCubit::initLocalMessages::LocalMessages',
parent: localConversationKey,
crypto: crypto,
smplWriter: writer))
writer: writer))
.deleteScope((messages) async => await callback(messages));
}

2
lib/contacts/views/contact_list_widget.dart
View File

@ -38,7 +38,7 @@ class ContactListWidget extends StatelessWidget {
? const EmptyContactListWidget()
: SearchableList<proto.Contact>(
initialList: contactList.toList(),
builder: (l, i, c) =>
itemBuilder: (c) =>
ContactItemWidget(contact: c, disabled: disabled)
.paddingLTRB(0, 4, 0, 0),
filter: (value) {

10
macos/Podfile.lock
View File

@ -1,6 +1,6 @@
PODS:
- FlutterMacOS (1.0.0)
- mobile_scanner (3.5.6):
- mobile_scanner (5.1.1):
- FlutterMacOS
- pasteboard (0.0.1):
- FlutterMacOS
@ -68,15 +68,15 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS:
FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24
mobile_scanner: 54ceceae0c8da2457e26a362a6be5c61154b1829
mobile_scanner: 1efac1e53c294b24e3bb55bcc7f4deee0233a86b
pasteboard: 9b69dba6fedbb04866be632205d532fe2f6b1d99
path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c
path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46
screen_retriever: 59634572a57080243dd1bf715e55b6c54f241a38
share_plus: 36537c04ce0c3e3f5bd297ce4318b6d5ee5fd6cf
shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695
shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78
smart_auth: b38e3ab4bfe089eacb1e233aca1a2340f96c28e9
sqflite: 673a0e54cc04b7d6dba8d24fb8095b31c3a99eec
url_launcher_macos: d2691c7dd33ed713bf3544850a623080ec693d95
url_launcher_macos: 5f437abeda8c85500ceb03f5c1938a8c5a705399
veilid: a54f57b7bcf0e4e072fe99272d76ca126b2026d0
window_manager: 3a1844359a6295ab1e47659b1a777e36773cd6e8

10
packages/veilid_support/build.yaml Normal file
View File

@ -0,0 +1,10 @@
targets:
$default:
sources:
exclude:
- example/**
builders:
json_serializable:
options:
explicit_to_json: true
field_rename: snake

35
packages/veilid_support/example/integration_test/app_test.dart
View File

@ -1,16 +1,16 @@
@Timeout(Duration(seconds: 240))
library veilid_support_integration_test;
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter/foundation.dart';
import 'package:integration_test/integration_test.dart';
import 'package:test/test.dart';
import 'package:veilid_test/veilid_test.dart';
import 'fixtures/fixtures.dart';
import 'test_dht_log.dart';
import 'test_dht_record_pool.dart';
import 'test_dht_short_array.dart';
void main() {
final startTime = DateTime.now();
IntegrationTestWidgetsFlutterBinding.ensureInitialized();
final veilidFixture =
DefaultVeilidFixture(programName: 'veilid_support integration test');
@ -22,9 +22,13 @@ void main() {
tickerFixture: tickerFixture,
updateProcessorFixture: updateProcessorFixture);
group('Started Tests', () {
group(timeout: const Timeout(Duration(seconds: 240)), 'Started Tests', () {
setUpAll(veilidFixture.setUp);
tearDownAll(veilidFixture.tearDown);
tearDownAll(() {
final endTime = DateTime.now();
debugPrintSynchronously('Duration: ${endTime.difference(startTime)}');
});
group('Attached Tests', () {
setUpAll(veilidFixture.attach);
@ -51,11 +55,26 @@ void main() {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
for (final stride in [256, 64, 32, 16, 8, 4, 2, 1]) {
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create shortarray stride=$stride',
makeTestDHTShortArrayCreateDelete(stride: stride));
test('add shortarray stride=$stride',
makeTestDHTShortArrayAdd(stride: 256));
makeTestDHTShortArrayAdd(stride: stride));
}
});
group('DHTLog Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create log stride=$stride',
makeTestDHTLogCreateDelete(stride: stride));
test(
timeout: const Timeout(Duration(seconds: 480)),
'add/truncate log stride=$stride',
makeTestDHTLogAddTruncate(stride: stride),
);
}
});
});

13
packages/veilid_support/example/integration_test/fixtures/dht_record_pool_fixture.dart
View File

@ -1,6 +1,7 @@
import 'dart:async';
import 'package:async_tools/async_tools.dart';
import 'package:flutter/foundation.dart';
import 'package:veilid_support/veilid_support.dart';
import 'package:veilid_test/veilid_test.dart';
@ -12,9 +13,13 @@ class DHTRecordPoolFixture implements TickerFixtureTickable {
UpdateProcessorFixture updateProcessorFixture;
TickerFixture tickerFixture;
Future<void> setUp() async {
Future<void> setUp({bool purge = true}) async {
await _fixtureMutex.acquire();
await DHTRecordPool.init();
if (purge) {
await Veilid.instance.debug('record purge local');
await Veilid.instance.debug('record purge remote');
}
await DHTRecordPool.init(logger: debugPrintSynchronously);
tickerFixture.register(this);
}
@ -22,6 +27,10 @@ class DHTRecordPoolFixture implements TickerFixtureTickable {
assert(_fixtureMutex.isLocked, 'should not tearDown without setUp');
tickerFixture.unregister(this);
await DHTRecordPool.close();
final recordList = await Veilid.instance.debug('record list local');
debugPrintSynchronously('DHT Record List:\n$recordList');
_fixtureMutex.release();
}

126
packages/veilid_support/example/integration_test/test_dht_log.dart Normal file
View File

@ -0,0 +1,126 @@
import 'dart:convert';
import 'package:test/test.dart';
import 'package:veilid_support/veilid_support.dart';
Future<void> Function() makeTestDHTLogCreateDelete({required int stride}) =>
() async {
// Close before delete
{
final dlog = await DHTLog.create(
debugName: 'log_create_delete 1 stride $stride', stride: stride);
expect(await dlog.operate((r) async => r.length), isZero);
expect(dlog.isOpen, isTrue);
await dlog.close();
expect(dlog.isOpen, isFalse);
await dlog.delete();
// Operate should fail
await expectLater(() async => dlog.operate((r) async => r.length),
throwsA(isA<StateError>()));
}
// Close after delete
{
final dlog = await DHTLog.create(
debugName: 'log_create_delete 2 stride $stride', stride: stride);
await dlog.delete();
// Operate should still succeed because things aren't closed
expect(await dlog.operate((r) async => r.length), isZero);
await dlog.close();
// Operate should fail
await expectLater(() async => dlog.operate((r) async => r.length),
throwsA(isA<StateError>()));
}
// Close after delete multiple
// Okay to request delete multiple times before close
{
final dlog = await DHTLog.create(
debugName: 'log_create_delete 3 stride $stride', stride: stride);
await dlog.delete();
await dlog.delete();
// Operate should still succeed because things aren't closed
expect(await dlog.operate((r) async => r.length), isZero);
await dlog.close();
await expectLater(() async => dlog.close(), throwsA(isA<StateError>()));
// Operate should fail
await expectLater(() async => dlog.operate((r) async => r.length),
throwsA(isA<StateError>()));
}
};
Future<void> Function() makeTestDHTLogAddTruncate({required int stride}) =>
() async {
final dlog = await DHTLog.create(
debugName: 'log_add 1 stride $stride', stride: stride);
final dataset = Iterable<int>.generate(1000)
.map((n) => utf8.encode('elem $n'))
.toList();
print('adding\n');
{
final res = await dlog.operateAppend((w) async {
const chunk = 25;
for (var n = 0; n < dataset.length; n += chunk) {
print('$n-${n + chunk - 1} ');
final success =
await w.tryAppendItems(dataset.sublist(n, n + chunk));
expect(success, isTrue);
}
});
expect(res, isNull);
}
print('get all\n');
{
final dataset2 = await dlog.operate((r) async => r.getItemRange(0));
expect(dataset2, equals(dataset));
}
{
final dataset3 =
await dlog.operate((r) async => r.getItemRange(64, length: 128));
expect(dataset3, equals(dataset.sublist(64, 64 + 128)));
}
{
final dataset4 =
await dlog.operate((r) async => r.getItemRange(0, length: 1000));
expect(dataset4, equals(dataset.sublist(0, 1000)));
}
{
final dataset5 =
await dlog.operate((r) async => r.getItemRange(500, length: 499));
expect(dataset5, equals(dataset.sublist(500, 999)));
}
print('truncate\n');
{
await dlog.operateAppend((w) async => w.truncate(5));
}
{
final dataset6 = await dlog
.operate((r) async => r.getItemRange(500 - 5, length: 499));
expect(dataset6, equals(dataset.sublist(500, 999)));
}
print('truncate 2\n');
{
await dlog.operateAppend((w) async => w.truncate(251));
}
{
final dataset7 = await dlog
.operate((r) async => r.getItemRange(500 - 256, length: 499));
expect(dataset7, equals(dataset.sublist(500, 999)));
}
print('clear\n');
{
await dlog.operateAppend((w) async => w.clear());
}
print('get all\n');
{
final dataset8 = await dlog.operate((r) async => r.getItemRange(0));
expect(dataset8, isEmpty);
}
print('delete and close\n');
await dlog.delete();
await dlog.close();
};

68
packages/veilid_support/example/integration_test/test_dht_record_pool.dart
View File

@ -1,7 +1,7 @@
import 'dart:convert';
import 'package:flutter/foundation.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:test/test.dart';
import 'package:veilid_support/veilid_support.dart';
Future<void> testDHTRecordPoolCreate() async {
@ -48,7 +48,7 @@ Future<void> testDHTRecordCreateDelete() async {
// Set should succeed still
await rec3.tryWriteBytes(utf8.encode('test'));
await rec3.close();
await rec3.close();
await expectLater(() async => rec3.close(), throwsA(isA<StateError>()));
// Set should fail
await expectLater(() async => rec3.tryWriteBytes(utf8.encode('test')),
throwsA(isA<VeilidAPIException>()));
@ -84,7 +84,7 @@ Future<void> testDHTRecordScopes() async {
} on Exception {
assert(false, 'should not throw');
}
await rec2.close();
await expectLater(() async => rec2.close(), throwsA(isA<StateError>()));
await pool.deleteRecord(rec2.key);
}
@ -115,6 +115,7 @@ Future<void> testDHTRecordGetSet() async {
final val = await rec.get();
await pool.deleteRecord(rec.key);
expect(val, isNull);
await rec.close();
}
// Test set then get
@ -125,6 +126,7 @@ Future<void> testDHTRecordGetSet() async {
// Invalid subkey should throw
await expectLater(
() async => rec2.get(subkey: 1), throwsA(isA<VeilidAPIException>()));
await rec2.close();
await pool.deleteRecord(rec2.key);
}
@ -151,7 +153,6 @@ Future<void> testDHTRecordDeepCreateDelete() async {
// Make root record
final recroot = await pool.createRecord(debugName: 'test_deep_create_delete');
for (var d = 0; d < numIterations; d++) {
// Make child set 1
var parent = recroot;
final children = <DHTRecord>[];
@ -162,6 +163,19 @@ Future<void> testDHTRecordDeepCreateDelete() async {
parent = child;
}
// Should mark for deletion
expect(await pool.deleteRecord(recroot.key), isFalse);
// Root should still be valid
expect(await pool.isValidRecordKey(recroot.key), isTrue);
// Close root record
await recroot.close();
// Root should still be valid because children still exist
expect(await pool.isValidRecordKey(recroot.key), isTrue);
for (var d = 0; d < numIterations; d++) {
// Make child set 2
final children2 = <DHTRecord>[];
parent = recroot;
@ -171,31 +185,31 @@ Future<void> testDHTRecordDeepCreateDelete() async {
children2.add(child);
parent = child;
}
// Should fail to delete root
await expectLater(
() async => pool.deleteRecord(recroot.key), throwsA(isA<StateError>()));
// Delete child set 2 in reverse order
for (var n = numChildren - 1; n >= 0; n--) {
expect(await pool.deleteRecord(children2[n].key), isFalse);
}
// Root should still be there
expect(await pool.isValidRecordKey(recroot.key), isTrue);
// Close child set 2
await children2.map((c) => c.close()).wait;
// All child set 2 should be invalid
for (final c2 in children2) {
// Children should be invalid and deleted now
expect(await pool.isValidRecordKey(c2.key), isFalse);
}
// Root should still be valid
expect(await pool.isValidRecordKey(recroot.key), isTrue);
}
// Close child set 1
await children.map((c) => c.close()).wait;
// Delete child set 1 in reverse order
for (var n = numChildren - 1; n >= 0; n--) {
await pool.deleteRecord(children[n].key);
}
// Should fail to delete root
await expectLater(
() async => pool.deleteRecord(recroot.key), throwsA(isA<StateError>()));
// Close child set 1
await children2.map((c) => c.close()).wait;
// Delete child set 2 in reverse order
for (var n = numChildren - 1; n >= 0; n--) {
await pool.deleteRecord(children2[n].key);
}
}
// Should be able to delete root now
await pool.deleteRecord(recroot.key);
// Root should have gone away
expect(await pool.isValidRecordKey(recroot.key), isFalse);
}

68
packages/veilid_support/example/integration_test/test_dht_short_array.dart
View File

@ -1,6 +1,6 @@
import 'dart:convert';
import 'package:flutter_test/flutter_test.dart';
import 'package:test/test.dart';
import 'package:veilid_support/veilid_support.dart';
Future<void> Function() makeTestDHTShortArrayCreateDelete(
@ -43,7 +43,7 @@ Future<void> Function() makeTestDHTShortArrayCreateDelete(
// Operate should still succeed because things aren't closed
expect(await arr.operate((r) async => r.length), isZero);
await arr.close();
await arr.close();
await expectLater(() async => arr.close(), throwsA(isA<StateError>()));
// Operate should fail
await expectLater(() async => arr.operate((r) async => r.length),
throwsA(isA<StateError>()));
@ -52,8 +52,6 @@ Future<void> Function() makeTestDHTShortArrayCreateDelete(
Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
() async {
final startTime = DateTime.now();
final arr = await DHTShortArray.create(
debugName: 'sa_add 1 stride $stride', stride: stride);
@ -61,41 +59,77 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
.map((n) => utf8.encode('elem $n'))
.toList();
print('adding\n');
print('adding singles\n');
{
final (res, ok) = await arr.operateWrite((w) async {
for (var n = 0; n < dataset.length; n++) {
final res = await arr.operateWrite((w) async {
for (var n = 4; n < 8; n++) {
print('$n ');
final success = await w.tryAddItem(dataset[n]);
expect(success, isTrue);
}
});
expect(res, isNull);
expect(ok, isTrue);
}
print('adding batch\n');
{
final res = await arr.operateWrite((w) async {
print('${dataset.length ~/ 2}-${dataset.length}');
final success = await w.tryAddItems(
dataset.sublist(dataset.length ~/ 2, dataset.length));
expect(success, isTrue);
});
expect(res, isNull);
}
print('inserting singles\n');
{
final res = await arr.operateWrite((w) async {
for (var n = 0; n < 4; n++) {
print('$n ');
final success = await w.tryInsertItem(n, dataset[n]);
expect(success, isTrue);
}
});
expect(res, isNull);
}
print('inserting batch\n');
{
final res = await arr.operateWrite((w) async {
print('8-${dataset.length ~/ 2}');
final success = await w.tryInsertItems(
8, dataset.sublist(8, dataset.length ~/ 2));
expect(success, isTrue);
});
expect(res, isNull);
}
//print('get all\n');
{
final dataset2 = await arr.operate((r) async => r.getAllItems());
final dataset2 = await arr.operate((r) async => r.getItemRange(0));
expect(dataset2, equals(dataset));
}
{
final dataset3 =
await arr.operate((r) async => r.getItemRange(64, length: 128));
expect(dataset3, equals(dataset.sublist(64, 64 + 128)));
}
//print('clear\n');
{
final (res, ok) = await arr.operateWrite((w) async => w.tryClear());
expect(res, isTrue);
expect(ok, isTrue);
await arr.operateWriteEventual((w) async {
await w.clear();
return true;
});
}
//print('get all\n');
{
final dataset3 = await arr.operate((r) async => r.getAllItems());
expect(dataset3, isEmpty);
final dataset4 = await arr.operate((r) async => r.getItemRange(0));
expect(dataset4, isEmpty);
}
await arr.delete();
await arr.close();
final endTime = DateTime.now();
print('Duration: ${endTime.difference(startTime)}');
};

292
packages/veilid_support/example/pubspec.lock
View File

@ -1,6 +1,30 @@
# Generated by pub
# See https://dart.dev/tools/pub/glossary#lockfile
packages:
_fe_analyzer_shared:
dependency: transitive
description:
name: _fe_analyzer_shared
sha256: "0b2f2bd91ba804e53a61d757b986f89f1f9eaed5b11e4b2f5a2468d86d6c9fc7"
url: "https://pub.dev"
source: hosted
version: "67.0.0"
analyzer:
dependency: transitive
description:
name: analyzer
sha256: "37577842a27e4338429a1cbc32679d508836510b056f1eedf0c8d20e39c1383d"
url: "https://pub.dev"
source: hosted
version: "6.4.1"
args:
dependency: transitive
description:
name: args
sha256: "7cf60b9f0cc88203c5a190b4cd62a99feea42759a7fa695010eb5de1c0b2252a"
url: "https://pub.dev"
source: hosted
version: "2.5.0"
async:
dependency: transitive
description:
@ -81,6 +105,30 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.18.0"
convert:
dependency: transitive
description:
name: convert
sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592"
url: "https://pub.dev"
source: hosted
version: "3.1.1"
coverage:
dependency: transitive
description:
name: coverage
sha256: "3945034e86ea203af7a056d98e98e42a5518fff200d6e8e6647e1886b07e936e"
url: "https://pub.dev"
source: hosted
version: "1.8.0"
crypto:
dependency: transitive
description:
name: crypto
sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab
url: "https://pub.dev"
source: hosted
version: "3.0.3"
cupertino_icons:
dependency: "direct main"
description:
@ -109,10 +157,10 @@ packages:
dependency: transitive
description:
name: fast_immutable_collections
sha256: "38fbc50df5b219dcfb83ebbc3275ec09872530ca1153858fc56fceadb310d037"
sha256: "533806a7f0c624c2e479d05d3fdce4c87109a7cd0db39b8cc3830d3a2e8dedc7"
url: "https://pub.dev"
source: hosted
version: "10.2.2"
version: "10.2.3"
ffi:
dependency: transitive
description:
@ -148,7 +196,7 @@ packages:
source: sdk
version: "0.0.0"
flutter_test:
dependency: "direct dev"
dependency: transitive
description: flutter
source: sdk
version: "0.0.0"
@ -165,11 +213,27 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.4.1"
frontend_server_client:
dependency: transitive
description:
name: frontend_server_client
sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694
url: "https://pub.dev"
source: hosted
version: "4.0.0"
fuchsia_remote_debug_protocol:
dependency: transitive
description: flutter
source: sdk
version: "0.0.0"
glob:
dependency: transitive
description:
name: glob
sha256: "0e7014b3b7d4dac1ca4d6114f82bf1782ee86745b9b42a92c9289c23d8a0ab63"
url: "https://pub.dev"
source: hosted
version: "2.1.2"
globbing:
dependency: transitive
description:
@ -178,11 +242,43 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.0"
http_multi_server:
dependency: transitive
description:
name: http_multi_server
sha256: "97486f20f9c2f7be8f514851703d0119c3596d14ea63227af6f7a481ef2b2f8b"
url: "https://pub.dev"
source: hosted
version: "3.2.1"
http_parser:
dependency: transitive
description:
name: http_parser
sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b"
url: "https://pub.dev"
source: hosted
version: "4.0.2"
integration_test:
dependency: "direct dev"
description: flutter
source: sdk
version: "0.0.0"
io:
dependency: transitive
description:
name: io
sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e"
url: "https://pub.dev"
source: hosted
version: "1.0.4"
js:
dependency: transitive
description:
name: js
sha256: c1b2e9b5ea78c45e1a0788d29606ba27dc5f71f019f32ca5140f61ef071838cf
url: "https://pub.dev"
source: hosted
version: "0.7.1"
json_annotation:
dependency: transitive
description:
@ -195,26 +291,26 @@ packages:
dependency: transitive
description:
name: leak_tracker
sha256: "78eb209deea09858f5269f5a5b02be4049535f568c07b275096836f01ea323fa"
sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a"
url: "https://pub.dev"
source: hosted
version: "10.0.0"
version: "10.0.4"
leak_tracker_flutter_testing:
dependency: transitive
description:
name: leak_tracker_flutter_testing
sha256: b46c5e37c19120a8a01918cfaf293547f47269f7cb4b0058f21531c2465d6ef0
sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8"
url: "https://pub.dev"
source: hosted
version: "2.0.1"
version: "3.0.3"
leak_tracker_testing:
dependency: transitive
description:
name: leak_tracker_testing
sha256: a597f72a664dbd293f3bfc51f9ba69816f84dcd403cdac7066cb3f6003f3ab47
sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3"
url: "https://pub.dev"
source: hosted
version: "2.0.1"
version: "3.0.1"
lint_hard:
dependency: "direct dev"
description:
@ -223,6 +319,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "4.0.0"
logging:
dependency: transitive
description:
name: logging
sha256: "623a88c9594aa774443aa3eb2d41807a48486b5613e67599fb4c41c0ad47c340"
url: "https://pub.dev"
source: hosted
version: "1.2.0"
loggy:
dependency: transitive
description:
@ -251,10 +355,34 @@ packages:
dependency: transitive
description:
name: meta
sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04
sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136"
url: "https://pub.dev"
source: hosted
version: "1.11.0"
version: "1.12.0"
mime:
dependency: transitive
description:
name: mime
sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2"
url: "https://pub.dev"
source: hosted
version: "1.0.5"
node_preamble:
dependency: transitive
description:
name: node_preamble
sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db"
url: "https://pub.dev"
source: hosted
version: "2.0.2"
package_config:
dependency: transitive
description:
name: package_config
sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd"
url: "https://pub.dev"
source: hosted
version: "2.1.0"
path:
dependency: transitive
description:
@ -327,6 +455,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.1.8"
pool:
dependency: transitive
description:
name: pool
sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a"
url: "https://pub.dev"
source: hosted
version: "1.5.1"
process:
dependency: transitive
description:
@ -343,11 +479,67 @@ packages:
url: "https://pub.dev"
source: hosted
version: "3.1.0"
pub_semver:
dependency: transitive
description:
name: pub_semver
sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c"
url: "https://pub.dev"
source: hosted
version: "2.1.4"
shelf:
dependency: transitive
description:
name: shelf
sha256: ad29c505aee705f41a4d8963641f91ac4cee3c8fad5947e033390a7bd8180fa4
url: "https://pub.dev"
source: hosted
version: "1.4.1"
shelf_packages_handler:
dependency: transitive
description:
name: shelf_packages_handler
sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e"
url: "https://pub.dev"
source: hosted
version: "3.0.2"
shelf_static:
dependency: transitive
description:
name: shelf_static
sha256: a41d3f53c4adf0f57480578c1d61d90342cd617de7fc8077b1304643c2d85c1e
url: "https://pub.dev"
source: hosted
version: "1.1.2"
shelf_web_socket:
dependency: transitive
description:
name: shelf_web_socket
sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1"
url: "https://pub.dev"
source: hosted
version: "1.0.4"
sky_engine:
dependency: transitive
description: flutter
source: sdk
version: "0.0.99"
source_map_stack_trace:
dependency: transitive
description:
name: source_map_stack_trace
sha256: "84cf769ad83aa6bb61e0aa5a18e53aea683395f196a6f39c4c881fb90ed4f7ae"
url: "https://pub.dev"
source: hosted
version: "2.1.1"
source_maps:
dependency: transitive
description:
name: source_maps
sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703"
url: "https://pub.dev"
source: hosted
version: "0.10.12"
source_span:
dependency: transitive
description:
@ -412,14 +604,38 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.2.1"
test:
dependency: "direct dev"
description:
name: test
sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073"
url: "https://pub.dev"
source: hosted
version: "1.25.2"
test_api:
dependency: transitive
description:
name: test_api
sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b"
sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f"
url: "https://pub.dev"
source: hosted
version: "0.6.1"
version: "0.7.0"
test_core:
dependency: transitive
description:
name: test_core
sha256: "2bc4b4ecddd75309300d8096f781c0e3280ca1ef85beda558d33fcbedc2eead4"
url: "https://pub.dev"
source: hosted
version: "0.6.0"
typed_data:
dependency: transitive
description:
name: typed_data
sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c
url: "https://pub.dev"
source: hosted
version: "1.3.2"
vector_math:
dependency: transitive
description:
@ -453,10 +669,34 @@ packages:
dependency: transitive
description:
name: vm_service
sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957
sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec"
url: "https://pub.dev"
source: hosted
version: "13.0.0"
version: "14.2.1"
watcher:
dependency: transitive
description:
name: watcher
sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8"
url: "https://pub.dev"
source: hosted
version: "1.1.0"
web:
dependency: transitive
description:
name: web
sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27"
url: "https://pub.dev"
source: hosted
version: "0.5.1"
web_socket_channel:
dependency: transitive
description:
name: web_socket_channel
sha256: "58c6666b342a38816b2e7e50ed0f1e261959630becd4c879c4f26bfa14aa5a42"
url: "https://pub.dev"
source: hosted
version: "2.4.5"
webdriver:
dependency: transitive
description:
@ -465,14 +705,22 @@ packages:
url: "https://pub.dev"
source: hosted
version: "3.0.3"
webkit_inspection_protocol:
dependency: transitive
description:
name: webkit_inspection_protocol
sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572"
url: "https://pub.dev"
source: hosted
version: "1.2.1"
win32:
dependency: transitive
description:
name: win32
sha256: "0eaf06e3446824099858367950a813472af675116bf63f008a4c2a75ae13e9cb"
sha256: a79dbe579cb51ecd6d30b17e0cae4e0ea15e2c0e66f69ad4198f22a6789e94f4
url: "https://pub.dev"
source: hosted
version: "5.5.0"
version: "5.5.1"
xdg_directories:
dependency: transitive
description:
@ -481,6 +729,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.4"
yaml:
dependency: transitive
description:
name: yaml
sha256: "75769501ea3489fca56601ff33454fe45507ea3bfb014161abc3b43ae25989d5"
url: "https://pub.dev"
source: hosted
version: "3.1.2"
sdks:
dart: ">=3.3.4 <4.0.0"
dart: ">=3.4.0 <4.0.0"
flutter: ">=3.19.1"

3
packages/veilid_support/example/pubspec.yaml
View File

@ -15,11 +15,10 @@ dependencies:
dev_dependencies:
async_tools: ^0.1.1
flutter_test:
sdk: flutter
integration_test:
sdk: flutter
lint_hard: ^4.0.0
test: ^1.25.2
veilid_test:
path: ../../../../veilid/veilid-flutter/packages/veilid_test

2
packages/veilid_support/lib/dht_support/dht_support.dart
View File

@ -2,5 +2,7 @@
library dht_support;
export 'src/dht_log/barrel.dart';
export 'src/dht_record/barrel.dart';
export 'src/dht_short_array/barrel.dart';
export 'src/interfaces/interfaces.dart';

26
packages/veilid_support/lib/dht_support/proto/dht.proto
View File

@ -23,6 +23,18 @@ message DHTData {
uint32 size = 4;
}
// DHTLog - represents a ring buffer of many elements with append/truncate semantics
// Header in subkey 0 of first key follows this structure
message DHTLog {
// Position of the start of the log (oldest items)
uint32 head = 1;
// Position of the end of the log (newest items)
uint32 tail = 2;
// Stride of each segment of the dhtlog
uint32 stride = 3;
}
// DHTShortArray - represents a re-orderable collection of up to 256 individual elements
// Header in subkey 0 of first key follows this structure
//
@ -50,20 +62,6 @@ message DHTShortArray {
// calculated through iteration
}
// DHTLog - represents a long ring buffer of elements utilizing a multi-level
// indirection table of DHTShortArrays.
message DHTLog {
// Keys to concatenate
repeated veilid.TypedKey keys = 1;
// Back link to another DHTLog further back
veilid.TypedKey back = 2;
// Count of subkeys in all keys in this DHTLog
repeated uint32 subkey_counts = 3;
// Total count of subkeys in all keys in this DHTLog including all backlogs
uint32 total_subkeys = 4;
}
// DataReference
// Pointer to data somewhere in Veilid
// Abstraction over DHTData and BlockStore

2
packages/veilid_support/lib/dht_support/src/dht_log/barrel.dart Normal file
View File

@ -0,0 +1,2 @@
export 'dht_log.dart';
export 'dht_log_cubit.dart';

312
packages/veilid_support/lib/dht_support/src/dht_log/dht_log.dart Normal file
View File

@ -0,0 +1,312 @@
import 'dart:async';
import 'dart:math';
import 'dart:typed_data';
import 'package:async_tools/async_tools.dart';
import 'package:collection/collection.dart';
import 'package:equatable/equatable.dart';
import 'package:meta/meta.dart';
import '../../../veilid_support.dart';
import '../../proto/proto.dart' as proto;
import '../interfaces/dht_append_truncate.dart';
part 'dht_log_spine.dart';
part 'dht_log_read.dart';
part 'dht_log_append.dart';
///////////////////////////////////////////////////////////////////////
@immutable
class DHTLogUpdate extends Equatable {
const DHTLogUpdate(
{required this.headDelta, required this.tailDelta, required this.length})
: assert(headDelta >= 0, 'should never have negative head delta'),
assert(tailDelta >= 0, 'should never have negative tail delta'),
assert(length >= 0, 'should never have negative length');
final int headDelta;
final int tailDelta;
final int length;
@override
List<Object?> get props => [headDelta, tailDelta, length];
}
/// DHTLog is a ring-buffer queue like data structure with the following
/// operations:
/// * Add elements to the tail
/// * Remove elements from the head
/// The structure has a 'spine' record that acts as an indirection table of
/// DHTShortArray record pointers spread over its subkeys.
/// Subkey 0 of the DHTLog is a head subkey that contains housekeeping data:
/// * The head and tail position of the log
/// - subkeyIdx = pos / recordsPerSubkey
/// - recordIdx = pos % recordsPerSubkey
class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
////////////////////////////////////////////////////////////////
// Constructors
DHTLog._({required _DHTLogSpine spine})
: _spine = spine,
_openCount = 1 {
_spine.onUpdatedSpine = (update) {
_watchController?.sink.add(update);
};
}
/// Create a DHTLog
static Future<DHTLog> create(
{required String debugName,
int stride = DHTShortArray.maxElements,
VeilidRoutingContext? routingContext,
TypedKey? parent,
DHTRecordCrypto? crypto,
KeyPair? writer}) async {
assert(stride <= DHTShortArray.maxElements, 'stride too long');
final pool = DHTRecordPool.instance;
late final DHTRecord spineRecord;
if (writer != null) {
final schema = DHTSchema.smpl(
oCnt: 0,
members: [DHTSchemaMember(mKey: writer.key, mCnt: spineSubkeys + 1)]);
spineRecord = await pool.createRecord(
debugName: debugName,
parent: parent,
routingContext: routingContext,
schema: schema,
crypto: crypto,
writer: writer);
} else {
const schema = DHTSchema.dflt(oCnt: spineSubkeys + 1);
spineRecord = await pool.createRecord(
debugName: debugName,
parent: parent,
routingContext: routingContext,
schema: schema,
crypto: crypto);
}
try {
final spine = await _DHTLogSpine.create(
spineRecord: spineRecord, segmentStride: stride);
return DHTLog._(spine: spine);
} on Exception catch (_) {
await spineRecord.close();
await spineRecord.delete();
rethrow;
}
}
static Future<DHTLog> openRead(TypedKey logRecordKey,
{required String debugName,
VeilidRoutingContext? routingContext,
TypedKey? parent,
DHTRecordCrypto? crypto}) async {
final spineRecord = await DHTRecordPool.instance.openRecordRead(
logRecordKey,
debugName: debugName,
parent: parent,
routingContext: routingContext,
crypto: crypto);
try {
final spine = await _DHTLogSpine.load(spineRecord: spineRecord);
final dhtLog = DHTLog._(spine: spine);
return dhtLog;
} on Exception catch (_) {
await spineRecord.close();
rethrow;
}
}
static Future<DHTLog> openWrite(
TypedKey logRecordKey,
KeyPair writer, {
required String debugName,
VeilidRoutingContext? routingContext,
TypedKey? parent,
DHTRecordCrypto? crypto,
}) async {
final spineRecord = await DHTRecordPool.instance.openRecordWrite(
logRecordKey, writer,
debugName: debugName,
parent: parent,
routingContext: routingContext,
crypto: crypto);
try {
final spine = await _DHTLogSpine.load(spineRecord: spineRecord);
final dhtLog = DHTLog._(spine: spine);
return dhtLog;
} on Exception catch (_) {
await spineRecord.close();
rethrow;
}
}
static Future<DHTLog> openOwned(
OwnedDHTRecordPointer ownedLogRecordPointer, {
required String debugName,
required TypedKey parent,
VeilidRoutingContext? routingContext,
DHTRecordCrypto? crypto,
}) =>
openWrite(
ownedLogRecordPointer.recordKey,
ownedLogRecordPointer.owner,
debugName: debugName,
routingContext: routingContext,
parent: parent,
crypto: crypto,
);
////////////////////////////////////////////////////////////////////////////
// DHTCloseable
/// Check if the DHTLog is open
@override
bool get isOpen => _openCount > 0;
/// The type of the openable scope
@override
FutureOr<DHTLog> scoped() => this;
/// Add a reference to this log
@override
Future<DHTLog> ref() async => _mutex.protect(() async {
_openCount++;
return this;
});
/// Free all resources for the DHTLog
@override
Future<void> close() async => _mutex.protect(() async {
if (_openCount == 0) {
throw StateError('already closed');
}
_openCount--;
if (_openCount != 0) {
return;
}
await _watchController?.close();
_watchController = null;
await _spine.close();
});
/// Free all resources for the DHTLog and delete it from the DHT
/// Will wait until the short array is closed to delete it
@override
Future<void> delete() async {
await _spine.delete();
}
////////////////////////////////////////////////////////////////////////////
// Public API
/// Get the record key for this log
TypedKey get recordKey => _spine.recordKey;
/// Get the record pointer foir this log
OwnedDHTRecordPointer get recordPointer => _spine.recordPointer;
/// Runs a closure allowing read-only access to the log
Future<T?> operate<T>(Future<T?> Function(DHTRandomRead) closure) async {
if (!isOpen) {
throw StateError('log is not open"');
}
return _spine.operate((spine) async {
final reader = _DHTLogRead._(spine);
return closure(reader);
});
}
/// Runs a closure allowing append/truncate access to the log
/// Makes only one attempt to consistently write the changes to the DHT
/// Returns result of the closure if the write could be performed
/// Throws DHTOperateException if the write could not be performed
/// at this time
Future<T> operateAppend<T>(
Future<T> Function(DHTAppendTruncateRandomRead) closure) async {
if (!isOpen) {
throw StateError('log is not open"');
}
return _spine.operateAppend((spine) async {
final writer = _DHTLogAppend._(spine);
return closure(writer);
});
}
/// Runs a closure allowing append/truncate access to the log
/// Will execute the closure multiple times if a consistent write to the DHT
/// is not achieved. Timeout if specified will be thrown as a
/// TimeoutException. The closure should return true if its changes also
/// succeeded, returning false will trigger another eventual consistency
/// attempt.
Future<void> operateAppendEventual(
Future<bool> Function(DHTAppendTruncateRandomRead) closure,
{Duration? timeout}) async {
if (!isOpen) {
throw StateError('log is not open"');
}
return _spine.operateAppendEventual((spine) async {
final writer = _DHTLogAppend._(spine);
return closure(writer);
}, timeout: timeout);
}
/// Listen to and any all changes to the structure of this log
/// regardless of where the changes are coming from
Future<StreamSubscription<void>> listen(
void Function(DHTLogUpdate) onChanged,
) {
if (!isOpen) {
throw StateError('log is not open"');
}
return _listenMutex.protect(() async {
// If don't have a controller yet, set it up
if (_watchController == null) {
// Set up watch requirements
_watchController =
StreamController<DHTLogUpdate>.broadcast(onCancel: () {
// If there are no more listeners then we can get
// rid of the controller and drop our subscriptions
unawaited(_listenMutex.protect(() async {
// Cancel watches of head record
await _spine.cancelWatch();
_watchController = null;
}));
});
// Start watching head subkey of the spine
await _spine.watch();
}
// Return subscription
return _watchController!.stream.listen((upd) => onChanged(upd));
});
}
////////////////////////////////////////////////////////////////
// Fields
// 56 subkeys * 512 segments * 36 bytes per typedkey =
// 1032192 bytes per record
// 512*36 = 18432 bytes per subkey
// 28672 shortarrays * 256 elements = 7340032 elements
static const spineSubkeys = 56;
static const segmentsPerSubkey = 512;
// Internal representation refreshed from spine record
final _DHTLogSpine _spine;
// Openable
int _openCount;
final _mutex = Mutex();
// Watch mutex to ensure we keep the representation valid
final Mutex _listenMutex = Mutex();
// Stream of external changes
StreamController<DHTLogUpdate>? _watchController;
}

94
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_append.dart Normal file
View File

@ -0,0 +1,94 @@
part of 'dht_log.dart';
////////////////////////////////////////////////////////////////////////////
// Append/truncate implementation
class _DHTLogAppend extends _DHTLogRead implements DHTAppendTruncateRandomRead {
_DHTLogAppend._(super.spine) : super._();
@override
Future<bool> tryAppendItem(Uint8List value) async {
// Allocate empty index at the end of the list
final insertPos = _spine.length;
_spine.allocateTail(1);
final lookup = await _spine.lookupPosition(insertPos);
if (lookup == null) {
throw StateError("can't write to dht log");
}
// Write item to the segment
return lookup.scope((sa) => sa.operateWrite((write) async {
// If this a new segment, then clear it in case we have wrapped around
if (lookup.pos == 0) {
await write.clear();
} else if (lookup.pos != write.length) {
// We should always be appending at the length
throw StateError('appending should be at the end');
}
return write.tryAddItem(value);
}));
}
@override
Future<bool> tryAppendItems(List<Uint8List> values) async {
// Allocate empty index at the end of the list
final insertPos = _spine.length;
_spine.allocateTail(values.length);
// Look up the first position and shortarray
final dws = DelayedWaitSet<void>();
var success = true;
for (var valueIdx = 0; valueIdx < values.length;) {
final remaining = values.length - valueIdx;
final lookup = await _spine.lookupPosition(insertPos + valueIdx);
if (lookup == null) {
throw StateError("can't write to dht log");
}
final sacount = min(remaining, DHTShortArray.maxElements - lookup.pos);
final sublistValues = values.sublist(valueIdx, valueIdx + sacount);
dws.add(() async {
final ok = await lookup.scope((sa) => sa.operateWrite((write) async {
// If this a new segment, then clear it in
// case we have wrapped around
if (lookup.pos == 0) {
await write.clear();
} else if (lookup.pos != write.length) {
// We should always be appending at the length
throw StateError('appending should be at the end');
}
return write.tryAddItems(sublistValues);
}));
if (!ok) {
success = false;
}
});
valueIdx += sacount;
}
await dws();
return success;
}
@override
Future<void> truncate(int count) async {
count = min(count, _spine.length);
if (count == 0) {
return;
}
if (count < 0) {
throw StateError('can not remove negative items');
}
await _spine.releaseHead(count);
}
@override
Future<void> clear() async {
await _spine.releaseHead(_spine.length);
}
}

220
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_cubit.dart Normal file
View File

@ -0,0 +1,220 @@
import 'dart:async';
import 'package:async_tools/async_tools.dart';
import 'package:bloc/bloc.dart';
import 'package:bloc_advanced_tools/bloc_advanced_tools.dart';
import 'package:equatable/equatable.dart';
import 'package:fast_immutable_collections/fast_immutable_collections.dart';
import 'package:meta/meta.dart';
import '../../../veilid_support.dart';
import '../interfaces/dht_append_truncate.dart';
@immutable
class DHTLogElementState<T> extends Equatable {
const DHTLogElementState({required this.value, required this.isOffline});
final T value;
final bool isOffline;
@override
List<Object?> get props => [value, isOffline];
}
@immutable
class DHTLogStateData<T> extends Equatable {
const DHTLogStateData(
{required this.elements,
required this.tail,
required this.count,
required this.follow});
// The view of the elements in the dhtlog
// Span is from [tail-length, tail)
final IList<DHTLogElementState<T>> elements;
// One past the end of the last element
final int tail;
// The total number of elements to try to keep in 'elements'
final int count;
// If we should have the tail following the log
final bool follow;
@override
List<Object?> get props => [elements, tail, count, follow];
}
typedef DHTLogState<T> = AsyncValue<DHTLogStateData<T>>;
typedef DHTLogBusyState<T> = BlocBusyState<DHTLogState<T>>;
class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
with BlocBusyWrapper<DHTLogState<T>> {
DHTLogCubit({
required Future<DHTLog> Function() open,
required T Function(List<int> data) decodeElement,
}) : _decodeElement = decodeElement,
super(const BlocBusyState(AsyncValue.loading())) {
_initWait.add(() async {
// Open DHT record
_log = await open();
_wantsCloseRecord = true;
// Make initial state update
await _refreshNoWait();
_subscription = await _log.listen(_update);
});
}
// Set the tail position of the log for pagination.
// If tail is 0, the end of the log is used.
// If tail is negative, the position is subtracted from the current log
// length.
// If tail is positive, the position is absolute from the head of the log
// If follow is enabled, the tail offset will update when the log changes
Future<void> setWindow(
{int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
await _initWait();
if (tail != null) {
_tail = tail;
}
if (count != null) {
_count = count;
}
if (follow != null) {
_follow = follow;
}
await _refreshNoWait(forceRefresh: forceRefresh);
}
Future<void> refresh({bool forceRefresh = false}) async {
await _initWait();
await _refreshNoWait(forceRefresh: forceRefresh);
}
Future<void> _refreshNoWait({bool forceRefresh = false}) async =>
busy((emit) async => _refreshInner(emit, forceRefresh: forceRefresh));
Future<void> _refreshInner(void Function(AsyncValue<DHTLogStateData<T>>) emit,
{bool forceRefresh = false}) async {
final avElements = await _loadElements(_tail, _count);
final err = avElements.asError;
if (err != null) {
emit(AsyncValue.error(err.error, err.stackTrace));
return;
}
final loading = avElements.asLoading;
if (loading != null) {
emit(const AsyncValue.loading());
return;
}
final elements = avElements.asData!.value;
emit(AsyncValue.data(DHTLogStateData(
elements: elements, tail: _tail, count: _count, follow: _follow)));
}
Future<AsyncValue<IList<DHTLogElementState<T>>>> _loadElements(
int tail, int count,
{bool forceRefresh = false}) async {
try {
final allItems = await _log.operate((reader) async {
final length = reader.length;
final end = ((tail - 1) % length) + 1;
final start = (count < end) ? end - count : 0;
final offlinePositions = await reader.getOfflinePositions();
final allItems = (await reader.getItemRange(start,
length: end - start, forceRefresh: forceRefresh))
?.indexed
.map((x) => DHTLogElementState(
value: _decodeElement(x.$2),
isOffline: offlinePositions.contains(x.$1)))
.toIList();
return allItems;
});
if (allItems == null) {
return const AsyncValue.loading();
}
return AsyncValue.data(allItems);
} on Exception catch (e, st) {
return AsyncValue.error(e, st);
}
}
void _update(DHTLogUpdate upd) {
// Run at most one background update process
// Because this is async, we could get an update while we're
// still processing the last one. Only called after init future has run
// so we dont have to wait for that here.
// Accumulate head and tail deltas
_headDelta += upd.headDelta;
_tailDelta += upd.tailDelta;
_sspUpdate.busyUpdate<T, DHTLogState<T>>(busy, (emit) async {
// apply follow
if (_follow) {
if (_tail <= 0) {
// Negative tail is already following tail changes
} else {
// Positive tail is measured from the head, so apply deltas
_tail = (_tail + _tailDelta - _headDelta) % upd.length;
}
} else {
if (_tail <= 0) {
// Negative tail is following tail changes so apply deltas
var posTail = _tail + upd.length;
posTail = (posTail + _tailDelta - _headDelta) % upd.length;
_tail = posTail - upd.length;
} else {
// Positive tail is measured from head so not following tail
}
}
_headDelta = 0;
_tailDelta = 0;
await _refreshInner(emit);
});
}
@override
Future<void> close() async {
await _initWait();
await _subscription?.cancel();
_subscription = null;
if (_wantsCloseRecord) {
await _log.close();
}
await super.close();
}
Future<R?> operate<R>(Future<R?> Function(DHTRandomRead) closure) async {
await _initWait();
return _log.operate(closure);
}
Future<R> operateAppend<R>(
Future<R> Function(DHTAppendTruncateRandomRead) closure) async {
await _initWait();
return _log.operateAppend(closure);
}
Future<void> operateAppendEventual(
Future<bool> Function(DHTAppendTruncateRandomRead) closure,
{Duration? timeout}) async {
await _initWait();
return _log.operateAppendEventual(closure, timeout: timeout);
}
final WaitSet<void> _initWait = WaitSet();
late final DHTLog _log;
final T Function(List<int> data) _decodeElement;
StreamSubscription<void>? _subscription;
bool _wantsCloseRecord = false;
final _sspUpdate = SingleStatelessProcessor();
// Accumulated deltas since last update
var _headDelta = 0;
var _tailDelta = 0;
// Cubit window into the DHTLog
var _tail = 0;
var _count = DHTShortArray.maxElements;
var _follow = true;
}

103
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_read.dart Normal file
View File

@ -0,0 +1,103 @@
part of 'dht_log.dart';
////////////////////////////////////////////////////////////////////////////
// Reader-only implementation
class _DHTLogRead implements DHTRandomRead {
_DHTLogRead._(_DHTLogSpine spine) : _spine = spine;
@override
int get length => _spine.length;
@override
Future<Uint8List?> getItem(int pos, {bool forceRefresh = false}) async {
if (pos < 0 || pos >= length) {
throw IndexError.withLength(pos, length);
}
final lookup = await _spine.lookupPosition(pos);
if (lookup == null) {
return null;
}
return lookup.scope((sa) => sa.operate(
(read) => read.getItem(lookup.pos, forceRefresh: forceRefresh)));
}
(int, int) _clampStartLen(int start, int? len) {
len ??= _spine.length;
if (start < 0) {
throw IndexError.withLength(start, _spine.length);
}
if (start > _spine.length) {
throw IndexError.withLength(start, _spine.length);
}
if ((len + start) > _spine.length) {
len = _spine.length - start;
}
return (start, len);
}
@override
Future<List<Uint8List>?> getItemRange(int start,
{int? length, bool forceRefresh = false}) async {
final out = <Uint8List>[];
(start, length) = _clampStartLen(start, length);
final chunks = Iterable<int>.generate(length).slices(maxDHTConcurrency).map(
(chunk) => chunk
.map((pos) => getItem(pos + start, forceRefresh: forceRefresh)));
for (final chunk in chunks) {
final elems = await chunk.wait;
if (elems.contains(null)) {
return null;
}
out.addAll(elems.cast<Uint8List>());
}
return out;
}
@override
Future<Set<int>> getOfflinePositions() async {
final positionOffline = <int>{};
// Iterate positions backward from most recent
for (var pos = _spine.length - 1; pos >= 0; pos--) {
final lookup = await _spine.lookupPosition(pos);
if (lookup == null) {
throw StateError('Unable to look up position');
}
// Check each segment for offline positions
var foundOffline = false;
await lookup.scope((sa) => sa.operate((read) async {
final segmentOffline = await read.getOfflinePositions();
// For each shortarray segment go through their segment positions
// in reverse order and see if they are offline
for (var segmentPos = lookup.pos;
segmentPos >= 0 && pos >= 0;
segmentPos--, pos--) {
// If the position in the segment is offline, then
// mark the position in the log as offline
if (segmentOffline.contains(segmentPos)) {
positionOffline.add(pos);
foundOffline = true;
}
}
}));
// If we found nothing offline in this segment then we can stop
if (!foundOffline) {
break;
}
}
return positionOffline;
}
////////////////////////////////////////////////////////////////////////////
// Fields
final _DHTLogSpine _spine;
}

707
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_spine.dart Normal file
View File

@ -0,0 +1,707 @@
part of 'dht_log.dart';
class _DHTLogPosition extends DHTCloseable<_DHTLogPosition, DHTShortArray> {
_DHTLogPosition._({
required _DHTLogSpine dhtLogSpine,
required DHTShortArray shortArray,
required this.pos,
required int segmentNumber,
}) : _segmentShortArray = shortArray,
_dhtLogSpine = dhtLogSpine,
_segmentNumber = segmentNumber;
final int pos;
final _DHTLogSpine _dhtLogSpine;
final DHTShortArray _segmentShortArray;
var _openCount = 1;
final int _segmentNumber;
final Mutex _mutex = Mutex();
/// Check if the DHTLogPosition is open
@override
bool get isOpen => _openCount > 0;
/// The type of the openable scope
@override
FutureOr<DHTShortArray> scoped() => _segmentShortArray;
/// Add a reference to this log
@override
Future<_DHTLogPosition> ref() async => _mutex.protect(() async {
_openCount++;
return this;
});
/// Free all resources for the DHTLogPosition
@override
Future<void> close() async => _mutex.protect(() async {
if (_openCount == 0) {
throw StateError('already closed');
}
_openCount--;
if (_openCount != 0) {
return;
}
await _dhtLogSpine._segmentClosed(_segmentNumber);
});
}
class _OpenedSegment {
_OpenedSegment._({
required this.shortArray,
});
final DHTShortArray shortArray;
int openCount = 1;
}
class _DHTLogSegmentLookup extends Equatable {
const _DHTLogSegmentLookup({required this.subkey, required this.segment});
final int subkey;
final int segment;
@override
List<Object?> get props => [subkey, segment];
}
class _SubkeyData {
_SubkeyData({required this.subkey, required this.data});
int subkey;
Uint8List data;
bool changed = false;
}
class _DHTLogSpine {
_DHTLogSpine._(
{required DHTRecord spineRecord,
required int head,
required int tail,
required int stride})
: _spineRecord = spineRecord,
_head = head,
_tail = tail,
_segmentStride = stride,
_openedSegments = {},
_spineCache = [];
// Create a new spine record and push it to the network
static Future<_DHTLogSpine> create(
{required DHTRecord spineRecord, required int segmentStride}) async {
// Construct new spinehead
final spine = _DHTLogSpine._(
spineRecord: spineRecord, head: 0, tail: 0, stride: segmentStride);
// Write new spine head record to the network
await spine.operate((spine) async {
final success = await spine.writeSpineHead();
assert(success, 'false return should never happen on create');
});
return spine;
}
// Pull the latest or updated copy of the spine head record from the network
static Future<_DHTLogSpine> load({required DHTRecord spineRecord}) async {
// Get an updated spine head record copy if one exists
final spineHead = await spineRecord.getProtobuf(proto.DHTLog.fromBuffer,
subkey: 0, refreshMode: DHTRecordRefreshMode.network);
if (spineHead == null) {
throw StateError('spine head missing during refresh');
}
return _DHTLogSpine._(
spineRecord: spineRecord,
head: spineHead.head,
tail: spineHead.tail,
stride: spineHead.stride);
}
proto.DHTLog _toProto() {
assert(_spineMutex.isLocked, 'should be in mutex here');
final logHead = proto.DHTLog()
..head = _head
..tail = _tail
..stride = _segmentStride;
return logHead;
}
Future<void> close() async {
await _spineMutex.protect(() async {
if (!isOpen) {
return;
}
final futures = <Future<void>>[_spineRecord.close()];
for (final (_, sc) in _spineCache) {
futures.add(sc.close());
}
await Future.wait(futures);
assert(_openedSegments.isEmpty, 'should have closed all segments by now');
});
}
Future<void> delete() async {
await _spineMutex.protect(() async {
// Will deep delete all segment records as they are children
await _spineRecord.delete();
});
}
Future<T> operate<T>(Future<T> Function(_DHTLogSpine) closure) async =>
// ignore: prefer_expression_function_bodies
_spineMutex.protect(() async {
return closure(this);
});
Future<T> operateAppend<T>(Future<T> Function(_DHTLogSpine) closure) async =>
_spineMutex.protect(() async {
final oldHead = _head;
final oldTail = _tail;
try {
final out = await closure(this);
// Write head assuming it has been changed
if (!await writeSpineHead(old: (oldHead, oldTail))) {
// Failed to write head means head got overwritten so write should
// be considered failed
throw DHTExceptionTryAgain();
}
return out;
} on Exception {
// Exception means state needs to be reverted
_head = oldHead;
_tail = oldTail;
rethrow;
}
});
Future<void> operateAppendEventual(
Future<bool> Function(_DHTLogSpine) closure,
{Duration? timeout}) async {
final timeoutTs = timeout == null
? null
: Veilid.instance.now().offset(TimestampDuration.fromDuration(timeout));
await _spineMutex.protect(() async {
late int oldHead;
late int oldTail;
try {
// Iterate until we have a successful element and head write
do {
// Save off old values each pass of writeSpineHead because the head
// will have changed
oldHead = _head;
oldTail = _tail;
// Try to do the element write
while (true) {
if (timeoutTs != null) {
final now = Veilid.instance.now();
if (now >= timeoutTs) {
throw TimeoutException('timeout reached');
}
}
if (await closure(this)) {
break;
}
// Failed to write in closure resets state
_head = oldHead;
_tail = oldTail;
}
// Try to do the head write
} while (!await writeSpineHead(old: (oldHead, oldTail)));
} on Exception {
// Exception means state needs to be reverted
_head = oldHead;
_tail = oldTail;
rethrow;
}
});
}
/// Serialize and write out the current spine head subkey, possibly updating
/// it if a newer copy is available online. Returns true if the write was
/// successful
Future<bool> writeSpineHead({(int, int)? old}) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
final headBuffer = _toProto().writeToBuffer();
final existingData = await _spineRecord.tryWriteBytes(headBuffer);
if (existingData != null) {
// Head write failed, incorporate update
await _updateHead(proto.DHTLog.fromBuffer(existingData));
if (old != null) {
sendUpdate(old.$1, old.$2);
}
return false;
}
if (old != null) {
sendUpdate(old.$1, old.$2);
}
return true;
}
/// Send a spine update callback
void sendUpdate(int oldHead, int oldTail) {
final oldLength = _ringDistance(oldTail, oldHead);
if (oldHead != _head || oldTail != _tail || oldLength != length) {
onUpdatedSpine?.call(DHTLogUpdate(
headDelta: _ringDistance(_head, oldHead),
tailDelta: _ringDistance(_tail, oldTail),
length: length));
}
}
/// Validate a new spine head subkey that has come in from the network
Future<void> _updateHead(proto.DHTLog spineHead) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
_head = spineHead.head;
_tail = spineHead.tail;
}
/////////////////////////////////////////////////////////////////////////////
// Spine element management
static final Uint8List _emptySegmentKey =
Uint8List.fromList(List.filled(TypedKey.decodedLength<TypedKey>(), 0));
static Uint8List _makeEmptySubkey() => Uint8List.fromList(List.filled(
DHTLog.segmentsPerSubkey * TypedKey.decodedLength<TypedKey>(), 0));
static TypedKey? _getSegmentKey(Uint8List subkeyData, int segment) {
final decodedLength = TypedKey.decodedLength<TypedKey>();
final segmentKeyBytes = subkeyData.sublist(
decodedLength * segment, decodedLength * (segment + 1));
if (segmentKeyBytes.equals(_emptySegmentKey)) {
return null;
}
return TypedKey.fromBytes(segmentKeyBytes);
}
static void _setSegmentKey(
Uint8List subkeyData, int segment, TypedKey? segmentKey) {
final decodedLength = TypedKey.decodedLength<TypedKey>();
late final Uint8List segmentKeyBytes;
if (segmentKey == null) {
segmentKeyBytes = _emptySegmentKey;
} else {
segmentKeyBytes = segmentKey.decode();
}
subkeyData.setRange(decodedLength * segment, decodedLength * (segment + 1),
segmentKeyBytes);
}
Future<DHTShortArray> _openOrCreateSegmentInner(int segmentNumber) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
assert(_spineRecord.writer != null, 'should be writable');
// Lookup what subkey and segment subrange has this position's segment
// shortarray
final l = _lookupSegment(segmentNumber);
final subkey = l.subkey;
final segment = l.segment;
var subkeyData = await _spineRecord.get(subkey: subkey);
subkeyData ??= _makeEmptySubkey();
while (true) {
final segmentKey = _getSegmentKey(subkeyData!, segment);
if (segmentKey == null) {
// Create a shortarray segment
final segmentRec = await DHTShortArray.create(
debugName: '${_spineRecord.debugName}_spine_${subkey}_$segment',
stride: _segmentStride,
crypto: _spineRecord.crypto,
parent: _spineRecord.key,
routingContext: _spineRecord.routingContext,
writer: _spineRecord.writer,
);
var success = false;
try {
// Write it back to the spine record
_setSegmentKey(subkeyData, segment, segmentRec.recordKey);
subkeyData =
await _spineRecord.tryWriteBytes(subkeyData, subkey: subkey);
// If the write was successful then we're done
if (subkeyData == null) {
// Return it
success = true;
return segmentRec;
}
} finally {
if (!success) {
await segmentRec.close();
await segmentRec.delete();
}
}
} else {
// Open a shortarray segment
final segmentRec = await DHTShortArray.openWrite(
segmentKey,
_spineRecord.writer!,
debugName: '${_spineRecord.debugName}_spine_${subkey}_$segment',
crypto: _spineRecord.crypto,
parent: _spineRecord.key,
routingContext: _spineRecord.routingContext,
);
return segmentRec;
}
// Loop if we need to try again with the new data from the network
}
}
Future<DHTShortArray?> _openSegmentInner(int segmentNumber) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
// Lookup what subkey and segment subrange has this position's segment
// shortarray
final l = _lookupSegment(segmentNumber);
final subkey = l.subkey;
final segment = l.segment;
final subkeyData = await _spineRecord.get(subkey: subkey);
if (subkeyData == null) {
return null;
}
final segmentKey = _getSegmentKey(subkeyData, segment);
if (segmentKey == null) {
return null;
}
// Open a shortarray segment
final segmentRec = await DHTShortArray.openRead(
segmentKey,
debugName: '${_spineRecord.debugName}_spine_${subkey}_$segment',
crypto: _spineRecord.crypto,
parent: _spineRecord.key,
routingContext: _spineRecord.routingContext,
);
return segmentRec;
}
Future<DHTShortArray> _openOrCreateSegment(int segmentNumber) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
// See if we already have this in the cache
for (var i = 0; i < _spineCache.length; i++) {
if (_spineCache[i].$1 == segmentNumber) {
// Touch the element
final x = _spineCache.removeAt(i);
_spineCache.add(x);
// Return the shortarray for this position
return x.$2.ref();
}
}
// If we don't have it in the cache, get/create it and then cache a ref
final segment = await _openOrCreateSegmentInner(segmentNumber);
_spineCache.add((segmentNumber, await segment.ref()));
if (_spineCache.length > _spineCacheLength) {
// Trim the LRU cache
final (_, sa) = _spineCache.removeAt(0);
await sa.close();
}
return segment;
}
Future<DHTShortArray?> _openSegment(int segmentNumber) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
// See if we already have this in the cache
for (var i = 0; i < _spineCache.length; i++) {
if (_spineCache[i].$1 == segmentNumber) {
// Touch the element
final x = _spineCache.removeAt(i);
_spineCache.add(x);
// Return the shortarray for this position
return x.$2.ref();
}
}
// If we don't have it in the cache, get it and then cache it
final segment = await _openSegmentInner(segmentNumber);
if (segment == null) {
return null;
}
_spineCache.add((segmentNumber, await segment.ref()));
if (_spineCache.length > _spineCacheLength) {
// Trim the LRU cache
final (_, sa) = _spineCache.removeAt(0);
await sa.close();
}
return segment;
}
_DHTLogSegmentLookup _lookupSegment(int segmentNumber) {
assert(_spineMutex.isLocked, 'should be in mutex here');
if (segmentNumber < 0) {
throw IndexError.withLength(
segmentNumber, DHTLog.spineSubkeys * DHTLog.segmentsPerSubkey);
}
final subkey = segmentNumber ~/ DHTLog.segmentsPerSubkey;
if (subkey >= DHTLog.spineSubkeys) {
throw IndexError.withLength(
segmentNumber, DHTLog.spineSubkeys * DHTLog.segmentsPerSubkey);
}
final segment = segmentNumber % DHTLog.segmentsPerSubkey;
return _DHTLogSegmentLookup(subkey: subkey + 1, segment: segment);
}
///////////////////////////////////////////
// API for public interfaces
Future<_DHTLogPosition?> lookupPosition(int pos) async {
assert(_spineMutex.isLocked, 'should be locked');
return _spineCacheMutex.protect(() async {
// Check if our position is in bounds
final endPos = length;
if (pos < 0 || pos >= endPos) {
throw IndexError.withLength(pos, endPos);
}
// Calculate absolute position, ring-buffer style
final absolutePosition = (_head + pos) % _positionLimit;
// Determine the segment number and position within the segment
final segmentNumber = absolutePosition ~/ DHTShortArray.maxElements;
final segmentPos = absolutePosition % DHTShortArray.maxElements;
// Get the segment shortArray
final openedSegment = _openedSegments[segmentNumber];
late final DHTShortArray shortArray;
if (openedSegment != null) {
openedSegment.openCount++;
shortArray = openedSegment.shortArray;
} else {
final newShortArray = (_spineRecord.writer == null)
? await _openSegment(segmentNumber)
: await _openOrCreateSegment(segmentNumber);
if (newShortArray == null) {
return null;
}
_openedSegments[segmentNumber] =
_OpenedSegment._(shortArray: newShortArray);
shortArray = newShortArray;
}
return _DHTLogPosition._(
dhtLogSpine: this,
shortArray: shortArray,
pos: segmentPos,
segmentNumber: segmentNumber);
});
}
Future<void> _segmentClosed(int segmentNumber) async {
assert(_spineMutex.isLocked, 'should be locked');
await _spineCacheMutex.protect(() async {
final os = _openedSegments[segmentNumber]!;
os.openCount--;
if (os.openCount == 0) {
_openedSegments.remove(segmentNumber);
await os.shortArray.close();
}
});
}
void allocateTail(int count) {
assert(_spineMutex.isLocked, 'should be locked');
final currentLength = length;
if (count <= 0) {
throw StateError('count should be > 0');
}
if (currentLength + count >= _positionLimit) {
throw StateError('ring buffer overflow');
}
_tail = (_tail + count) % _positionLimit;
}
Future<void> releaseHead(int count) async {
assert(_spineMutex.isLocked, 'should be locked');
final currentLength = length;
if (count <= 0) {
throw StateError('count should be > 0');
}
if (count > currentLength) {
throw StateError('ring buffer underflow');
}
final oldHead = _head;
_head = (_head + count) % _positionLimit;
final newHead = _head;
await _purgeSegments(oldHead, newHead);
}
Future<void> _deleteSegmentsContiguous(int start, int end) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
DHTRecordPool.instance
.log('_deleteSegmentsContiguous: start=$start, end=$end');
final startSegmentNumber = start ~/ DHTShortArray.maxElements;
final startSegmentPos = start % DHTShortArray.maxElements;
final endSegmentNumber = end ~/ DHTShortArray.maxElements;
final endSegmentPos = end % DHTShortArray.maxElements;
final firstDeleteSegment =
(startSegmentPos == 0) ? startSegmentNumber : startSegmentNumber + 1;
final lastDeleteSegment =
(endSegmentPos == 0) ? endSegmentNumber - 1 : endSegmentNumber - 2;
_SubkeyData? lastSubkeyData;
for (var segmentNumber = firstDeleteSegment;
segmentNumber <= lastDeleteSegment;
segmentNumber++) {
// Lookup what subkey and segment subrange has this position's segment
// shortarray
final l = _lookupSegment(segmentNumber);
final subkey = l.subkey;
final segment = l.segment;
if (subkey != lastSubkeyData?.subkey) {
// Flush subkey writes
if (lastSubkeyData != null && lastSubkeyData.changed) {
await _spineRecord.eventualWriteBytes(lastSubkeyData.data,
subkey: lastSubkeyData.subkey);
}
// Get next subkey if available locally
final data = await _spineRecord.get(
subkey: subkey, refreshMode: DHTRecordRefreshMode.local);
if (data != null) {
lastSubkeyData = _SubkeyData(subkey: subkey, data: data);
} else {
lastSubkeyData = null;
// If the subkey was not available locally we can go to the
// last segment number at the end of this subkey
segmentNumber = ((subkey + 1) * DHTLog.segmentsPerSubkey) - 1;
}
}
if (lastSubkeyData != null) {
final segmentKey = _getSegmentKey(lastSubkeyData.data, segment);
if (segmentKey != null) {
await DHTRecordPool.instance.deleteRecord(segmentKey);
_setSegmentKey(lastSubkeyData.data, segment, null);
lastSubkeyData.changed = true;
}
}
}
// Flush subkey writes
if (lastSubkeyData != null) {
await _spineRecord.eventualWriteBytes(lastSubkeyData.data,
subkey: lastSubkeyData.subkey);
}
}
Future<void> _purgeSegments(int from, int to) async {
assert(_spineMutex.isLocked, 'should be in mutex here');
if (from < to) {
await _deleteSegmentsContiguous(from, to);
} else if (from > to) {
await _deleteSegmentsContiguous(from, _positionLimit);
await _deleteSegmentsContiguous(0, to);
}
}
/////////////////////////////////////////////////////////////////////////////
// Watch For Updates
// Watch head for changes
Future<void> watch() async {
// This will update any existing watches if necessary
try {
await _spineRecord.watch(subkeys: [ValueSubkeyRange.single(0)]);
// Update changes to the head record
// Don't watch for local changes because this class already handles
// notifying listeners and knows when it makes local changes
_subscription ??=
await _spineRecord.listen(localChanges: false, _onSpineChanged);
} on Exception {
// If anything fails, try to cancel the watches
await cancelWatch();
rethrow;
}
}
// Stop watching for changes to head and linked records
Future<void> cancelWatch() async {
await _spineRecord.cancelWatch();
await _subscription?.cancel();
_subscription = null;
}
// Called when the log changes online and we find out from a watch
// but not when we make a change locally
Future<void> _onSpineChanged(
DHTRecord record, Uint8List? data, List<ValueSubkeyRange> subkeys) async {
// If head record subkey zero changes, then the layout
// of the dhtshortarray has changed
if (data == null) {
throw StateError('spine head changed without data');
}
if (record.key != _spineRecord.key ||
subkeys.length != 1 ||
subkeys[0] != ValueSubkeyRange.single(0)) {
throw StateError('watch returning wrong subkey range');
}
// Decode updated head
final headData = proto.DHTLog.fromBuffer(data);
// Then update the head record
await _spineMutex.protect(() async {
final oldHead = _head;
final oldTail = _tail;
await _updateHead(headData);
sendUpdate(oldHead, oldTail);
});
}
////////////////////////////////////////////////////////////////////////////
TypedKey get recordKey => _spineRecord.key;
OwnedDHTRecordPointer get recordPointer => _spineRecord.ownedDHTRecordPointer;
int get length => _ringDistance(_tail, _head);
bool get isOpen => _spineRecord.isOpen;
// Ring buffer distance from old to new
static int _ringDistance(int n, int o) =>
(n < o) ? (_positionLimit - o) + n : n - o;
static const _positionLimit = DHTLog.segmentsPerSubkey *
DHTLog.spineSubkeys *
DHTShortArray.maxElements;
// Spine head mutex to ensure we keep the representation valid
final Mutex _spineMutex = Mutex();
// Subscription to head record internal changes
StreamSubscription<DHTRecordWatchChange>? _subscription;
// Notify closure for external spine head changes
void Function(DHTLogUpdate)? onUpdatedSpine;
// Spine DHT record
final DHTRecord _spineRecord;
// Segment stride to use for spine elements
final int _segmentStride;
// Position of the start of the log (oldest items)
int _head;
// Position of the end of the log (newest items) (exclusive)
int _tail;
// LRU cache of DHT spine elements accessed recently
// Pair of position and associated shortarray segment
final Mutex _spineCacheMutex = Mutex();
final List<(int, DHTShortArray)> _spineCache;
final Map<int, _OpenedSegment> _openedSegments;
static const int _spineCacheLength = 3;
}

3
packages/veilid_support/lib/dht_support/src/dht_record/default_dht_record_cubit.dart
View File

@ -38,7 +38,8 @@ class DefaultDHTRecordCubit<T> extends DHTRecordCubit<T> {
final Uint8List data;
final firstSubkey = subkeys.firstOrNull!.low;
if (firstSubkey != defaultSubkey || updatedata == null) {
final maybeData = await record.get(forceRefresh: true);
final maybeData =
await record.get(refreshMode: DHTRecordRefreshMode.network);
if (maybeData == null) {
return null;
}

257
packages/veilid_support/lib/dht_support/src/dht_record/dht_record.dart
View File

@ -13,9 +13,30 @@ class DHTRecordWatchChange extends Equatable {
List<Object?> get props => [local, data, subkeys];
}
/// Refresh mode for DHT record 'get'
enum DHTRecordRefreshMode {
/// Return existing subkey values if they exist locally already
/// And then check the network for a newer value
/// This is the default refresh mode
cached,
/// Return existing subkey values only if they exist locally already
local,
/// Always check the network for a newer subkey value
network,
/// Always check the network for a newer subkey value but only
/// return that value if its sequence number is newer than the local value
update;
bool get _forceRefresh => this == network || this == update;
bool get _inspectLocal => this == local || this == update;
}
/////////////////////////////////////////////////
class DHTRecord {
class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
DHTRecord._(
{required VeilidRoutingContext routingContext,
required SharedDHTRecordData sharedDHTRecordData,
@ -27,23 +48,52 @@ class DHTRecord {
_routingContext = routingContext,
_defaultSubkey = defaultSubkey,
_writer = writer,
_open = true,
_openCount = 1,
_sharedDHTRecordData = sharedDHTRecordData;
final SharedDHTRecordData _sharedDHTRecordData;
final VeilidRoutingContext _routingContext;
final int _defaultSubkey;
final KeyPair? _writer;
final DHTRecordCrypto _crypto;
final String debugName;
////////////////////////////////////////////////////////////////////////////
// DHTCloseable
bool _open;
@internal
StreamController<DHTRecordWatchChange>? watchController;
@internal
WatchState? watchState;
/// Check if the DHTRecord is open
@override
bool get isOpen => _openCount > 0;
int subkeyOrDefault(int subkey) => (subkey == -1) ? _defaultSubkey : subkey;
/// The type of the openable scope
@override
FutureOr<DHTRecord> scoped() => this;
/// Add a reference to this DHTRecord
@override
Future<DHTRecord> ref() async => _mutex.protect(() async {
_openCount++;
return this;
});
/// Free all resources for the DHTRecord
@override
Future<void> close() async => _mutex.protect(() async {
if (_openCount == 0) {
throw StateError('already closed');
}
_openCount--;
if (_openCount != 0) {
return;
}
await _watchController?.close();
_watchController = null;
await DHTRecordPool.instance._recordClosed(this);
});
/// Free all resources for the DHTRecord and delete it from the DHT
/// Will wait until the record is closed to delete it
@override
Future<void> delete() async => _mutex.protect(() async {
await DHTRecordPool.instance.deleteRecord(key);
});
////////////////////////////////////////////////////////////////////////////
// Public API
VeilidRoutingContext get routingContext => _routingContext;
TypedKey get key => _sharedDHTRecordData.recordDescriptor.key;
@ -57,85 +107,69 @@ class DHTRecord {
DHTRecordCrypto get crypto => _crypto;
OwnedDHTRecordPointer get ownedDHTRecordPointer =>
OwnedDHTRecordPointer(recordKey: key, owner: ownerKeyPair!);
bool get isOpen => _open;
Future<void> close() async {
if (!_open) {
return;
}
await watchController?.close();
await DHTRecordPool.instance._recordClosed(this);
_open = false;
}
Future<T> scope<T>(Future<T> Function(DHTRecord) scopeFunction) async {
try {
return await scopeFunction(this);
} finally {
await close();
}
}
Future<T> deleteScope<T>(Future<T> Function(DHTRecord) scopeFunction) async {
try {
final out = await scopeFunction(this);
if (_open) {
await close();
}
return out;
} on Exception catch (_) {
if (_open) {
await close();
}
await DHTRecordPool.instance.deleteRecord(key);
rethrow;
}
}
Future<T> maybeDeleteScope<T>(
bool delete, Future<T> Function(DHTRecord) scopeFunction) async {
if (delete) {
return deleteScope(scopeFunction);
} else {
return scope(scopeFunction);
}
}
int subkeyOrDefault(int subkey) => (subkey == -1) ? _defaultSubkey : subkey;
/// Get a subkey value from this record.
/// Returns the most recent value data for this subkey or null if this subkey
/// has not yet been written to.
/// * 'refreshMode' determines whether or not to return a locally existing
/// value or always check the network
/// * 'outSeqNum' optionally returns the sequence number of the value being
/// returned if one was returned.
Future<Uint8List?> get(
{int subkey = -1,
DHTRecordCrypto? crypto,
bool forceRefresh = false,
bool onlyUpdates = false,
DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
Output<int>? outSeqNum}) async {
subkey = subkeyOrDefault(subkey);
// Get the last sequence number if we need it
final lastSeq =
refreshMode._inspectLocal ? await _localSubkeySeq(subkey) : null;
// See if we only ever want the locally stored value
if (refreshMode == DHTRecordRefreshMode.local && lastSeq == null) {
// If it's not available locally already just return null now
return null;
}
final valueData = await _routingContext.getDHTValue(key, subkey,
forceRefresh: forceRefresh);
forceRefresh: refreshMode._forceRefresh);
if (valueData == null) {
return null;
}
final lastSeq = _sharedDHTRecordData.subkeySeqCache[subkey];
if (onlyUpdates && lastSeq != null && valueData.seq <= lastSeq) {
// See if this get resulted in a newer sequence number
if (refreshMode == DHTRecordRefreshMode.update &&
lastSeq != null &&
valueData.seq <= lastSeq) {
// If we're only returning updates then punt now
return null;
}
// If we're returning a value, decrypt it
final out = (crypto ?? _crypto).decrypt(valueData.data, subkey);
if (outSeqNum != null) {
outSeqNum.save(valueData.seq);
}
_sharedDHTRecordData.subkeySeqCache[subkey] = valueData.seq;
return out;
}
/// Get a subkey value from this record.
/// Process the record returned with a JSON unmarshal function 'fromJson'.
/// Returns the most recent value data for this subkey or null if this subkey
/// has not yet been written to.
/// * 'refreshMode' determines whether or not to return a locally existing
/// value or always check the network
/// * 'outSeqNum' optionally returns the sequence number of the value being
/// returned if one was returned.
Future<T?> getJson<T>(T Function(dynamic) fromJson,
{int subkey = -1,
DHTRecordCrypto? crypto,
bool forceRefresh = false,
bool onlyUpdates = false,
DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
Output<int>? outSeqNum}) async {
final data = await get(
subkey: subkey,
crypto: crypto,
forceRefresh: forceRefresh,
onlyUpdates: onlyUpdates,
refreshMode: refreshMode,
outSeqNum: outSeqNum);
if (data == null) {
return null;
@ -143,18 +177,25 @@ class DHTRecord {
return jsonDecodeBytes(fromJson, data);
}
/// Get a subkey value from this record.
/// Process the record returned with a protobuf unmarshal
/// function 'fromBuffer'.
/// Returns the most recent value data for this subkey or null if this subkey
/// has not yet been written to.
/// * 'refreshMode' determines whether or not to return a locally existing
/// value or always check the network
/// * 'outSeqNum' optionally returns the sequence number of the value being
/// returned if one was returned.
Future<T?> getProtobuf<T extends GeneratedMessage>(
T Function(List<int> i) fromBuffer,
{int subkey = -1,
DHTRecordCrypto? crypto,
bool forceRefresh = false,
bool onlyUpdates = false,
DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
Output<int>? outSeqNum}) async {
final data = await get(
subkey: subkey,
crypto: crypto,
forceRefresh: forceRefresh,
onlyUpdates: onlyUpdates,
refreshMode: refreshMode,
outSeqNum: outSeqNum);
if (data == null) {
return null;
@ -162,13 +203,16 @@ class DHTRecord {
return fromBuffer(data.toList());
}
/// Attempt to write a byte buffer to a DHTRecord subkey
/// If a newer value was found on the network, it is returned
/// If the value was succesfully written, null is returned
Future<Uint8List?> tryWriteBytes(Uint8List newValue,
{int subkey = -1,
DHTRecordCrypto? crypto,
KeyPair? writer,
Output<int>? outSeqNum}) async {
subkey = subkeyOrDefault(subkey);
final lastSeq = _sharedDHTRecordData.subkeySeqCache[subkey];
final lastSeq = await _localSubkeySeq(subkey);
final encryptedNewValue =
await (crypto ?? _crypto).encrypt(newValue, subkey);
@ -190,7 +234,6 @@ class DHTRecord {
if (isUpdated && outSeqNum != null) {
outSeqNum.save(newValueData.seq);
}
_sharedDHTRecordData.subkeySeqCache[subkey] = newValueData.seq;
// See if the encrypted data returned is exactly the same
// if so, shortcut and don't bother decrypting it
@ -211,13 +254,16 @@ class DHTRecord {
return decryptedNewValue;
}
/// Attempt to write a byte buffer to a DHTRecord subkey
/// If a newer value was found on the network, another attempt
/// will be made to write the subkey until this succeeds
Future<void> eventualWriteBytes(Uint8List newValue,
{int subkey = -1,
DHTRecordCrypto? crypto,
KeyPair? writer,
Output<int>? outSeqNum}) async {
subkey = subkeyOrDefault(subkey);
final lastSeq = _sharedDHTRecordData.subkeySeqCache[subkey];
final lastSeq = await _localSubkeySeq(subkey);
final encryptedNewValue =
await (crypto ?? _crypto).encrypt(newValue, subkey);
@ -243,7 +289,6 @@ class DHTRecord {
if (outSeqNum != null) {
outSeqNum.save(newValueData.seq);
}
_sharedDHTRecordData.subkeySeqCache[subkey] = newValueData.seq;
// The encrypted data returned should be exactly the same
// as what we are trying to set,
@ -256,6 +301,11 @@ class DHTRecord {
}
}
/// Attempt to write a byte buffer to a DHTRecord subkey
/// If a newer value was found on the network, another attempt
/// will be made to write the subkey until this succeeds
/// Each attempt to write the value calls an update function with the
/// old value to determine what new value should be attempted for that write.
Future<void> eventualUpdateBytes(
Future<Uint8List> Function(Uint8List? oldValue) update,
{int subkey = -1,
@ -281,6 +331,7 @@ class DHTRecord {
} while (oldValue != null);
}
/// Like 'tryWriteBytes' but with JSON marshal/unmarshal of the value
Future<T?> tryWriteJson<T>(T Function(dynamic) fromJson, T newValue,
{int subkey = -1,
DHTRecordCrypto? crypto,
@ -298,6 +349,7 @@ class DHTRecord {
return jsonDecodeBytes(fromJson, out);
});
/// Like 'tryWriteBytes' but with protobuf marshal/unmarshal of the value
Future<T?> tryWriteProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, T newValue,
{int subkey = -1,
@ -316,6 +368,7 @@ class DHTRecord {
return fromBuffer(out);
});
/// Like 'eventualWriteBytes' but with JSON marshal/unmarshal of the value
Future<void> eventualWriteJson<T>(T newValue,
{int subkey = -1,
DHTRecordCrypto? crypto,
@ -324,6 +377,7 @@ class DHTRecord {
eventualWriteBytes(jsonEncodeBytes(newValue),
subkey: subkey, crypto: crypto, writer: writer, outSeqNum: outSeqNum);
/// Like 'eventualWriteBytes' but with protobuf marshal/unmarshal of the value
Future<void> eventualWriteProtobuf<T extends GeneratedMessage>(T newValue,
{int subkey = -1,
DHTRecordCrypto? crypto,
@ -332,6 +386,7 @@ class DHTRecord {
eventualWriteBytes(newValue.writeToBuffer(),
subkey: subkey, crypto: crypto, writer: writer, outSeqNum: outSeqNum);
/// Like 'eventualUpdateBytes' but with JSON marshal/unmarshal of the value
Future<void> eventualUpdateJson<T>(
T Function(dynamic) fromJson, Future<T> Function(T?) update,
{int subkey = -1,
@ -341,6 +396,7 @@ class DHTRecord {
eventualUpdateBytes(jsonUpdate(fromJson, update),
subkey: subkey, crypto: crypto, writer: writer, outSeqNum: outSeqNum);
/// Like 'eventualUpdateBytes' but with protobuf marshal/unmarshal of the value
Future<void> eventualUpdateProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, Future<T> Function(T?) update,
{int subkey = -1,
@ -350,6 +406,8 @@ class DHTRecord {
eventualUpdateBytes(protobufUpdate(fromBuffer, update),
subkey: subkey, crypto: crypto, writer: writer, outSeqNum: outSeqNum);
/// Watch a subkey range of this DHT record for changes
/// Takes effect on the next DHTRecordPool tick
Future<void> watch(
{List<ValueSubkeyRange>? subkeys,
Timestamp? expiration,
@ -363,6 +421,13 @@ class DHTRecord {
}
}
/// Register a callback for changes made on this this DHT record.
/// You must 'watch' the record as well as listen to it in order for this
/// call back to be called.
/// * 'localChanges' also enables calling the callback if changed are made
/// locally, otherwise only changes seen from the network itself are
/// reported
///
Future<StreamSubscription<DHTRecordWatchChange>> listen(
Future<void> Function(
DHTRecord record, Uint8List? data, List<ValueSubkeyRange> subkeys)
@ -371,13 +436,13 @@ class DHTRecord {
DHTRecordCrypto? crypto,
}) async {
// Set up watch requirements
watchController ??=
_watchController ??=
StreamController<DHTRecordWatchChange>.broadcast(onCancel: () {
// If there are no more listeners then we can get rid of the controller
watchController = null;
_watchController = null;
});
return watchController!.stream.listen(
return _watchController!.stream.listen(
(change) {
if (change.local && !localChanges) {
return;
@ -400,11 +465,13 @@ class DHTRecord {
},
cancelOnError: true,
onError: (e) async {
await watchController!.close();
watchController = null;
await _watchController!.close();
_watchController = null;
});
}
/// Stop watching this record for changes
/// Takes effect on the next DHTRecordPool tick
Future<void> cancelWatch() async {
// Tear down watch requirements
if (watchState != null) {
@ -413,11 +480,23 @@ class DHTRecord {
}
}
/// Return the inspection state of a set of subkeys of the DHTRecord
/// See Veilid's 'inspectDHTRecord' call for details on how this works
Future<DHTRecordReport> inspect(
{List<ValueSubkeyRange>? subkeys,
DHTReportScope scope = DHTReportScope.local}) =>
_routingContext.inspectDHTRecord(key, subkeys: subkeys, scope: scope);
//////////////////////////////////////////////////////////////////////////
Future<int?> _localSubkeySeq(int subkey) async {
final rr = await _routingContext.inspectDHTRecord(
key,
subkeys: [ValueSubkeyRange.single(subkey)],
);
return rr.localSeqs.firstOrNull ?? 0xFFFFFFFF;
}
void _addValueChange(
{required bool local,
required Uint8List? data,
@ -427,7 +506,7 @@ class DHTRecord {
final watchedSubkeys = ws.subkeys;
if (watchedSubkeys == null) {
// Report all subkeys
watchController?.add(
_watchController?.add(
DHTRecordWatchChange(local: local, data: data, subkeys: subkeys));
} else {
// Only some subkeys are being watched, see if the reported update
@ -442,7 +521,7 @@ class DHTRecord {
overlappedFirstSubkey == updateFirstSubkey ? data : null;
// Report only watched subkeys
watchController?.add(DHTRecordWatchChange(
_watchController?.add(DHTRecordWatchChange(
local: local, data: updatedData, subkeys: overlappedSubkeys));
}
}
@ -458,4 +537,18 @@ class DHTRecord {
_addValueChange(
local: false, data: update.value?.data, subkeys: update.subkeys);
}
//////////////////////////////////////////////////////////////
final SharedDHTRecordData _sharedDHTRecordData;
final VeilidRoutingContext _routingContext;
final int _defaultSubkey;
final KeyPair? _writer;
final DHTRecordCrypto _crypto;
final String debugName;
final _mutex = Mutex();
int _openCount;
StreamController<DHTRecordWatchChange>? _watchController;
@internal
WatchState? watchState;
}

2
packages/veilid_support/lib/dht_support/src/dht_record/dht_record_cubit.dart
View File

@ -93,7 +93,7 @@ class DHTRecordCubit<T> extends Cubit<AsyncValue<T>> {
for (final skr in subkeys) {
for (var sk = skr.low; sk <= skr.high; sk++) {
final data = await _record.get(
subkey: sk, forceRefresh: true, onlyUpdates: true);
subkey: sk, refreshMode: DHTRecordRefreshMode.update);
if (data != null) {
final newState = await _stateFunction(_record, updateSubkeys, data);
if (newState != null) {

109
packages/veilid_support/lib/dht_support/src/dht_record/dht_record_pool.dart
View File

@ -88,10 +88,8 @@ class SharedDHTRecordData {
DHTRecordDescriptor recordDescriptor;
KeyPair? defaultWriter;
VeilidRoutingContext defaultRoutingContext;
Map<int, int> subkeySeqCache = {};
bool needsWatchStateUpdate = false;
WatchState? unionWatchState;
bool deleteOnClose = false;
}
// Per opened record data
@ -128,6 +126,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
: _state = const DHTRecordPoolAllocations(),
_mutex = Mutex(),
_opened = <TypedKey, OpenedRecordInfo>{},
_markedForDelete = <TypedKey>{},
_routingContext = routingContext,
_veilid = veilid;
@ -140,6 +139,8 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
final Mutex _mutex;
// Which DHT records are currently open
final Map<TypedKey, OpenedRecordInfo> _opened;
// Which DHT records are marked for deletion
final Set<TypedKey> _markedForDelete;
// Default routing context to use for new keys
final VeilidRoutingContext _routingContext;
// Convenience accessor
@ -288,6 +289,8 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
return openedRecordInfo;
}
// Called when a DHTRecord is closed
// Cleans up the opened record housekeeping and processes any late deletions
Future<void> _recordClosed(DHTRecord record) async {
await _mutex.protect(() async {
final key = record.key;
@ -301,14 +304,37 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
}
if (openedRecordInfo.records.isEmpty) {
await _routingContext.closeDHTRecord(key);
if (openedRecordInfo.shared.deleteOnClose) {
await _deleteRecordInner(key);
}
_opened.remove(key);
await _checkForLateDeletesInner(key);
}
});
}
// Check to see if this key can finally be deleted
// If any parents are marked for deletion, try them first
Future<void> _checkForLateDeletesInner(TypedKey key) async {
// Get parent list in bottom up order including our own key
final parents = <TypedKey>[];
TypedKey? nextParent = key;
while (nextParent != null) {
parents.add(nextParent);
nextParent = getParentRecordKey(nextParent);
}
// If any parent is ready to delete all its children do it
for (final parent in parents) {
if (_markedForDelete.contains(parent)) {
final deleted = await _deleteRecordInner(parent);
if (!deleted) {
// If we couldn't delete a child then no 'marked for delete' parents
// above us will be ready to delete either
break;
}
}
}
}
// Collect all dependencies (including the record itself)
// in reverse (bottom-up/delete order)
List<TypedKey> _collectChildrenInner(TypedKey recordKey) {
@ -328,7 +354,13 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
return allDeps.reversedView;
}
String _debugChildren(TypedKey recordKey, {List<TypedKey>? allDeps}) {
/// Collect all dependencies (including the record itself)
/// in reverse (bottom-up/delete order)
Future<List<TypedKey>> collectChildren(TypedKey recordKey) =>
_mutex.protect(() async => _collectChildrenInner(recordKey));
/// Print children
String debugChildren(TypedKey recordKey, {List<TypedKey>? allDeps}) {
allDeps ??= _collectChildrenInner(recordKey);
// ignore: avoid_print
var out =
@ -342,32 +374,48 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
return out;
}
Future<void> _deleteRecordInner(TypedKey recordKey) async {
log('deleteDHTRecord: key=$recordKey');
// Actual delete function
Future<void> _finalizeDeleteRecordInner(TypedKey recordKey) async {
log('_finalizeDeleteRecordInner: key=$recordKey');
// Remove this child from parents
await _removeDependenciesInner([recordKey]);
await _routingContext.deleteDHTRecord(recordKey);
_markedForDelete.remove(recordKey);
}
Future<void> deleteRecord(TypedKey recordKey) async {
await _mutex.protect(() async {
final allDeps = _collectChildrenInner(recordKey);
if (allDeps.singleOrNull != recordKey) {
final dbgstr = _debugChildren(recordKey, allDeps: allDeps);
throw StateError('must delete children first: $dbgstr');
}
final ori = _opened[recordKey];
if (ori != null) {
// delete after close
ori.shared.deleteOnClose = true;
} else {
// Deep delete mechanism inside mutex
Future<bool> _deleteRecordInner(TypedKey recordKey) async {
final toDelete = _readyForDeleteInner(recordKey);
if (toDelete.isNotEmpty) {
// delete now
await _deleteRecordInner(recordKey);
for (final deleteKey in toDelete) {
await _finalizeDeleteRecordInner(deleteKey);
}
});
return true;
}
// mark for deletion
_markedForDelete.add(recordKey);
return false;
}
/// Delete a record and its children if they are all closed
/// otherwise mark that record for deletion eventually
/// Returns true if the deletion was processed immediately
/// Returns false if the deletion was marked for later
Future<bool> deleteRecord(TypedKey recordKey) async =>
_mutex.protect(() async => _deleteRecordInner(recordKey));
// If everything underneath is closed including itself, return the
// list of children (and itself) to finally actually delete
List<TypedKey> _readyForDeleteInner(TypedKey recordKey) {
final allDeps = _collectChildrenInner(recordKey);
for (final dep in allDeps) {
if (_opened.containsKey(dep)) {
return [];
}
}
return allDeps;
}
void _validateParentInner(TypedKey? parent, TypedKey child) {
@ -456,6 +504,19 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
}
}
bool _isValidRecordKeyInner(TypedKey key) {
if (_state.rootRecords.contains(key)) {
return true;
}
if (_state.childrenByParent.containsKey(key.toJson())) {
return true;
}
return false;
}
Future<bool> isValidRecordKey(TypedKey key) =>
_mutex.protect(() async => _isValidRecordKeyInner(key));
///////////////////////////////////////////////////////////////////////
/// Create a root DHTRecord that has no dependent records

136
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array.dart
View File

@ -3,7 +3,6 @@ import 'dart:typed_data';
import 'package:async_tools/async_tools.dart';
import 'package:collection/collection.dart';
import 'package:protobuf/protobuf.dart';
import '../../../veilid_support.dart';
import '../../proto/proto.dart' as proto;
@ -14,12 +13,13 @@ part 'dht_short_array_write.dart';
///////////////////////////////////////////////////////////////////////
class DHTShortArray {
class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
////////////////////////////////////////////////////////////////
// Constructors
DHTShortArray._({required DHTRecord headRecord})
: _head = _DHTShortArrayHead(headRecord: headRecord) {
: _head = _DHTShortArrayHead(headRecord: headRecord),
_openCount = 1 {
_head.onUpdatedHead = () {
_watchController?.sink.add(null);
};
@ -34,22 +34,22 @@ class DHTShortArray {
VeilidRoutingContext? routingContext,
TypedKey? parent,
DHTRecordCrypto? crypto,
KeyPair? smplWriter}) async {
KeyPair? writer}) async {
assert(stride <= maxElements, 'stride too long');
final pool = DHTRecordPool.instance;
late final DHTRecord dhtRecord;
if (smplWriter != null) {
if (writer != null) {
final schema = DHTSchema.smpl(
oCnt: 0,
members: [DHTSchemaMember(mKey: smplWriter.key, mCnt: stride + 1)]);
members: [DHTSchemaMember(mKey: writer.key, mCnt: stride + 1)]);
dhtRecord = await pool.createRecord(
debugName: debugName,
parent: parent,
routingContext: routingContext,
schema: schema,
crypto: crypto,
writer: smplWriter);
writer: writer);
} else {
final schema = DHTSchema.dflt(oCnt: stride + 1);
dhtRecord = await pool.createRecord(
@ -120,21 +120,62 @@ class DHTShortArray {
}
static Future<DHTShortArray> openOwned(
OwnedDHTRecordPointer ownedDHTRecordPointer, {
OwnedDHTRecordPointer ownedShortArrayRecordPointer, {
required String debugName,
required TypedKey parent,
VeilidRoutingContext? routingContext,
DHTRecordCrypto? crypto,
}) =>
openWrite(
ownedDHTRecordPointer.recordKey,
ownedDHTRecordPointer.owner,
ownedShortArrayRecordPointer.recordKey,
ownedShortArrayRecordPointer.owner,
debugName: debugName,
routingContext: routingContext,
parent: parent,
crypto: crypto,
);
////////////////////////////////////////////////////////////////////////////
// DHTCloseable
/// Check if the shortarray is open
@override
bool get isOpen => _openCount > 0;
/// The type of the openable scope
@override
FutureOr<DHTShortArray> scoped() => this;
/// Add a reference to this shortarray
@override
Future<DHTShortArray> ref() async => _mutex.protect(() async {
_openCount++;
return this;
});
/// Free all resources for the DHTShortArray
@override
Future<void> close() async => _mutex.protect(() async {
if (_openCount == 0) {
throw StateError('already closed');
}
_openCount--;
if (_openCount != 0) {
return;
}
await _watchController?.close();
_watchController = null;
await _head.close();
});
/// Free all resources for the DHTShortArray and delete it from the DHT
/// Will wait until the short array is closed to delete it
@override
Future<void> delete() async {
await _head.delete();
}
////////////////////////////////////////////////////////////////////////////
// Public API
@ -144,59 +185,8 @@ class DHTShortArray {
/// Get the record pointer foir this shortarray
OwnedDHTRecordPointer get recordPointer => _head.recordPointer;
/// Check if the shortarray is open
bool get isOpen => _head.isOpen;
/// Free all resources for the DHTShortArray
Future<void> close() async {
if (!isOpen) {
return;
}
await _watchController?.close();
_watchController = null;
await _head.close();
}
/// Free all resources for the DHTShortArray and delete it from the DHT
/// Will wait until the short array is closed to delete it
Future<void> delete() async {
await _head.delete();
}
/// Runs a closure that guarantees the DHTShortArray
/// will be closed upon exit, even if an uncaught exception is thrown
Future<T> scope<T>(Future<T> Function(DHTShortArray) scopeFunction) async {
if (!isOpen) {
throw StateError('short array is not open"');
}
try {
return await scopeFunction(this);
} finally {
await close();
}
}
/// Runs a closure that guarantees the DHTShortArray
/// will be closed upon exit, and deleted if an an
/// uncaught exception is thrown
Future<T> deleteScope<T>(
Future<T> Function(DHTShortArray) scopeFunction) async {
if (!isOpen) {
throw StateError('short array is not open"');
}
try {
final out = await scopeFunction(this);
await close();
return out;
} on Exception catch (_) {
await delete();
rethrow;
}
}
/// Runs a closure allowing read-only access to the shortarray
Future<T?> operate<T>(Future<T?> Function(DHTShortArrayRead) closure) async {
Future<T> operate<T>(Future<T> Function(DHTRandomRead) closure) async {
if (!isOpen) {
throw StateError('short array is not open"');
}
@ -209,14 +199,20 @@ class DHTShortArray {
/// Runs a closure allowing read-write access to the shortarray
/// Makes only one attempt to consistently write the changes to the DHT
/// Returns (result, true) of the closure if the write could be performed
/// Returns (null, false) if the write could not be performed at this time
Future<(T?, bool)> operateWrite<T>(
Future<T?> Function(DHTShortArrayWrite) closure) async =>
_head.operateWrite((head) async {
/// Returns result of the closure if the write could be performed
/// Throws DHTOperateException if the write could not be performed
/// at this time
Future<T> operateWrite<T>(
Future<T> Function(DHTRandomReadWrite) closure) async {
if (!isOpen) {
throw StateError('short array is not open"');
}
return _head.operateWrite((head) async {
final writer = _DHTShortArrayWrite._(head);
return closure(writer);
});
}
/// Runs a closure allowing read-write access to the shortarray
/// Will execute the closure multiple times if a consistent write to the DHT
@ -225,7 +221,7 @@ class DHTShortArray {
/// succeeded, returning false will trigger another eventual consistency
/// attempt.
Future<void> operateWriteEventual(
Future<bool> Function(DHTShortArrayWrite) closure,
Future<bool> Function(DHTRandomReadWrite) closure,
{Duration? timeout}) async {
if (!isOpen) {
throw StateError('short array is not open"');
@ -276,6 +272,10 @@ class DHTShortArray {
// Internal representation refreshed from head record
final _DHTShortArrayHead _head;
// Openable
int _openCount;
final _mutex = Mutex();
// Watch mutex to ensure we keep the representation valid
final Mutex _listenMutex = Mutex();
// Stream of external changes

24
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_cubit.dart
View File

@ -41,19 +41,6 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
});
}
// DHTShortArrayCubit.value({
// required DHTShortArray shortArray,
// required T Function(List<int> data) decodeElement,
// }) : _shortArray = shortArray,
// _decodeElement = decodeElement,
// super(const BlocBusyState(AsyncValue.loading())) {
// _initFuture = Future(() async {
// // Make initial state update
// unawaited(_refreshNoWait());
// _subscription = await shortArray.listen(_update);
// });
// }
Future<void> refresh({bool forceRefresh = false}) async {
await _initWait();
await _refreshNoWait(forceRefresh: forceRefresh);
@ -67,7 +54,8 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
try {
final newState = await _shortArray.operate((reader) async {
final offlinePositions = await reader.getOfflinePositions();
final allItems = (await reader.getAllItems(forceRefresh: forceRefresh))
final allItems =
(await reader.getItemRange(0, forceRefresh: forceRefresh))
?.indexed
.map((x) => DHTShortArrayElementState(
value: _decodeElement(x.$2),
@ -103,19 +91,19 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
await super.close();
}
Future<R?> operate<R>(Future<R?> Function(DHTShortArrayRead) closure) async {
Future<R> operate<R>(Future<R> Function(DHTRandomRead) closure) async {
await _initWait();
return _shortArray.operate(closure);
}
Future<(R?, bool)> operateWrite<R>(
Future<R?> Function(DHTShortArrayWrite) closure) async {
Future<R> operateWrite<R>(
Future<R> Function(DHTRandomReadWrite) closure) async {
await _initWait();
return _shortArray.operateWrite(closure);
}
Future<void> operateWriteEventual(
Future<bool> Function(DHTShortArrayWrite) closure,
Future<bool> Function(DHTRandomReadWrite) closure,
{Duration? timeout}) async {
await _initWait();
return _shortArray.operateWriteEventual(closure, timeout: timeout);

60
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_head.dart
View File

@ -67,12 +67,8 @@ class _DHTShortArrayHead {
Future<void> delete() async {
await _headMutex.protect(() async {
final pool = DHTRecordPool.instance;
final futures = <Future<void>>[pool.deleteRecord(_headRecord.key)];
for (final lr in _linkedRecords) {
futures.add(pool.deleteRecord(lr.key));
}
await Future.wait(futures);
// Will deep delete all linked records as they are children
await _headRecord.delete();
});
}
@ -82,8 +78,8 @@ class _DHTShortArrayHead {
return closure(this);
});
Future<(T?, bool)> operateWrite<T>(
Future<T?> Function(_DHTShortArrayHead) closure) async =>
Future<T> operateWrite<T>(
Future<T> Function(_DHTShortArrayHead) closure) async =>
_headMutex.protect(() async {
final oldLinkedRecords = List.of(_linkedRecords);
final oldIndex = List.of(_index);
@ -95,11 +91,11 @@ class _DHTShortArrayHead {
if (!await _writeHead()) {
// Failed to write head means head got overwritten so write should
// be considered failed
return (null, false);
throw DHTExceptionTryAgain();
}
onUpdatedHead?.call();
return (out, true);
return out;
} on Exception {
// Exception means state needs to be reverted
_linkedRecords = oldLinkedRecords;
@ -219,7 +215,7 @@ class _DHTShortArrayHead {
}
} on Exception catch (_) {
// On any exception close the records we have opened
await Future.wait(newRecords.entries.map((e) => e.value.close()));
await newRecords.entries.map((e) => e.value.close()).wait;
rethrow;
}
@ -249,34 +245,36 @@ class _DHTShortArrayHead {
}
// Pull the latest or updated copy of the head record from the network
Future<bool> _loadHead(
{bool forceRefresh = true, bool onlyUpdates = false}) async {
Future<void> _loadHead() async {
// Get an updated head record copy if one exists
final head = await _headRecord.getProtobuf(proto.DHTShortArray.fromBuffer,
subkey: 0, forceRefresh: forceRefresh, onlyUpdates: onlyUpdates);
subkey: 0, refreshMode: DHTRecordRefreshMode.network);
if (head == null) {
if (onlyUpdates) {
// No update
return false;
}
throw StateError('head missing during refresh');
throw StateError('shortarray head missing during refresh');
}
await _updateHead(head);
return true;
}
/////////////////////////////////////////////////////////////////////////////
// Linked record management
Future<DHTRecord> _getOrCreateLinkedRecord(int recordNumber) async {
Future<DHTRecord> _getOrCreateLinkedRecord(
int recordNumber, bool allowCreate) async {
if (recordNumber == 0) {
return _headRecord;
}
final pool = DHTRecordPool.instance;
recordNumber--;
while (recordNumber >= _linkedRecords.length) {
if (recordNumber < _linkedRecords.length) {
return _linkedRecords[recordNumber];
}
if (!allowCreate) {
throw StateError("asked for non-existent record and can't create");
}
final pool = DHTRecordPool.instance;
for (var rn = _linkedRecords.length; rn <= recordNumber; rn++) {
// Linked records must use SMPL schema so writer can be specified
// Use the same writer as the head record
final smplWriter = _headRecord.writer!;
@ -298,9 +296,6 @@ class _DHTShortArrayHead {
// Add to linked records
_linkedRecords.add(dhtRecord);
}
if (!await _writeHead()) {
throw StateError('failed to add linked record');
}
return _linkedRecords[recordNumber];
}
@ -324,15 +319,16 @@ class _DHTShortArrayHead {
);
}
Future<DHTShortArrayHeadLookup> lookupPosition(int pos) async {
Future<DHTShortArrayHeadLookup> lookupPosition(
int pos, bool allowCreate) async {
final idx = _index[pos];
return lookupIndex(idx);
return lookupIndex(idx, allowCreate);
}
Future<DHTShortArrayHeadLookup> lookupIndex(int idx) async {
Future<DHTShortArrayHeadLookup> lookupIndex(int idx, bool allowCreate) async {
final seq = idx < _seqs.length ? _seqs[idx] : 0xFFFFFFFF;
final recordNumber = idx ~/ _stride;
final record = await _getOrCreateLinkedRecord(recordNumber);
final record = await _getOrCreateLinkedRecord(recordNumber, allowCreate);
final recordSubkey = (idx % _stride) + ((recordNumber == 0) ? 1 : 0);
return DHTShortArrayHeadLookup(
record: record, recordSubkey: recordSubkey, seq: seq);
@ -389,7 +385,7 @@ class _DHTShortArrayHead {
assert(
newKeys.length <=
(DHTShortArray.maxElements + (_stride - 1)) ~/ _stride,
'too many keys');
'too many keys: $newKeys.length');
assert(newKeys.length == linkedKeys.length, 'duplicated linked keys');
final newIndex = index.toSet();
assert(newIndex.length <= DHTShortArray.maxElements, 'too many indexes');

101
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_read.dart
View File

@ -1,83 +1,29 @@
part of 'dht_short_array.dart';
////////////////////////////////////////////////////////////////////////////
// Reader interface
abstract class DHTShortArrayRead {
/// Returns the number of elements in the DHTShortArray
int get length;
/// Return the item at position 'pos' in the DHTShortArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
Future<Uint8List?> getItem(int pos, {bool forceRefresh = false});
/// Return a list of all of the items in the DHTShortArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
Future<List<Uint8List>?> getAllItems({bool forceRefresh = false});
/// Get a list of the positions that were written offline and not flushed yet
Future<Set<int>> getOfflinePositions();
}
extension DHTShortArrayReadExt on DHTShortArrayRead {
/// Convenience function:
/// Like getItem but also parses the returned element as JSON
Future<T?> getItemJson<T>(T Function(dynamic) fromJson, int pos,
{bool forceRefresh = false}) =>
getItem(pos, forceRefresh: forceRefresh)
.then((out) => jsonDecodeOptBytes(fromJson, out));
/// Convenience function:
/// Like getAllItems but also parses the returned elements as JSON
Future<List<T>?> getAllItemsJson<T>(T Function(dynamic) fromJson,
{bool forceRefresh = false}) =>
getAllItems(forceRefresh: forceRefresh)
.then((out) => out?.map(fromJson).toList());
/// Convenience function:
/// Like getItem but also parses the returned element as a protobuf object
Future<T?> getItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int pos,
{bool forceRefresh = false}) =>
getItem(pos, forceRefresh: forceRefresh)
.then((out) => (out == null) ? null : fromBuffer(out));
/// Convenience function:
/// Like getAllItems but also parses the returned elements as protobuf objects
Future<List<T>?> getAllItemsProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer,
{bool forceRefresh = false}) =>
getAllItems(forceRefresh: forceRefresh)
.then((out) => out?.map(fromBuffer).toList());
}
////////////////////////////////////////////////////////////////////////////
// Reader-only implementation
class _DHTShortArrayRead implements DHTShortArrayRead {
class _DHTShortArrayRead implements DHTRandomRead {
_DHTShortArrayRead._(_DHTShortArrayHead head) : _head = head;
/// Returns the number of elements in the DHTShortArray
@override
int get length => _head.length;
/// Return the item at position 'pos' in the DHTShortArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
@override
Future<Uint8List?> getItem(int pos, {bool forceRefresh = false}) async {
if (pos < 0 || pos >= length) {
throw IndexError.withLength(pos, length);
}
final lookup = await _head.lookupPosition(pos);
final lookup = await _head.lookupPosition(pos, false);
final refresh = forceRefresh || _head.positionNeedsRefresh(pos);
final outSeqNum = Output<int>();
final out = lookup.record.get(
subkey: lookup.recordSubkey,
forceRefresh: refresh,
refreshMode: refresh
? DHTRecordRefreshMode.network
: DHTRecordRefreshMode.cached,
outSeqNum: outSeqNum);
if (outSeqNum.value != null) {
_head.updatePositionSeq(pos, false, outSeqNum.value!);
@ -86,17 +32,29 @@ class _DHTShortArrayRead implements DHTShortArrayRead {
return out;
}
/// Return a list of all of the items in the DHTShortArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
@override
Future<List<Uint8List>?> getAllItems({bool forceRefresh = false}) async {
final out = <Uint8List>[];
(int, int) _clampStartLen(int start, int? len) {
len ??= _head.length;
if (start < 0) {
throw IndexError.withLength(start, _head.length);
}
if (start > _head.length) {
throw IndexError.withLength(start, _head.length);
}
if ((len + start) > _head.length) {
len = _head.length - start;
}
return (start, len);
}
final chunks = Iterable<int>.generate(_head.length)
.slices(maxDHTConcurrency)
.map((chunk) =>
chunk.map((pos) => getItem(pos, forceRefresh: forceRefresh)));
@override
Future<List<Uint8List>?> getItemRange(int start,
{int? length, bool forceRefresh = false}) async {
final out = <Uint8List>[];
(start, length) = _clampStartLen(start, length);
final chunks = Iterable<int>.generate(length).slices(maxDHTConcurrency).map(
(chunk) => chunk
.map((pos) => getItem(pos + start, forceRefresh: forceRefresh)));
for (final chunk in chunks) {
final elems = await chunk.wait;
@ -109,9 +67,10 @@ class _DHTShortArrayRead implements DHTShortArrayRead {
return out;
}
/// Get a list of the positions that were written offline and not flushed yet
@override
Future<Set<int>> getOfflinePositions() async {
final (start, length) = _clampStartLen(0, DHTShortArray.maxElements);
final indexOffline = <int>{};
final inspects = await [
_head._headRecord.inspect(),
@ -134,7 +93,7 @@ class _DHTShortArrayRead implements DHTShortArrayRead {
// See which positions map to offline indexes
final positionOffline = <int>{};
for (var i = 0; i < _head._index.length; i++) {
for (var i = start; i < (start + length); i++) {
final idx = _head._index[i];
if (indexOffline.contains(idx)) {
positionOffline.add(i);

228
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_write.dart
View File

@ -1,134 +1,98 @@
part of 'dht_short_array.dart';
////////////////////////////////////////////////////////////////////////////
// Writer interface
abstract class DHTShortArrayWrite implements DHTShortArrayRead {
/// Try to add an item to the end of the DHTShortArray. Return true if the
/// element was successfully added, and false if the state changed before
/// the element could be added or a newer value was found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryAddItem(Uint8List value);
/// Try to insert an item as position 'pos' of the DHTShortArray.
/// Return true if the element was successfully inserted, and false if the
/// state changed before the element could be inserted or a newer value was
/// found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryInsertItem(int pos, Uint8List value);
/// Try to swap items at position 'aPos' and 'bPos' in the DHTShortArray.
/// Return true if the elements were successfully swapped, and false if the
/// state changed before the elements could be swapped or newer values were
/// found on the network.
/// This may throw an exception if either of the positions swapped exceed
/// the length of the list
Future<bool> trySwapItem(int aPos, int bPos);
/// Try to remove an item at position 'pos' in the DHTShortArray.
/// Return the element if it was successfully removed, and null if the
/// state changed before the elements could be removed or newer values were
/// found on the network.
/// This may throw an exception if the position removed exceeeds the length of
/// the list.
Future<Uint8List?> tryRemoveItem(int pos);
/// Try to remove all items in the DHTShortArray.
/// Return true if it was successfully cleared, and false if the
/// state changed before the elements could be cleared or newer values were
/// found on the network.
Future<bool> tryClear();
/// Try to set an item at position 'pos' of the DHTShortArray.
/// If the set was successful this returns:
/// * The prior contents of the element, or null if there was no value yet
/// * A boolean true
/// If the set was found a newer value on the network:
/// * The newer value of the element, or null if the head record
/// changed.
/// * A boolean false
/// This may throw an exception if the position exceeds the built-in limit of
/// 'maxElements = 256' entries.
Future<(Uint8List?, bool)> tryWriteItem(int pos, Uint8List newValue);
}
extension DHTShortArrayWriteExt on DHTShortArrayWrite {
/// Convenience function:
/// Like removeItem but also parses the returned element as JSON
Future<T?> tryRemoveItemJson<T>(
T Function(dynamic) fromJson,
int pos,
) =>
tryRemoveItem(pos).then((out) => jsonDecodeOptBytes(fromJson, out));
/// Convenience function:
/// Like removeItem but also parses the returned element as JSON
Future<T?> tryRemoveItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int pos) =>
getItem(pos).then((out) => (out == null) ? null : fromBuffer(out));
/// Convenience function:
/// Like tryWriteItem but also encodes the input value as JSON and parses the
/// returned element as JSON
Future<(T?, bool)> tryWriteItemJson<T>(
T Function(dynamic) fromJson,
int pos,
T newValue,
) =>
tryWriteItem(pos, jsonEncodeBytes(newValue))
.then((out) => (jsonDecodeOptBytes(fromJson, out.$1), out.$2));
/// Convenience function:
/// Like tryWriteItem but also encodes the input value as a protobuf object
/// and parses the returned element as a protobuf object
Future<(T?, bool)> tryWriteItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer,
int pos,
T newValue,
) =>
tryWriteItem(pos, newValue.writeToBuffer()).then(
(out) => ((out.$1 == null ? null : fromBuffer(out.$1!)), out.$2));
}
////////////////////////////////////////////////////////////////////////////
// Writer implementation
class _DHTShortArrayWrite extends _DHTShortArrayRead
implements DHTShortArrayWrite {
implements DHTRandomReadWrite {
_DHTShortArrayWrite._(super.head) : super._();
@override
Future<bool> tryAddItem(Uint8List value) async {
// Allocate empty index at the end of the list
final pos = _head.length;
_head.allocateIndex(pos);
Future<bool> tryAddItem(Uint8List value) =>
tryInsertItem(_head.length, value);
// Write item
final (_, wasSet) = await tryWriteItem(pos, value);
if (!wasSet) {
return false;
}
return true;
}
@override
Future<bool> tryAddItems(List<Uint8List> values) =>
tryInsertItems(_head.length, values);
@override
Future<bool> tryInsertItem(int pos, Uint8List value) async {
// Allocate empty index at position
_head.allocateIndex(pos);
// Write item
final (_, wasSet) = await tryWriteItem(pos, value);
if (!wasSet) {
return false;
if (pos < 0 || pos > _head.length) {
throw IndexError.withLength(pos, _head.length);
}
// Allocate empty index at position
_head.allocateIndex(pos);
var success = false;
try {
// Write item
success = await tryWriteItem(pos, value);
} finally {
if (!success) {
_head.freeIndex(pos);
}
}
return true;
}
@override
Future<bool> trySwapItem(int aPos, int bPos) async {
Future<bool> tryInsertItems(int pos, List<Uint8List> values) async {
if (pos < 0 || pos > _head.length) {
throw IndexError.withLength(pos, _head.length);
}
// Allocate empty indices
for (var i = 0; i < values.length; i++) {
_head.allocateIndex(pos + i);
}
var success = true;
final outSeqNums = List.generate(values.length, (_) => Output<int>());
final lookups = <DHTShortArrayHeadLookup>[];
try {
// do all lookups
for (var i = 0; i < values.length; i++) {
final lookup = await _head.lookupPosition(pos + i, true);
lookups.add(lookup);
}
// Write items in parallel
final dws = DelayedWaitSet<void>();
for (var i = 0; i < values.length; i++) {
final lookup = lookups[i];
final value = values[i];
final outSeqNum = outSeqNums[i];
dws.add(() async {
final outValue = await lookup.record.tryWriteBytes(value,
subkey: lookup.recordSubkey, outSeqNum: outSeqNum);
if (outValue != null) {
success = false;
}
});
}
await dws(chunkSize: maxDHTConcurrency, onChunkDone: (_) => success);
} finally {
// Update sequence numbers
for (var i = 0; i < values.length; i++) {
if (outSeqNums[i].value != null) {
_head.updatePositionSeq(pos + i, true, outSeqNums[i].value!);
}
}
// Free indices if this was a failure
if (!success) {
for (var i = 0; i < values.length; i++) {
_head.freeIndex(pos);
}
}
}
return success;
}
@override
Future<void> swapItem(int aPos, int bPos) async {
if (aPos < 0 || aPos >= _head.length) {
throw IndexError.withLength(aPos, _head.length);
}
@ -137,16 +101,14 @@ class _DHTShortArrayWrite extends _DHTShortArrayRead
}
// Swap indices
_head.swapIndex(aPos, bPos);
return true;
}
@override
Future<Uint8List> tryRemoveItem(int pos) async {
Future<void> removeItem(int pos, {Output<Uint8List>? output}) async {
if (pos < 0 || pos >= _head.length) {
throw IndexError.withLength(pos, _head.length);
}
final lookup = await _head.lookupPosition(pos);
final lookup = await _head.lookupPosition(pos, true);
final outSeqNum = Output<int>();
@ -162,44 +124,44 @@ class _DHTShortArrayWrite extends _DHTShortArrayRead
throw StateError('Element does not exist');
}
_head.freeIndex(pos);
return result;
output?.save(result);
}
@override
Future<bool> tryClear() async {
Future<void> clear() async {
_head.clearIndex();
return true;
}
@override
Future<(Uint8List?, bool)> tryWriteItem(int pos, Uint8List newValue) async {
Future<bool> tryWriteItem(int pos, Uint8List newValue,
{Output<Uint8List>? output}) async {
if (pos < 0 || pos >= _head.length) {
throw IndexError.withLength(pos, _head.length);
}
final lookup = await _head.lookupPosition(pos);
final outSeqNum = Output<int>();
final lookup = await _head.lookupPosition(pos, true);
final outSeqNumRead = Output<int>();
final oldValue = lookup.seq == 0xFFFFFFFF
? null
: await lookup.record
.get(subkey: lookup.recordSubkey, outSeqNum: outSeqNum);
if (outSeqNum.value != null) {
_head.updatePositionSeq(pos, false, outSeqNum.value!);
.get(subkey: lookup.recordSubkey, outSeqNum: outSeqNumRead);
if (outSeqNumRead.value != null) {
_head.updatePositionSeq(pos, false, outSeqNumRead.value!);
}
final outSeqNumWrite = Output<int>();
final result = await lookup.record.tryWriteBytes(newValue,
subkey: lookup.recordSubkey, outSeqNum: outSeqNum);
if (outSeqNum.value != null) {
_head.updatePositionSeq(pos, true, outSeqNum.value!);
subkey: lookup.recordSubkey, outSeqNum: outSeqNumWrite);
if (outSeqNumWrite.value != null) {
_head.updatePositionSeq(pos, true, outSeqNumWrite.value!);
}
if (result != null) {
// A result coming back means the element was overwritten already
return (result, false);
output?.save(result);
return false;
}
return (oldValue, true);
output?.save(oldValue);
return true;
}
}

51
packages/veilid_support/lib/dht_support/src/interfaces/dht_append_truncate.dart Normal file
View File

@ -0,0 +1,51 @@
import 'dart:typed_data';
import 'package:protobuf/protobuf.dart';
import '../../../veilid_support.dart';
////////////////////////////////////////////////////////////////////////////
// Append/truncate interface
abstract class DHTAppendTruncate {
/// Try to add an item to the end of the DHT data structure.
/// Return true if the element was successfully added, and false if the state
/// changed before the element could be added or a newer value was found on
/// the network.
/// This may throw an exception if the number elements added exceeds limits.
Future<bool> tryAppendItem(Uint8List value);
/// Try to add a list of items to the end of the DHT data structure.
/// Return true if the elements were successfully added, and false if the
/// state changed before the element could be added or a newer value was found
/// on the network.
/// This may throw an exception if the number elements added exceeds limits.
Future<bool> tryAppendItems(List<Uint8List> values);
/// Try to remove a number of items from the head of the DHT data structure.
/// Throws StateError if count < 0
Future<void> truncate(int count);
/// Remove all items in the DHT data structure.
Future<void> clear();
}
abstract class DHTAppendTruncateRandomRead
implements DHTAppendTruncate, DHTRandomRead {}
extension DHTAppendTruncateExt on DHTAppendTruncate {
/// Convenience function:
/// Like tryAppendItem but also encodes the input value as JSON and parses the
/// returned element as JSON
Future<bool> tryAppendItemJson<T>(
T newValue,
) =>
tryAppendItem(jsonEncodeBytes(newValue));
/// Convenience function:
/// Like tryAppendItem but also encodes the input value as a protobuf object
/// and parses the returned element as a protobuf object
Future<bool> tryAppendItemProtobuf<T extends GeneratedMessage>(
T newValue,
) =>
tryAppendItem(newValue.writeToBuffer());
}

59
packages/veilid_support/lib/dht_support/src/interfaces/dht_closeable.dart Normal file
View File

@ -0,0 +1,59 @@
import 'dart:async';
import 'package:meta/meta.dart';
abstract class DHTCloseable<C, D> {
bool get isOpen;
@protected
FutureOr<D> scoped();
Future<C> ref();
Future<void> close();
}
abstract class DHTDeleteable<C, D> extends DHTCloseable<C, D> {
Future<void> delete();
}
extension DHTCloseableExt<C, D> on DHTCloseable<C, D> {
/// Runs a closure that guarantees the DHTCloseable
/// will be closed upon exit, even if an uncaught exception is thrown
Future<T> scope<T>(Future<T> Function(D) scopeFunction) async {
if (!isOpen) {
throw StateError('not open in scope');
}
try {
return await scopeFunction(await scoped());
} finally {
await close();
}
}
}
extension DHTDeletableExt<C, D> on DHTDeleteable<C, D> {
/// Runs a closure that guarantees the DHTCloseable
/// will be closed upon exit, and deleted if an an
/// uncaught exception is thrown
Future<T> deleteScope<T>(Future<T> Function(D) scopeFunction) async {
if (!isOpen) {
throw StateError('not open in deleteScope');
}
try {
return await scopeFunction(await scoped());
} on Exception {
await delete();
rethrow;
} finally {
await close();
}
}
/// Scopes a closure that conditionally deletes the DHTCloseable on exit
Future<T> maybeDeleteScope<T>(
bool delete, Future<T> Function(D) scopeFunction) async {
if (delete) {
return deleteScope(scopeFunction);
}
return scope(scopeFunction);
}
}

63
packages/veilid_support/lib/dht_support/src/interfaces/dht_random_read.dart Normal file
View File

@ -0,0 +1,63 @@
import 'dart:typed_data';
import 'package:protobuf/protobuf.dart';
import '../../../veilid_support.dart';
////////////////////////////////////////////////////////////////////////////
// Reader interface
abstract class DHTRandomRead {
/// Returns the number of elements in the DHTArray
/// This number will be >= 0 and <= DHTShortArray.maxElements (256)
int get length;
/// Return the item at position 'pos' in the DHTArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
/// * 'pos' must be >= 0 and < 'length'
Future<Uint8List?> getItem(int pos, {bool forceRefresh = false});
/// Return a list of a range of items in the DHTArray. If 'forceRefresh'
/// is specified, the network will always be checked for newer values
/// rather than returning the existing locally stored copy of the elements.
/// * 'start' must be >= 0
/// * 'len' must be >= 0 and <= DHTShortArray.maxElements (256) and defaults
/// to the maximum length
Future<List<Uint8List>?> getItemRange(int start,
{int? length, bool forceRefresh = false});
/// Get a list of the positions that were written offline and not flushed yet
Future<Set<int>> getOfflinePositions();
}
extension DHTRandomReadExt on DHTRandomRead {
/// Convenience function:
/// Like getItem but also parses the returned element as JSON
Future<T?> getItemJson<T>(T Function(dynamic) fromJson, int pos,
{bool forceRefresh = false}) =>
getItem(pos, forceRefresh: forceRefresh)
.then((out) => jsonDecodeOptBytes(fromJson, out));
/// Convenience function:
/// Like getAllItems but also parses the returned elements as JSON
Future<List<T>?> getItemRangeJson<T>(T Function(dynamic) fromJson, int start,
{int? length, bool forceRefresh = false}) =>
getItemRange(start, length: length, forceRefresh: forceRefresh)
.then((out) => out?.map(fromJson).toList());
/// Convenience function:
/// Like getItem but also parses the returned element as a protobuf object
Future<T?> getItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int pos,
{bool forceRefresh = false}) =>
getItem(pos, forceRefresh: forceRefresh)
.then((out) => (out == null) ? null : fromBuffer(out));
/// Convenience function:
/// Like getAllItems but also parses the returned elements as protobuf objects
Future<List<T>?> getItemRangeProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int start,
{int? length, bool forceRefresh = false}) =>
getItemRange(start, length: length, forceRefresh: forceRefresh)
.then((out) => out?.map(fromBuffer).toList());
}

119
packages/veilid_support/lib/dht_support/src/interfaces/dht_random_write.dart Normal file
View File

@ -0,0 +1,119 @@
import 'dart:typed_data';
import 'package:protobuf/protobuf.dart';
import '../../../veilid_support.dart';
////////////////////////////////////////////////////////////////////////////
// Writer interface
abstract class DHTRandomWrite {
/// Try to set an item at position 'pos' of the DHTArray.
/// If the set was successful this returns:
/// * A boolean true
/// * outValue will return the prior contents of the element,
/// or null if there was no value yet
///
/// If the set was found a newer value on the network this returns:
/// * A boolean false
/// * outValue will return the newer value of the element,
/// or null if the head record changed.
///
/// This may throw an exception if the position exceeds the built-in limit of
/// 'maxElements = 256' entries.
Future<bool> tryWriteItem(int pos, Uint8List newValue,
{Output<Uint8List>? output});
/// Try to add an item to the end of the DHTArray. Return true if the
/// element was successfully added, and false if the state changed before
/// the element could be added or a newer value was found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryAddItem(Uint8List value);
/// Try to add a list of items to the end of the DHTArray. Return true if the
/// elements were successfully added, and false if the state changed before
/// the elements could be added or a newer value was found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryAddItems(List<Uint8List> values);
/// Try to insert an item as position 'pos' of the DHTArray.
/// Return true if the element was successfully inserted, and false if the
/// state changed before the element could be inserted or a newer value was
/// found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryInsertItem(int pos, Uint8List value);
/// Try to insert items at position 'pos' of the DHTArray.
/// Return true if the elements were successfully inserted, and false if the
/// state changed before the elements could be inserted or a newer value was
/// found on the network.
/// This may throw an exception if the number elements added exceeds the
/// built-in limit of 'maxElements = 256' entries.
Future<bool> tryInsertItems(int pos, List<Uint8List> values);
/// Swap items at position 'aPos' and 'bPos' in the DHTArray.
/// Throws IndexError if either of the positions swapped exceed
/// the length of the list
Future<void> swapItem(int aPos, int bPos);
/// Remove an item at position 'pos' in the DHTArray.
/// If the remove was successful this returns:
/// * outValue will return the prior contents of the element
/// Throws IndexError if the position removed exceeds the length of
/// the list.
Future<void> removeItem(int pos, {Output<Uint8List>? output});
/// Remove all items in the DHTShortArray.
Future<void> clear();
}
extension DHTRandomWriteExt on DHTRandomWrite {
/// Convenience function:
/// Like tryWriteItem but also encodes the input value as JSON and parses the
/// returned element as JSON
Future<bool> tryWriteItemJson<T>(
T Function(dynamic) fromJson, int pos, T newValue,
{Output<T>? output}) async {
final outValueBytes = output == null ? null : Output<Uint8List>();
final out = await tryWriteItem(pos, jsonEncodeBytes(newValue),
output: outValueBytes);
output.mapSave(outValueBytes, (b) => jsonDecodeBytes(fromJson, b));
return out;
}
/// Convenience function:
/// Like tryWriteItem but also encodes the input value as a protobuf object
/// and parses the returned element as a protobuf object
Future<bool> tryWriteItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int pos, T newValue,
{Output<T>? output}) async {
final outValueBytes = output == null ? null : Output<Uint8List>();
final out = await tryWriteItem(pos, newValue.writeToBuffer(),
output: outValueBytes);
output.mapSave(outValueBytes, fromBuffer);
return out;
}
/// Convenience function:
/// Like removeItem but also parses the returned element as JSON
Future<void> removeItemJson<T>(T Function(dynamic) fromJson, int pos,
{Output<T>? output}) async {
final outValueBytes = output == null ? null : Output<Uint8List>();
await removeItem(pos, output: outValueBytes);
output.mapSave(outValueBytes, (b) => jsonDecodeBytes(fromJson, b));
}
/// Convenience function:
/// Like removeItem but also parses the returned element as JSON
Future<void> removeItemProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int pos,
{Output<T>? output}) async {
final outValueBytes = output == null ? null : Output<Uint8List>();
await removeItem(pos, output: outValueBytes);
output.mapSave(outValueBytes, fromBuffer);
}
}
abstract class DHTRandomReadWrite implements DHTRandomRead, DHTRandomWrite {}

5
packages/veilid_support/lib/dht_support/src/interfaces/exceptions.dart Normal file
View File

@ -0,0 +1,5 @@
class DHTExceptionTryAgain implements Exception {
DHTExceptionTryAgain(
[this.cause = 'operation failed due to newer dht value']);
String cause;
}

4
packages/veilid_support/lib/dht_support/src/interfaces/interfaces.dart Normal file
View File

@ -0,0 +1,4 @@
export 'dht_closeable.dart';
export 'dht_random_read.dart';
export 'dht_random_write.dart';
export 'exceptions.dart';

124
packages/veilid_support/lib/proto/dht.pb.dart
View File

@ -83,6 +83,68 @@ class DHTData extends $pb.GeneratedMessage {
void clearSize() => clearField(4);
}
class DHTLog extends $pb.GeneratedMessage {
factory DHTLog() => create();
DHTLog._() : super();
factory DHTLog.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory DHTLog.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DHTLog', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
..a<$core.int>(1, _omitFieldNames ? '' : 'head', $pb.PbFieldType.OU3)
..a<$core.int>(2, _omitFieldNames ? '' : 'tail', $pb.PbFieldType.OU3)
..a<$core.int>(3, _omitFieldNames ? '' : 'stride', $pb.PbFieldType.OU3)
..hasRequiredFields = false
;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
DHTLog clone() => DHTLog()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
DHTLog copyWith(void Function(DHTLog) updates) => super.copyWith((message) => updates(message as DHTLog)) as DHTLog;
$pb.BuilderInfo get info_ => _i;
@$core.pragma('dart2js:noInline')
static DHTLog create() => DHTLog._();
DHTLog createEmptyInstance() => create();
static $pb.PbList<DHTLog> createRepeated() => $pb.PbList<DHTLog>();
@$core.pragma('dart2js:noInline')
static DHTLog getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<DHTLog>(create);
static DHTLog? _defaultInstance;
@$pb.TagNumber(1)
$core.int get head => $_getIZ(0);
@$pb.TagNumber(1)
set head($core.int v) { $_setUnsignedInt32(0, v); }
@$pb.TagNumber(1)
$core.bool hasHead() => $_has(0);
@$pb.TagNumber(1)
void clearHead() => clearField(1);
@$pb.TagNumber(2)
$core.int get tail => $_getIZ(1);
@$pb.TagNumber(2)
set tail($core.int v) { $_setUnsignedInt32(1, v); }
@$pb.TagNumber(2)
$core.bool hasTail() => $_has(1);
@$pb.TagNumber(2)
void clearTail() => clearField(2);
@$pb.TagNumber(3)
$core.int get stride => $_getIZ(2);
@$pb.TagNumber(3)
set stride($core.int v) { $_setUnsignedInt32(2, v); }
@$pb.TagNumber(3)
$core.bool hasStride() => $_has(2);
@$pb.TagNumber(3)
void clearStride() => clearField(3);
}
class DHTShortArray extends $pb.GeneratedMessage {
factory DHTShortArray() => create();
DHTShortArray._() : super();
@ -133,68 +195,6 @@ class DHTShortArray extends $pb.GeneratedMessage {
$core.List<$core.int> get seqs => $_getList(2);
}
class DHTLog extends $pb.GeneratedMessage {
factory DHTLog() => create();
DHTLog._() : super();
factory DHTLog.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory DHTLog.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DHTLog', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
..pc<$0.TypedKey>(1, _omitFieldNames ? '' : 'keys', $pb.PbFieldType.PM, subBuilder: $0.TypedKey.create)
..aOM<$0.TypedKey>(2, _omitFieldNames ? '' : 'back', subBuilder: $0.TypedKey.create)
..p<$core.int>(3, _omitFieldNames ? '' : 'subkeyCounts', $pb.PbFieldType.KU3)
..a<$core.int>(4, _omitFieldNames ? '' : 'totalSubkeys', $pb.PbFieldType.OU3)
..hasRequiredFields = false
;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
DHTLog clone() => DHTLog()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
DHTLog copyWith(void Function(DHTLog) updates) => super.copyWith((message) => updates(message as DHTLog)) as DHTLog;
$pb.BuilderInfo get info_ => _i;
@$core.pragma('dart2js:noInline')
static DHTLog create() => DHTLog._();
DHTLog createEmptyInstance() => create();
static $pb.PbList<DHTLog> createRepeated() => $pb.PbList<DHTLog>();
@$core.pragma('dart2js:noInline')
static DHTLog getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<DHTLog>(create);
static DHTLog? _defaultInstance;
@$pb.TagNumber(1)
$core.List<$0.TypedKey> get keys => $_getList(0);
@$pb.TagNumber(2)
$0.TypedKey get back => $_getN(1);
@$pb.TagNumber(2)
set back($0.TypedKey v) { setField(2, v); }
@$pb.TagNumber(2)
$core.bool hasBack() => $_has(1);
@$pb.TagNumber(2)
void clearBack() => clearField(2);
@$pb.TagNumber(2)
$0.TypedKey ensureBack() => $_ensure(1);
@$pb.TagNumber(3)
$core.List<$core.int> get subkeyCounts => $_getList(2);
@$pb.TagNumber(4)
$core.int get totalSubkeys => $_getIZ(3);
@$pb.TagNumber(4)
set totalSubkeys($core.int v) { $_setUnsignedInt32(3, v); }
@$pb.TagNumber(4)
$core.bool hasTotalSubkeys() => $_has(3);
@$pb.TagNumber(4)
void clearTotalSubkeys() => clearField(4);
}
enum DataReference_Kind {
dhtData,
notSet

32
packages/veilid_support/lib/proto/dht.pbjson.dart
View File

@ -30,6 +30,21 @@ final $typed_data.Uint8List dHTDataDescriptor = $convert.base64Decode(
'gCIAEoCzIQLnZlaWxpZC5UeXBlZEtleVIEaGFzaBIUCgVjaHVuaxgDIAEoDVIFY2h1bmsSEgoE'
'c2l6ZRgEIAEoDVIEc2l6ZQ==');
@$core.Deprecated('Use dHTLogDescriptor instead')
const DHTLog$json = {
'1': 'DHTLog',
'2': [
{'1': 'head', '3': 1, '4': 1, '5': 13, '10': 'head'},
{'1': 'tail', '3': 2, '4': 1, '5': 13, '10': 'tail'},
{'1': 'stride', '3': 3, '4': 1, '5': 13, '10': 'stride'},
],
};
/// Descriptor for `DHTLog`. Decode as a `google.protobuf.DescriptorProto`.
final $typed_data.Uint8List dHTLogDescriptor = $convert.base64Decode(
'CgZESFRMb2cSEgoEaGVhZBgBIAEoDVIEaGVhZBISCgR0YWlsGAIgASgNUgR0YWlsEhYKBnN0cm'
'lkZRgDIAEoDVIGc3RyaWRl');
@$core.Deprecated('Use dHTShortArrayDescriptor instead')
const DHTShortArray$json = {
'1': 'DHTShortArray',
@ -45,23 +60,6 @@ final $typed_data.Uint8List dHTShortArrayDescriptor = $convert.base64Decode(
'Cg1ESFRTaG9ydEFycmF5EiQKBGtleXMYASADKAsyEC52ZWlsaWQuVHlwZWRLZXlSBGtleXMSFA'
'oFaW5kZXgYAiABKAxSBWluZGV4EhIKBHNlcXMYAyADKA1SBHNlcXM=');
@$core.Deprecated('Use dHTLogDescriptor instead')
const DHTLog$json = {
'1': 'DHTLog',
'2': [
{'1': 'keys', '3': 1, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'keys'},
{'1': 'back', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'back'},
{'1': 'subkey_counts', '3': 3, '4': 3, '5': 13, '10': 'subkeyCounts'},
{'1': 'total_subkeys', '3': 4, '4': 1, '5': 13, '10': 'totalSubkeys'},
],
};
/// Descriptor for `DHTLog`. Decode as a `google.protobuf.DescriptorProto`.
final $typed_data.Uint8List dHTLogDescriptor = $convert.base64Decode(
'CgZESFRMb2cSJAoEa2V5cxgBIAMoCzIQLnZlaWxpZC5UeXBlZEtleVIEa2V5cxIkCgRiYWNrGA'
'IgASgLMhAudmVpbGlkLlR5cGVkS2V5UgRiYWNrEiMKDXN1YmtleV9jb3VudHMYAyADKA1SDHN1'
'YmtleUNvdW50cxIjCg10b3RhbF9zdWJrZXlzGAQgASgNUgx0b3RhbFN1YmtleXM=');
@$core.Deprecated('Use dataReferenceDescriptor instead')
const DataReference$json = {
'1': 'DataReference',

4
packages/veilid_support/lib/src/identity.dart
View File

@ -300,8 +300,8 @@ Future<IdentityMaster> openIdentityMaster(
debugName:
'IdentityMaster::openIdentityMaster::IdentityMasterRecord'))
.deleteScope((masterRec) async {
final identityMaster =
(await masterRec.getJson(IdentityMaster.fromJson, forceRefresh: true))!;
final identityMaster = (await masterRec.getJson(IdentityMaster.fromJson,
refreshMode: DHTRecordRefreshMode.network))!;
// Validate IdentityMaster
final masterRecordKey = masterRec.key;

33
packages/veilid_support/lib/src/output.dart Normal file
View File

@ -0,0 +1,33 @@
import 'package:fast_immutable_collections/fast_immutable_collections.dart';
export 'package:fast_immutable_collections/fast_immutable_collections.dart'
show Output;
extension OutputNullExt<T> on Output<T>? {
void mapSave<S>(Output<S>? other, T Function(S output) closure) {
if (this == null) {
return;
}
if (other == null) {
return;
}
final v = other.value;
if (v == null) {
return;
}
return this!.save(closure(v));
}
}
extension OutputExt<T> on Output<T> {
void mapSave<S>(Output<S>? other, T Function(S output) closure) {
if (other == null) {
return;
}
final v = other.value;
if (v == null) {
return;
}
return save(closure(v));
}
}

1
packages/veilid_support/lib/veilid_support.dart
View File

@ -10,6 +10,7 @@ export 'src/config.dart';
export 'src/identity.dart';
export 'src/json_tools.dart';
export 'src/memory_tools.dart';
export 'src/output.dart';
export 'src/persistent_queue.dart';
export 'src/protobuf_tools.dart';
export 'src/table_db.dart';

22
packages/veilid_support/pubspec.lock
View File

@ -85,10 +85,10 @@ packages:
dependency: transitive
description:
name: build_daemon
sha256: "0343061a33da9c5810b2d6cee51945127d8f4c060b7fbdd9d54917f0a3feaaa1"
sha256: "79b2aef6ac2ed00046867ed354c88778c9c0f029df8a20fe10b5436826721ef9"
url: "https://pub.dev"
source: hosted
version: "4.0.1"
version: "4.0.2"
build_resolvers:
dependency: transitive
description:
@ -101,10 +101,10 @@ packages:
dependency: "direct dev"
description:
name: build_runner
sha256: "3ac61a79bfb6f6cc11f693591063a7f19a7af628dc52f141743edac5c16e8c22"
sha256: "1414d6d733a85d8ad2f1dfcb3ea7945759e35a123cb99ccfac75d0758f75edfa"
url: "https://pub.dev"
source: hosted
version: "2.4.9"
version: "2.4.10"
build_runner_core:
dependency: transitive
description:
@ -221,10 +221,10 @@ packages:
dependency: "direct main"
description:
name: fast_immutable_collections
sha256: "38fbc50df5b219dcfb83ebbc3275ec09872530ca1153858fc56fceadb310d037"
sha256: "533806a7f0c624c2e479d05d3fdce4c87109a7cd0db39b8cc3830d3a2e8dedc7"
url: "https://pub.dev"
source: hosted
version: "10.2.2"
version: "10.2.3"
ffi:
dependency: transitive
description:
@ -399,10 +399,10 @@ packages:
dependency: "direct main"
description:
name: meta
sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04
sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136"
url: "https://pub.dev"
source: hosted
version: "1.11.0"
version: "1.12.0"
mime:
dependency: transitive
description:
@ -763,10 +763,10 @@ packages:
dependency: transitive
description:
name: win32
sha256: "0eaf06e3446824099858367950a813472af675116bf63f008a4c2a75ae13e9cb"
sha256: a79dbe579cb51ecd6d30b17e0cae4e0ea15e2c0e66f69ad4198f22a6789e94f4
url: "https://pub.dev"
source: hosted
version: "5.5.0"
version: "5.5.1"
xdg_directories:
dependency: transitive
description:
@ -784,5 +784,5 @@ packages:
source: hosted
version: "3.1.2"
sdks:
dart: ">=3.3.0 <4.0.0"
dart: ">=3.4.0 <4.0.0"
flutter: ">=3.19.1"

6
packages/veilid_support/pubspec.yaml
View File

@ -12,11 +12,11 @@ dependencies:
bloc_advanced_tools: ^0.1.1
collection: ^1.18.0
equatable: ^2.0.5
fast_immutable_collections: ^10.2.2
fast_immutable_collections: ^10.2.3
freezed_annotation: ^2.4.1
json_annotation: ^4.9.0
loggy: ^2.0.3
meta: ^1.11.0
meta: ^1.12.0
protobuf: ^3.1.0
veilid:
@ -24,7 +24,7 @@ dependencies:
path: ../../../veilid/veilid-flutter
dev_dependencies:
build_runner: ^2.4.9
build_runner: ^2.4.10
freezed: ^2.5.2
json_serializable: ^6.8.0
lint_hard: ^4.0.0

134
pubspec.lock
View File

@ -37,10 +37,10 @@ packages:
dependency: "direct main"
description:
name: archive
sha256: "22600aa1e926be775fa5fe7e6894e7fb3df9efda8891c73f70fb3262399a432d"
sha256: ecf4273855368121b1caed0d10d4513c7241dfc813f7d3c8933b36622ae9b265
url: "https://pub.dev"
source: hosted
version: "3.4.10"
version: "3.5.1"
args:
dependency: transitive
description:
@ -68,10 +68,10 @@ packages:
dependency: "direct main"
description:
name: awesome_extensions
sha256: c3bf11d07a69fe10ff5541717b920661c7a87a791ee182851f1c92a2d15b95a2
sha256: "07e52221467e651cab9219a26286245760831c3852ea2c54883a48a54f120d7c"
url: "https://pub.dev"
source: hosted
version: "2.0.14"
version: "2.0.16"
badges:
dependency: "direct main"
description:
@ -139,10 +139,10 @@ packages:
dependency: transitive
description:
name: build_daemon
sha256: "0343061a33da9c5810b2d6cee51945127d8f4c060b7fbdd9d54917f0a3feaaa1"
sha256: "79b2aef6ac2ed00046867ed354c88778c9c0f029df8a20fe10b5436826721ef9"
url: "https://pub.dev"
source: hosted
version: "4.0.1"
version: "4.0.2"
build_resolvers:
dependency: transitive
description:
@ -155,10 +155,10 @@ packages:
dependency: "direct dev"
description:
name: build_runner
sha256: "3ac61a79bfb6f6cc11f693591063a7f19a7af628dc52f141743edac5c16e8c22"
sha256: "1414d6d733a85d8ad2f1dfcb3ea7945759e35a123cb99ccfac75d0758f75edfa"
url: "https://pub.dev"
source: hosted
version: "2.4.9"
version: "2.4.10"
build_runner_core:
dependency: transitive
description:
@ -203,34 +203,34 @@ packages:
dependency: transitive
description:
name: cached_network_image_web
sha256: "42a835caa27c220d1294311ac409a43361088625a4f23c820b006dd9bffb3316"
sha256: "205d6a9f1862de34b93184f22b9d2d94586b2f05c581d546695e3d8f6a805cd7"
url: "https://pub.dev"
source: hosted
version: "1.1.1"
version: "1.2.0"
camera:
dependency: transitive
description:
name: camera
sha256: "9499cbc2e51d8eb0beadc158b288380037618ce4e30c9acbc4fae1ac3ecb5797"
sha256: dfa8fc5a1adaeb95e7a54d86a5bd56f4bb0e035515354c8ac6d262e35cec2ec8
url: "https://pub.dev"
source: hosted
version: "0.10.5+9"
version: "0.10.6"
camera_android:
dependency: transitive
description:
name: camera_android
sha256: "7b0aba6398afa8475e2bc9115d976efb49cf8db781e922572d443795c04a4f4f"
sha256: b350ac087f111467e705b2b76cc1322f7f5bdc122aa83b4b243b0872f390d229
url: "https://pub.dev"
source: hosted
version: "0.10.9+1"
version: "0.10.9+2"
camera_avfoundation:
dependency: transitive
description:
name: camera_avfoundation
sha256: "9dbbb253aaf201a69c40cf95571f366ca936305d2de012684e21f6f1b1433d31"
sha256: "7d021e8cd30d9b71b8b92b4ad669e80af432d722d18d6aac338572754a786c15"
url: "https://pub.dev"
source: hosted
version: "0.9.15+4"
version: "0.9.16"
camera_platform_interface:
dependency: transitive
description:
@ -387,10 +387,10 @@ packages:
dependency: transitive
description:
name: diffutil_dart
sha256: e0297e4600b9797edff228ed60f4169a778ea357691ec98408fa3b72994c7d06
sha256: "5e74883aedf87f3b703cb85e815bdc1ed9208b33501556e4a8a5572af9845c81"
url: "https://pub.dev"
source: hosted
version: "3.0.0"
version: "4.0.1"
equatable:
dependency: "direct main"
description:
@ -403,10 +403,10 @@ packages:
dependency: "direct main"
description:
name: fast_immutable_collections
sha256: "38fbc50df5b219dcfb83ebbc3275ec09872530ca1153858fc56fceadb310d037"
sha256: "533806a7f0c624c2e479d05d3fdce4c87109a7cd0db39b8cc3830d3a2e8dedc7"
url: "https://pub.dev"
source: hosted
version: "10.2.2"
version: "10.2.3"
ffi:
dependency: transitive
description:
@ -456,10 +456,10 @@ packages:
dependency: transitive
description:
name: flutter_cache_manager
sha256: "8207f27539deb83732fdda03e259349046a39a4c767269285f449ade355d54ba"
sha256: "395d6b7831f21f3b989ebedbb785545932adb9afe2622c1ffacf7f4b53a7e544"
url: "https://pub.dev"
source: hosted
version: "3.3.1"
version: "3.3.2"
flutter_chat_types:
dependency: "direct main"
description:
@ -472,10 +472,10 @@ packages:
dependency: "direct main"
description:
name: flutter_chat_ui
sha256: c8580c85e2d29359ffc84147e643d08d883eb6e757208652377f0105ef58807f
sha256: "40fb37acc328dd179eadc3d67bf8bd2d950dc0da34464aa8d48e8707e0234c09"
url: "https://pub.dev"
source: hosted
version: "1.6.12"
version: "1.6.13"
flutter_form_builder:
dependency: "direct main"
description:
@ -634,10 +634,10 @@ packages:
dependency: "direct main"
description:
name: go_router
sha256: "771c8feb40ad0ef639973d7ecf1b43d55ffcedb2207fd43fab030f5639e40446"
sha256: aa073287b8f43553678e6fa9e8bb9c83212ff76e09542129a8099bbc8db4df65
url: "https://pub.dev"
source: hosted
version: "13.2.4"
version: "14.1.2"
graphs:
dependency: transitive
description:
@ -714,10 +714,10 @@ packages:
dependency: "direct main"
description:
name: intl
sha256: "3bc132a9dbce73a7e4a21a17d06e1878839ffbf975568bc875c60537824b0c4d"
sha256: d6f56758b7d3014a48af9701c085700aac781a92a87a62b1333b46d8879661cf
url: "https://pub.dev"
source: hosted
version: "0.18.1"
version: "0.19.0"
io:
dependency: transitive
description:
@ -802,10 +802,10 @@ packages:
dependency: "direct main"
description:
name: meta
sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04
sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136"
url: "https://pub.dev"
source: hosted
version: "1.11.0"
version: "1.12.0"
mime:
dependency: transitive
description:
@ -818,10 +818,10 @@ packages:
dependency: "direct main"
description:
name: mobile_scanner
sha256: "827765afbd4792ff3fd105ad593821ac0f6d8a7d352689013b07ee85be336312"
sha256: b8c0e9afcfd52534f85ec666f3d52156f560b5e6c25b1e3d4fe2087763607926
url: "https://pub.dev"
source: hosted
version: "4.0.1"
version: "5.1.1"
motion_toast:
dependency: "direct main"
description:
@ -898,10 +898,10 @@ packages:
dependency: transitive
description:
name: path_provider_foundation
sha256: "5a7999be66e000916500be4f15a3633ebceb8302719b47b9cc49ce924125350f"
sha256: f234384a3fdd67f989b4d54a5d73ca2a6c422fa55ae694381ae0f4375cd1ea16
url: "https://pub.dev"
source: hosted
version: "2.3.2"
version: "2.4.0"
path_provider_linux:
dependency: transitive
description:
@ -938,10 +938,10 @@ packages:
dependency: transitive
description:
name: photo_view
sha256: "8036802a00bae2a78fc197af8a158e3e2f7b500561ed23b4c458107685e645bb"
sha256: "1fc3d970a91295fbd1364296575f854c9863f225505c28c46e0a03e48960c75e"
url: "https://pub.dev"
source: hosted
version: "0.14.0"
version: "0.15.0"
pinput:
dependency: "direct main"
description:
@ -970,10 +970,10 @@ packages:
dependency: transitive
description:
name: pointycastle
sha256: "79fbafed02cfdbe85ef3fd06c7f4bc2cbcba0177e61b765264853d4253b21744"
sha256: "4be0097fcf3fd3e8449e53730c631200ebc7b88016acecab2b0da2f0149222fe"
url: "https://pub.dev"
source: hosted
version: "3.9.0"
version: "3.9.1"
pool:
dependency: transitive
description:
@ -1106,26 +1106,26 @@ packages:
dependency: "direct main"
description:
name: searchable_listview
sha256: d8513a968bdd540cb011220a5670b23b346e04a7bcb99690a859ed58092f72a4
sha256: dfa6358f5e097f45b5b51a160cb6189e112e3abe0f728f4740349cd3b6575617
url: "https://pub.dev"
source: hosted
version: "2.11.2"
version: "2.13.0"
share_plus:
dependency: "direct main"
description:
name: share_plus
sha256: fb5319f3aab4c5dda5ebb92dca978179ba21f8c783ee4380910ef4c1c6824f51
sha256: ef3489a969683c4f3d0239010cc8b7a2a46543a8d139e111c06c558875083544
url: "https://pub.dev"
source: hosted
version: "8.0.3"
version: "9.0.0"
share_plus_platform_interface:
dependency: transitive
description:
name: share_plus_platform_interface
sha256: "251eb156a8b5fa9ce033747d73535bf53911071f8d3b6f4f0b578505ce0d4496"
sha256: "0f9e4418835d1b2c3ae78fdb918251959106cefdbc4dd43526e182f80e82f6d4"
url: "https://pub.dev"
source: hosted
version: "3.4.0"
version: "4.0.0"
shared_preferences:
dependency: "direct main"
description:
@ -1146,10 +1146,10 @@ packages:
dependency: transitive
description:
name: shared_preferences_foundation
sha256: "7708d83064f38060c7b39db12aefe449cb8cdc031d6062280087bc4cdb988f5c"
sha256: "0a8a893bf4fd1152f93fec03a415d11c27c74454d96e2318a7ac38dd18683ab7"
url: "https://pub.dev"
source: hosted
version: "2.3.5"
version: "2.4.0"
shared_preferences_linux:
dependency: transitive
description:
@ -1194,10 +1194,10 @@ packages:
dependency: transitive
description:
name: shelf_web_socket
sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1"
sha256: "073c147238594ecd0d193f3456a5fe91c4b0abbcc68bf5cd95b36c4e194ac611"
url: "https://pub.dev"
source: hosted
version: "1.0.4"
version: "2.0.0"
signal_strength_indicator:
dependency: "direct main"
description:
@ -1263,10 +1263,10 @@ packages:
dependency: transitive
description:
name: sqflite
sha256: "5ce2e1a15e822c3b4bfb5400455775e421da7098eed8adc8f26298ada7c9308c"
sha256: a43e5a27235518c03ca238e7b4732cf35eabe863a369ceba6cbefa537a66f16d
url: "https://pub.dev"
source: hosted
version: "2.3.3"
version: "2.3.3+1"
sqflite_common:
dependency: transitive
description:
@ -1399,18 +1399,18 @@ packages:
dependency: transitive
description:
name: url_launcher_android
sha256: "360a6ed2027f18b73c8d98e159dda67a61b7f2e0f6ec26e86c3ada33b0621775"
sha256: "17cd5e205ea615e2c6ea7a77323a11712dffa0720a8a90540db57a01347f9ad9"
url: "https://pub.dev"
source: hosted
version: "6.3.1"
version: "6.3.2"
url_launcher_ios:
dependency: transitive
description:
name: url_launcher_ios
sha256: "9149d493b075ed740901f3ee844a38a00b33116c7c5c10d7fb27df8987fb51d5"
sha256: "7068716403343f6ba4969b4173cbf3b84fc768042124bc2c011e5d782b24fe89"
url: "https://pub.dev"
source: hosted
version: "6.2.5"
version: "6.3.0"
url_launcher_linux:
dependency: transitive
description:
@ -1423,10 +1423,10 @@ packages:
dependency: transitive
description:
name: url_launcher_macos
sha256: b7244901ea3cf489c5335bdacda07264a6e960b1c1b1a9f91e4bc371d9e68234
sha256: "9a1a42d5d2d95400c795b2914c36fdcb525870c752569438e4ebb09a2b5d90de"
url: "https://pub.dev"
source: hosted
version: "3.1.0"
version: "3.2.0"
url_launcher_platform_interface:
dependency: transitive
description:
@ -1529,30 +1529,38 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.5.1"
web_socket:
dependency: transitive
description:
name: web_socket
sha256: "217f49b5213796cb508d6a942a5dc604ce1cb6a0a6b3d8cb3f0c314f0ecea712"
url: "https://pub.dev"
source: hosted
version: "0.1.4"
web_socket_channel:
dependency: transitive
description:
name: web_socket_channel
sha256: "58c6666b342a38816b2e7e50ed0f1e261959630becd4c879c4f26bfa14aa5a42"
sha256: a2d56211ee4d35d9b344d9d4ce60f362e4f5d1aafb988302906bd732bc731276
url: "https://pub.dev"
source: hosted
version: "2.4.5"
version: "3.0.0"
win32:
dependency: transitive
description:
name: win32
sha256: "0a989dc7ca2bb51eac91e8fd00851297cfffd641aa7538b165c62637ca0eaa4a"
sha256: a79dbe579cb51ecd6d30b17e0cae4e0ea15e2c0e66f69ad4198f22a6789e94f4
url: "https://pub.dev"
source: hosted
version: "5.4.0"
version: "5.5.1"
window_manager:
dependency: "direct main"
description:
name: window_manager
sha256: b3c895bdf936c77b83c5254bec2e6b3f066710c1f89c38b20b8acc382b525494
sha256: "8699323b30da4cdbe2aa2e7c9de567a6abd8a97d9a5c850a3c86dcd0b34bbfbf"
url: "https://pub.dev"
source: hosted
version: "0.3.8"
version: "0.3.9"
xdg_directories:
dependency: transitive
description:
@ -1610,5 +1618,5 @@ packages:
source: hosted
version: "1.1.2"
sdks:
dart: ">=3.3.0 <4.0.0"
dart: ">=3.4.0 <4.0.0"
flutter: ">=3.19.1"

17
pubspec.yaml
View File

@ -10,7 +10,7 @@ environment:
dependencies:
animated_theme_switcher: ^2.0.10
ansicolor: ^2.0.2
archive: ^3.4.10
archive: ^3.5.1
async_tools: ^0.1.1
awesome_extensions: ^2.0.14
badges: ^3.1.2
@ -44,14 +44,14 @@ dependencies:
flutter_translate: ^4.0.4
form_builder_validators: ^9.1.0
freezed_annotation: ^2.4.1
go_router: ^13.2.4
go_router: ^14.1.2
hydrated_bloc: ^9.1.5
image: ^4.1.7
intl: ^0.18.1
json_annotation: ^4.8.1
json_annotation: ^4.9.0
loggy: ^2.0.3
meta: ^1.11.0
mobile_scanner: ^4.0.1
mobile_scanner: ^5.1.1
motion_toast: ^2.9.1
pasteboard: ^0.2.0
path: ^1.9.0
@ -65,8 +65,8 @@ dependencies:
quickalert: ^1.1.0
radix_colors: ^1.0.4
reorderable_grid: ^1.0.10
searchable_listview: ^2.11.2
share_plus: ^8.0.3
searchable_listview: ^2.12.0
share_plus: ^9.0.0
shared_preferences: ^2.2.3
signal_strength_indicator: ^0.4.1
split_view: ^3.2.1
@ -88,12 +88,15 @@ dependency_overrides:
path: ../dart_async_tools
bloc_advanced_tools:
path: ../bloc_advanced_tools
# REMOVE ONCE form_builder_validators HAS A FIX UPSTREAM
intl: 0.19.0
dev_dependencies:
build_runner: ^2.4.9
freezed: ^2.5.2
icons_launcher: ^2.1.7
json_serializable: ^6.7.1
json_serializable: ^6.8.0
lint_hard: ^4.0.0
flutter_native_splash: