Git-Mirrors/veilidchat
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilidchat.git synced 2024-09-19 13:05:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

debugging

Browse Source
This commit is contained in:
Christien Rioux 2024-06-02 11:04:19 -04:00
parent f780a60d69
commit 0e4606f35e
20 changed files with 521 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
assets/i18n/en.json
View File

@ -67,6 +67,7 @@
"new_chat": "New Chat"
},
"chat": {
"start_a_conversation": "Start A Conversation",
"say_something": "Say Something"
},
"create_invitation_dialog": {

38
lib/chat/cubits/reconciliation/author_input_queue.dart
View File

@ -18,7 +18,7 @@ class AuthorInputQueue {
_onError = onError,
_inputSource = inputSource,
_outputPosition = outputPosition,
_lastMessage = outputPosition?.message,
_lastMessage = outputPosition?.message.content,
_messageIntegrity = messageIntegrity,
_currentPosition = inputSource.currentWindow.last;
@ -43,8 +43,8 @@ class AuthorInputQueue {
////////////////////////////////////////////////////////////////////////////
// Public interface
// Check if there are no messages in this queue to reconcile
bool get isEmpty => _currentMessage == null;
// Check if there are no messages left in this queue to reconcile
bool get isDone => _isDone;
// Get the current message that needs reconciliation
proto.Message? get current => _currentMessage;
@ -58,6 +58,9 @@ class AuthorInputQueue {
// Remove a reconciled message and move to the next message
// Returns true if there is more work to do
Future<bool> consume() async {
if (_isDone) {
return false;
}
while (true) {
_lastMessage = _currentMessage;
@ -66,6 +69,7 @@ class AuthorInputQueue {
// Get more window if we need to
if (!await _updateWindow()) {
// Window is not available so this queue can't work right now
_isDone = true;
return false;
}
final nextMessage = _inputSource.currentWindow
@ -73,9 +77,9 @@ class AuthorInputQueue {
// Drop the 'offline' elements because we don't reconcile
// anything until it has been confirmed to be committed to the DHT
if (nextMessage.isOffline) {
continue;
}
// if (nextMessage.isOffline) {
// continue;
// }
if (_lastMessage != null) {
// Ensure the timestamp is not moving backward
@ -112,7 +116,7 @@ class AuthorInputQueue {
outer:
while (true) {
// Iterate through current window backward
for (var i = _inputSource.currentWindow.elements.length;
for (var i = _inputSource.currentWindow.elements.length - 1;
i >= 0 && _currentPosition >= 0;
i--, _currentPosition--) {
final elem = _inputSource.currentWindow.elements[i];
@ -134,13 +138,24 @@ class AuthorInputQueue {
if (!await _updateWindow()) {
// Window is not available or things are empty so this
// queue can't work right now
_isDone = true;
return false;
}
}
// The current position should be equal to the first message to process
// and the current window to process should not be empty
return _inputSource.currentWindow.elements.isNotEmpty;
// _currentPosition points to either before the input source starts
// or the position of the previous element. We still need to set the
// _currentMessage to the previous element so consume() can compare
// against it if we can.
if (_currentPosition >= 0) {
_currentMessage = _inputSource.currentWindow
.elements[_currentPosition - _inputSource.currentWindow.first].value;
}
// After this consume(), the currentPosition and _currentMessage should
// be equal to the first message to process and the current window to
// process should not be empty
return consume();
}
// Slide the window toward the current position and load the batch around it
@ -186,6 +201,9 @@ class AuthorInputQueue {
int _currentPosition;
// The current message we're looking at
proto.Message? _currentMessage;
// If we have reached the end
bool _isDone = false;
// Desired maximum window length
static const int _maxWindowLength = 256;
}

7
lib/chat/cubits/reconciliation/author_input_source.dart
View File

@ -21,9 +21,10 @@ class AuthorInputSource {
{required DHTLogStateData<proto.Message> cubitState,
required this.cubit}) {
_currentWindow = InputWindow(
elements: cubitState.elements,
first: cubitState.tail - cubitState.elements.length,
last: cubitState.tail - 1);
elements: cubitState.window,
first: (cubitState.windowTail - cubitState.window.length) %
cubitState.length,
last: (cubitState.windowTail - 1) % cubitState.length);
}
////////////////////////////////////////////////////////////////////////////

38
lib/chat/cubits/reconciliation/message_reconciliation.dart
View File

@ -12,7 +12,7 @@ import 'output_position.dart';
class MessageReconciliation {
MessageReconciliation(
{required TableDBArrayCubit<proto.ReconciledMessage> output,
{required TableDBArrayProtobufCubit<proto.ReconciledMessage> output,
required void Function(Object, StackTrace?) onError})
: _outputCubit = output,
_onError = onError;
@ -23,7 +23,7 @@ class MessageReconciliation {
TypedKey author,
DHTLogStateData<proto.Message> inputMessagesCubitState,
DHTLogCubit<proto.Message> inputMessagesCubit) {
if (inputMessagesCubitState.elements.isEmpty) {
if (inputMessagesCubitState.window.isEmpty) {
return;
}
@ -84,11 +84,11 @@ class MessageReconciliation {
_outputCubit.operate((arr) async {
var pos = arr.length - 1;
while (pos >= 0) {
final message = await arr.getProtobuf(proto.Message.fromBuffer, pos);
final message = await arr.get(pos);
if (message == null) {
throw StateError('should have gotten last message');
}
if (message.author.toVeilid() == author) {
if (message.content.author.toVeilid() == author) {
return OutputPosition(message, pos);
}
pos--;
@ -99,11 +99,11 @@ class MessageReconciliation {
// Process a list of author input queues and insert their messages
// into the output array, performing validation steps along the way
Future<void> _reconcileInputQueues({
required TableDBArray reconciledArray,
required TableDBArrayProtobuf<proto.ReconciledMessage> reconciledArray,
required List<AuthorInputQueue> inputQueues,
}) async {
// Ensure queues all have something to do
inputQueues.removeWhere((q) => q.isEmpty);
inputQueues.removeWhere((q) => q.isDone);
if (inputQueues.isEmpty) {
return;
}
@ -124,8 +124,7 @@ class MessageReconciliation {
// Get the timestamp for this output position
var currentOutputMessage = firstOutputPos == null
? null
: await reconciledArray.getProtobuf(
proto.Message.fromBuffer, firstOutputPos);
: await reconciledArray.get(firstOutputPos);
var currentOutputPos = firstOutputPos ?? 0;
@ -143,7 +142,7 @@ class MessageReconciliation {
for (final inputQueue in inputQueues) {
final inputCurrent = inputQueue.current!;
if (currentOutputMessage == null ||
inputCurrent.timestamp <= currentOutputMessage.timestamp) {
inputCurrent.timestamp < currentOutputMessage.content.timestamp) {
toInsert.add(inputCurrent);
added = true;
@ -156,7 +155,7 @@ class MessageReconciliation {
}
// Remove empty queues now that we're done iterating
if (someQueueEmpty) {
inputQueues.removeWhere((q) => q.isEmpty);
inputQueues.removeWhere((q) => q.isDone);
}
if (toInsert.length >= _maxReconcileChunk) {
@ -166,13 +165,24 @@ class MessageReconciliation {
// Perform insertions in bulk
if (toInsert.isNotEmpty) {
await reconciledArray.insertAllProtobuf(currentOutputPos, toInsert);
final reconciledTime = Veilid.instance.now().toInt64();
// Add reconciled timestamps
final reconciledInserts = toInsert
.map((message) => proto.ReconciledMessage()
..reconciledTime = reconciledTime
..content = message)
.toList();
await reconciledArray.insertAll(currentOutputPos, reconciledInserts);
toInsert.clear();
} else {
// If there's nothing to insert at this position move to the next one
currentOutputPos++;
currentOutputMessage = await reconciledArray.getProtobuf(
proto.Message.fromBuffer, currentOutputPos);
currentOutputMessage = (currentOutputPos == reconciledArray.length)
? null
: await reconciledArray.get(currentOutputPos);
}
}
}
@ -180,7 +190,7 @@ class MessageReconciliation {
////////////////////////////////////////////////////////////////////////////
Map<TypedKey, AuthorInputSource> _inputSources = {};
final TableDBArrayCubit<proto.ReconciledMessage> _outputCubit;
final TableDBArrayProtobufCubit<proto.ReconciledMessage> _outputCubit;
final void Function(Object, StackTrace?) _onError;
static const int _maxReconcileChunk = 65536;

2
lib/chat/cubits/reconciliation/output_position.dart
View File

@ -6,7 +6,7 @@ import '../../../proto/proto.dart' as proto;
@immutable
class OutputPosition extends Equatable {
const OutputPosition(this.message, this.pos);
final proto.Message message;
final proto.ReconciledMessage message;
final int pos;
@override
List<Object?> get props => [message, pos];

130
lib/chat/cubits/single_contact_messages_cubit.dart
View File

@ -22,15 +22,18 @@ class RenderStateElement {
if (!isLocal) {
return null;
}
if (sent && !sentOffline) {
if (reconciledTimestamp != null) {
return MessageSendState.delivered;
}
if (reconciledTimestamp != null) {
if (sent) {
if (!sentOffline) {
return MessageSendState.sent;
}
} else {
return MessageSendState.sending;
}
}
return null;
}
proto.Message message;
bool isLocal;
@ -66,7 +69,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
Future<void> close() async {
await _initWait();
await _sendingMessagesQueue.close();
await _unsentMessagesQueue.close();
await _sentSubscription?.cancel();
await _rcvdSubscription?.cancel();
await _reconciledSubscription?.cancel();
@ -78,11 +81,11 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
// Initialize everything
Future<void> _init() async {
_sendingMessagesQueue = PersistentQueue<proto.Message>(
table: 'SingleContactSendingMessages',
_unsentMessagesQueue = PersistentQueue<proto.Message>(
table: 'SingleContactUnsentMessages',
key: _remoteConversationRecordKey.toString(),
fromBuffer: proto.Message.fromBuffer,
closure: _processSendingMessages,
closure: _processUnsentMessages,
);
// Make crypto
@ -144,13 +147,16 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
// Open reconciled chat record key
Future<void> _initReconciledMessagesCubit() async {
final tableName =
_localConversationRecordKey.toString().replaceAll(':', '_');
_reconciledMessagesTableDBName(_localConversationRecordKey);
final crypto = await _makeLocalMessagesCrypto();
_reconciledMessagesCubit = TableDBArrayCubit(
open: () async => TableDBArray.make(table: tableName, crypto: crypto),
decodeElement: proto.ReconciledMessage.fromBuffer);
_reconciledMessagesCubit = TableDBArrayProtobufCubit(
open: () async => TableDBArrayProtobuf.make(
table: tableName,
crypto: crypto,
fromBuffer: proto.ReconciledMessage.fromBuffer),
);
_reconciliation = MessageReconciliation(
output: _reconciledMessagesCubit!,
@ -200,6 +206,9 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
_activeAccountInfo.localAccount.identityMaster.identityPublicTypedKey(),
sentMessages,
_sentMessagesCubit!);
// Update the view
_renderState();
}
// Called when the received messages cubit gets a change
@ -211,11 +220,14 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
_reconciliation.reconcileMessages(
_remoteIdentityPublicKey, rcvdMessages, _rcvdMessagesCubit!);
// Update the view
_renderState();
}
// Called when the reconciled messages window gets a change
void _updateReconciledMessagesState(
TableDBArrayBusyState<proto.ReconciledMessage> avmessages) {
TableDBArrayProtobufBusyState<proto.ReconciledMessage> avmessages) {
// Update the view
_renderState();
}
@ -237,7 +249,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
}
// Async process to send messages in the background
Future<void> _processSendingMessages(IList<proto.Message> messages) async {
Future<void> _processUnsentMessages(IList<proto.Message> messages) async {
// Go through and assign ids to all the messages in order
proto.Message? previousMessage;
final processedMessages = messages.toList();
@ -258,7 +270,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
// Get all sent messages
final sentMessages = _sentMessagesCubit?.state.state.asData?.value;
// Get all items in the unsent queue
final sendingMessages = _sendingMessagesQueue.queue;
// final unsentMessages = _unsentMessagesQueue.queue;
// If we aren't ready to render a state, say we're loading
if (reconciledMessages == null || sentMessages == null) {
@ -267,63 +279,49 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
}
// Generate state for each message
// final reconciledMessagesMap =
// IMap<String, proto.ReconciledMessage>.fromValues(
// keyMapper: (x) => x.content.authorUniqueIdString,
// values: reconciledMessages.elements,
// );
final sentMessagesMap =
IMap<String, OnlineElementState<proto.Message>>.fromValues(
keyMapper: (x) => x.value.authorUniqueIdString,
values: sentMessages.elements,
);
final reconciledMessagesMap =
IMap<String, proto.ReconciledMessage>.fromValues(
keyMapper: (x) => x.content.authorUniqueIdString,
values: reconciledMessages.elements,
);
final sendingMessagesMap = IMap<String, proto.Message>.fromValues(
keyMapper: (x) => x.authorUniqueIdString,
values: sendingMessages,
values: sentMessages.window,
);
// final unsentMessagesMap = IMap<String, proto.Message>.fromValues(
// keyMapper: (x) => x.authorUniqueIdString,
// values: unsentMessages,
// );
final renderedElements = <String, RenderStateElement>{};
final renderedElements = <RenderStateElement>[];
for (final m in reconciledMessagesMap.entries) {
renderedElements[m.key] = RenderStateElement(
message: m.value.content,
isLocal: m.value.content.author.toVeilid() ==
for (final m in reconciledMessages.elements) {
final isLocal = m.content.author.toVeilid() ==
_activeAccountInfo.localAccount.identityMaster
.identityPublicTypedKey(),
reconciledTimestamp: Timestamp.fromInt64(m.value.reconciledTime),
);
}
for (final m in sentMessagesMap.entries) {
renderedElements.putIfAbsent(
m.key,
() => RenderStateElement(
message: m.value.value,
isLocal: true,
))
..sent = true
..sentOffline = m.value.isOffline;
}
for (final m in sendingMessagesMap.entries) {
renderedElements
.putIfAbsent(
m.key,
() => RenderStateElement(
message: m.value,
isLocal: true,
))
.sent = false;
.identityPublicTypedKey();
final reconciledTimestamp = Timestamp.fromInt64(m.reconciledTime);
final sm =
isLocal ? sentMessagesMap[m.content.authorUniqueIdString] : null;
final sent = isLocal && sm != null;
final sentOffline = isLocal && sm != null && sm.isOffline;
renderedElements.add(RenderStateElement(
message: m.content,
isLocal: isLocal,
reconciledTimestamp: reconciledTimestamp,
sent: sent,
sentOffline: sentOffline,
));
}
// Render the state
final messageKeys = renderedElements.entries
.toIList()
.sort((x, y) => x.key.compareTo(y.key));
final renderedState = messageKeys
final renderedState = renderedElements
.map((x) => MessageState(
content: x.value.message,
sentTimestamp: Timestamp.fromInt64(x.value.message.timestamp),
reconciledTimestamp: x.value.reconciledTimestamp,
sendState: x.value.sendState))
content: x.message,
sentTimestamp: Timestamp.fromInt64(x.message.timestamp),
reconciledTimestamp: x.reconciledTimestamp,
sendState: x.sendState))
.toIList();
// Emit the rendered state
@ -340,7 +338,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
..timestamp = Veilid.instance.now().toInt64();
// Put in the queue
_sendingMessagesQueue.addSync(message);
_unsentMessagesQueue.addSync(message);
// Update the view
_renderState();
@ -358,7 +356,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
static String _reconciledMessagesTableDBName(
TypedKey localConversationRecordKey) =>
'msg_$localConversationRecordKey';
'msg_${localConversationRecordKey.toString().replaceAll(':', '_')}';
/////////////////////////////////////////////////////////////////////////
@ -375,14 +373,14 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
DHTLogCubit<proto.Message>? _sentMessagesCubit;
DHTLogCubit<proto.Message>? _rcvdMessagesCubit;
TableDBArrayCubit<proto.ReconciledMessage>? _reconciledMessagesCubit;
TableDBArrayProtobufCubit<proto.ReconciledMessage>? _reconciledMessagesCubit;
late final MessageReconciliation _reconciliation;
late final PersistentQueue<proto.Message> _sendingMessagesQueue;
late final PersistentQueue<proto.Message> _unsentMessagesQueue;
StreamSubscription<DHTLogBusyState<proto.Message>>? _sentSubscription;
StreamSubscription<DHTLogBusyState<proto.Message>>? _rcvdSubscription;
StreamSubscription<TableDBArrayBusyState<proto.ReconciledMessage>>?
StreamSubscription<TableDBArrayProtobufBusyState<proto.ReconciledMessage>>?
_reconciledSubscription;
}

14
lib/chat/models/message_state.freezed.dart
View File

@ -21,7 +21,7 @@ MessageState _$MessageStateFromJson(Map<String, dynamic> json) {
/// @nodoc
mixin _$MessageState {
// Content of the message
@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
proto.Message get content =>
throw _privateConstructorUsedError; // Sent timestamp
Timestamp get sentTimestamp =>
@ -43,7 +43,7 @@ abstract class $MessageStateCopyWith<$Res> {
_$MessageStateCopyWithImpl<$Res, MessageState>;
@useResult
$Res call(
{@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
{@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
proto.Message content,
Timestamp sentTimestamp,
Timestamp? reconciledTimestamp,
@ -98,7 +98,7 @@ abstract class _$$MessageStateImplCopyWith<$Res>
@override
@useResult
$Res call(
{@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
{@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
proto.Message content,
Timestamp sentTimestamp,
Timestamp? reconciledTimestamp,
@ -146,7 +146,7 @@ class __$$MessageStateImplCopyWithImpl<$Res>
@JsonSerializable()
class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
const _$MessageStateImpl(
{@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
{@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
required this.content,
required this.sentTimestamp,
required this.reconciledTimestamp,
@ -157,7 +157,7 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
// Content of the message
@override
@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
final proto.Message content;
// Sent timestamp
@override
@ -220,7 +220,7 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
abstract class _MessageState implements MessageState {
const factory _MessageState(
{@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
{@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
required final proto.Message content,
required final Timestamp sentTimestamp,
required final Timestamp? reconciledTimestamp,
@ -230,7 +230,7 @@ abstract class _MessageState implements MessageState {
_$MessageStateImpl.fromJson;
@override // Content of the message
@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
proto.Message get content;
@override // Sent timestamp
Timestamp get sentTimestamp;

2
lib/chat/views/chat_component.dart
View File

@ -282,7 +282,7 @@ class ChatComponent extends StatelessWidget {
//showUserAvatars: false,
//showUserNames: true,
user: _localUser,
),
emptyState: const EmptyChatWidget()),
),
),
],

21
lib/chat/views/no_conversation_widget.dart
View File

@ -1,4 +1,7 @@
import 'package:flutter/material.dart';
import 'package:flutter_translate/flutter_translate.dart';
import '../../theme/models/scale_scheme.dart';
class NoConversationWidget extends StatelessWidget {
const NoConversationWidget({super.key});
@ -7,28 +10,32 @@ class NoConversationWidget extends StatelessWidget {
// ignore: prefer_expression_function_bodies
Widget build(
BuildContext context,
) =>
Container(
) {
final theme = Theme.of(context);
final scale = theme.extension<ScaleScheme>()!;
return Container(
width: double.infinity,
height: double.infinity,
decoration: BoxDecoration(
color: Theme.of(context).primaryColor,
color: Theme.of(context).scaffoldBackgroundColor,
),
child: Column(
mainAxisAlignment: MainAxisAlignment.center,
children: [
Icon(
Icons.emoji_people_outlined,
color: Theme.of(context).disabledColor,
Icons.diversity_3,
color: scale.primaryScale.subtleBorder,
size: 48,
),
Text(
'Choose A Conversation To Chat',
translate('chat.start_a_conversation'),
style: Theme.of(context).textTheme.bodyMedium?.copyWith(
color: Theme.of(context).disabledColor,
color: scale.primaryScale.subtleBorder,
),
),
],
),
);
}
}

2
lib/layout/home/home_account_ready/home_account_ready_chat.dart
View File

@ -31,7 +31,7 @@ class HomeAccountReadyChatState extends State<HomeAccountReadyChat> {
final activeChatLocalConversationKey =
context.watch<ActiveChatCubit>().state;
if (activeChatLocalConversationKey == null) {
return const EmptyChatWidget();
return const NoConversationWidget();
}
return ChatComponent.builder(
localConversationRecordKey: activeChatLocalConversationKey);

2
lib/layout/home/home_account_ready/home_account_ready_main.dart
View File

@ -69,7 +69,7 @@ class _HomeAccountReadyMainState extends State<HomeAccountReadyMain> {
final activeChatLocalConversationKey =
context.watch<ActiveChatCubit>().state;
if (activeChatLocalConversationKey == null) {
return const EmptyChatWidget();
return const NoConversationWidget();
}
return ChatComponent.builder(
localConversationRecordKey: activeChatLocalConversationKey);

4
lib/main.dart
View File

@ -6,6 +6,7 @@ import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_translate/flutter_translate.dart';
import 'package:intl/date_symbol_data_local.dart';
import 'package:stack_trace/stack_trace.dart';
import 'app.dart';
import 'settings/preferences_repository.dart';
@ -52,7 +53,8 @@ void main() async {
if (kDebugMode) {
// In debug mode, run the app without catching exceptions for debugging
await mainFunc();
// but do a much deeper async stack trace capture
await Chain.capture(mainFunc);
} else {
// Catch errors in production without killing the app
await runZonedGuarded(mainFunc, (error, stackTrace) {

74
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_cubit.dart
View File

@ -12,22 +12,25 @@ import '../../../veilid_support.dart';
@immutable
class DHTLogStateData<T> extends Equatable {
const DHTLogStateData(
{required this.elements,
required this.tail,
required this.count,
{required this.length,
required this.window,
required this.windowTail,
required this.windowSize,
required this.follow});
// The view of the elements in the dhtlog
// Span is from [tail-length, tail)
final IList<OnlineElementState<T>> elements;
// One past the end of the last element
final int tail;
// The total number of elements to try to keep in 'elements'
final int count;
// If we should have the tail following the log
// The total number of elements in the whole log
final int length;
// The view window of the elements in the dhtlog
// Span is from [tail - window.length, tail)
final IList<OnlineElementState<T>> window;
// The position of the view window, one past the last element
final int windowTail;
// The total number of elements to try to keep in the window
final int windowSize;
// If we have the window following the log
final bool follow;
@override
List<Object?> get props => [elements, tail, count, follow];
List<Object?> get props => [length, window, windowTail, windowSize, follow];
}
typedef DHTLogState<T> = AsyncValue<DHTLogStateData<T>>;
@ -58,13 +61,16 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
// If tail is positive, the position is absolute from the head of the log
// If follow is enabled, the tail offset will update when the log changes
Future<void> setWindow(
{int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
{int? windowTail,
int? windowSize,
bool? follow,
bool forceRefresh = false}) async {
await _initWait();
if (tail != null) {
_tail = tail;
if (windowTail != null) {
_windowTail = windowTail;
}
if (count != null) {
_count = count;
if (windowSize != null) {
_windowSize = windowSize;
}
if (follow != null) {
_follow = follow;
@ -82,8 +88,13 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
Future<void> _refreshInner(void Function(AsyncValue<DHTLogStateData<T>>) emit,
{bool forceRefresh = false}) async {
final avElements = await operate(
(reader) => loadElementsFromReader(reader, _tail, _count));
late final AsyncValue<IList<OnlineElementState<T>>> avElements;
late final int length;
await _log.operate((reader) async {
length = reader.length;
avElements =
await loadElementsFromReader(reader, _windowTail, _windowSize);
});
final err = avElements.asError;
if (err != null) {
emit(AsyncValue.error(err.error, err.stackTrace));
@ -94,9 +105,13 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
emit(const AsyncValue.loading());
return;
}
final elements = avElements.asData!.value;
final window = avElements.asData!.value;
emit(AsyncValue.data(DHTLogStateData(
elements: elements, tail: _tail, count: _count, follow: _follow)));
length: length,
window: window,
windowTail: _windowTail,
windowSize: _windowSize,
follow: _follow)));
}
// Tail is one past the last element to load
@ -105,6 +120,9 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
{bool forceRefresh = false}) async {
try {
final length = reader.length;
if (length == 0) {
return const AsyncValue.data(IList.empty());
}
final end = ((tail - 1) % length) + 1;
final start = (count < end) ? end - count : 0;
@ -138,18 +156,18 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
_sspUpdate.busyUpdate<T, DHTLogState<T>>(busy, (emit) async {
// apply follow
if (_follow) {
if (_tail <= 0) {
if (_windowTail <= 0) {
// Negative tail is already following tail changes
} else {
// Positive tail is measured from the head, so apply deltas
_tail = (_tail + _tailDelta - _headDelta) % upd.length;
_windowTail = (_windowTail + _tailDelta - _headDelta) % upd.length;
}
} else {
if (_tail <= 0) {
if (_windowTail <= 0) {
// Negative tail is following tail changes so apply deltas
var posTail = _tail + upd.length;
var posTail = _windowTail + upd.length;
posTail = (posTail + _tailDelta - _headDelta) % upd.length;
_tail = posTail - upd.length;
_windowTail = posTail - upd.length;
} else {
// Positive tail is measured from head so not following tail
}
@ -202,7 +220,7 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
var _tailDelta = 0;
// Cubit window into the DHTLog
var _tail = 0;
var _count = DHTShortArray.maxElements;
var _windowTail = 0;
var _windowSize = DHTShortArray.maxElements;
var _follow = true;
}

65
packages/veilid_support/lib/dht_support/src/dht_log/dht_log_spine.dart
View File

@ -451,22 +451,9 @@ class _DHTLogSpine {
///////////////////////////////////////////
// API for public interfaces
Future<_DHTLogPosition?> lookupPosition(int pos) async {
assert(_spineMutex.isLocked, 'should be locked');
return _spineCacheMutex.protect(() async {
// Check if our position is in bounds
final endPos = length;
if (pos < 0 || pos >= endPos) {
throw IndexError.withLength(pos, endPos);
}
// Calculate absolute position, ring-buffer style
final absolutePosition = (_head + pos) % _positionLimit;
// Determine the segment number and position within the segment
final segmentNumber = absolutePosition ~/ DHTShortArray.maxElements;
final segmentPos = absolutePosition % DHTShortArray.maxElements;
Future<_DHTLogPosition?> lookupPositionBySegmentNumber(
int segmentNumber, int segmentPos) async =>
_spineCacheMutex.protect(() async {
// Get the segment shortArray
final openedSegment = _openedSegments[segmentNumber];
late final DHTShortArray shortArray;
@ -493,6 +480,24 @@ class _DHTLogSpine {
pos: segmentPos,
segmentNumber: segmentNumber);
});
Future<_DHTLogPosition?> lookupPosition(int pos) async {
assert(_spineMutex.isLocked, 'should be locked');
// Check if our position is in bounds
final endPos = length;
if (pos < 0 || pos >= endPos) {
throw IndexError.withLength(pos, endPos);
}
// Calculate absolute position, ring-buffer style
final absolutePosition = (_head + pos) % _positionLimit;
// Determine the segment number and position within the segment
final segmentNumber = absolutePosition ~/ DHTShortArray.maxElements;
final segmentPos = absolutePosition % DHTShortArray.maxElements;
return lookupPositionBySegmentNumber(segmentNumber, segmentPos);
}
Future<void> _segmentClosed(int segmentNumber) async {
@ -660,6 +665,34 @@ class _DHTLogSpine {
final oldHead = _head;
final oldTail = _tail;
await _updateHead(headData);
// Lookup tail position segments that have changed
// and force their short arrays to refresh their heads
final segmentsToRefresh = <_DHTLogPosition>[];
int? lastSegmentNumber;
for (var curTail = oldTail;
curTail != _tail;
curTail = (curTail + 1) % _positionLimit) {
final segmentNumber = curTail ~/ DHTShortArray.maxElements;
final segmentPos = curTail % DHTShortArray.maxElements;
if (segmentNumber == lastSegmentNumber) {
continue;
}
lastSegmentNumber = segmentNumber;
final dhtLogPosition =
await lookupPositionBySegmentNumber(segmentNumber, segmentPos);
if (dhtLogPosition == null) {
throw Exception('missing segment in dht log');
}
segmentsToRefresh.add(dhtLogPosition);
}
// Refresh the segments that have probably changed
await segmentsToRefresh.map((p) async {
await p.shortArray.refresh();
await p.close();
}).wait;
sendUpdate(oldHead, oldTail);
});
}

11
packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array.dart
View File

@ -185,6 +185,17 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
/// Get the record pointer foir this shortarray
OwnedDHTRecordPointer get recordPointer => _head.recordPointer;
/// Refresh this DHTShortArray
/// Useful if you aren't 'watching' the array and want to poll for an update
Future<void> refresh() async {
if (!isOpen) {
throw StateError('short array is not open"');
}
await _head.operate((head) async {
await head._loadHead();
});
}
/// Runs a closure allowing read-only access to the shortarray
Future<T> operate<T>(
Future<T> Function(DHTShortArrayReadOperations) closure) async {

269
packages/veilid_support/lib/src/table_db_array.dart
View File

@ -23,8 +23,8 @@ class TableDBArrayUpdate extends Equatable {
List<Object?> get props => [headDelta, tailDelta, length];
}
class TableDBArray {
TableDBArray({
class _TableDBArrayBase {
_TableDBArrayBase({
required String table,
required VeilidCrypto crypto,
}) : _table = table,
@ -32,14 +32,14 @@ class TableDBArray {
_initWait.add(_init);
}
static Future<TableDBArray> make({
required String table,
required VeilidCrypto crypto,
}) async {
final out = TableDBArray(table: table, crypto: crypto);
await out._initWait();
return out;
}
// static Future<TableDBArray> make({
// required String table,
// required VeilidCrypto crypto,
// }) async {
// final out = TableDBArray(table: table, crypto: crypto);
// await out._initWait();
// return out;
// }
Future<void> initWait() async {
await _initWait();
@ -99,27 +99,27 @@ class TableDBArray {
bool get isOpen => _open;
Future<void> add(Uint8List value) async {
Future<void> _add(Uint8List value) async {
await _initWait();
return _writeTransaction((t) async => _addInner(t, value));
}
Future<void> addAll(List<Uint8List> values) async {
Future<void> _addAll(List<Uint8List> values) async {
await _initWait();
return _writeTransaction((t) async => _addAllInner(t, values));
}
Future<void> insert(int pos, Uint8List value) async {
Future<void> _insert(int pos, Uint8List value) async {
await _initWait();
return _writeTransaction((t) async => _insertInner(t, pos, value));
}
Future<void> insertAll(int pos, List<Uint8List> values) async {
Future<void> _insertAll(int pos, List<Uint8List> values) async {
await _initWait();
return _writeTransaction((t) async => _insertAllInner(t, pos, values));
}
Future<Uint8List> get(int pos) async {
Future<Uint8List> _get(int pos) async {
await _initWait();
return _mutex.protect(() async {
if (!_open) {
@ -129,7 +129,7 @@ class TableDBArray {
});
}
Future<List<Uint8List>> getRange(int start, [int? end]) async {
Future<List<Uint8List>> _getRange(int start, [int? end]) async {
await _initWait();
return _mutex.protect(() async {
if (!_open) {
@ -139,12 +139,12 @@ class TableDBArray {
});
}
Future<void> remove(int pos, {Output<Uint8List>? out}) async {
Future<void> _remove(int pos, {Output<Uint8List>? out}) async {
await _initWait();
return _writeTransaction((t) async => _removeInner(t, pos, out: out));
}
Future<void> removeRange(int start, int end,
Future<void> _removeRange(int start, int end,
{Output<List<Uint8List>>? out}) async {
await _initWait();
return _writeTransaction(
@ -374,7 +374,9 @@ class TableDBArray {
Future<Uint8List?> _loadEntry(int entry) async {
final encryptedValue = await _tableDB.load(0, _entryKey(entry));
return (encryptedValue == null) ? null : _crypto.decrypt(encryptedValue);
return (encryptedValue == null)
? null
: await _crypto.decrypt(encryptedValue);
}
Future<int> _getIndexEntry(int pos) async {
@ -631,77 +633,170 @@ class TableDBArray {
StreamController.broadcast();
}
extension TableDBArrayExt on TableDBArray {
/// Convenience function:
/// Like get but also parses the returned element as JSON
Future<T?> getJson<T>(
T Function(dynamic) fromJson,
//////////////////////////////////////////////////////////////////////////////
class TableDBArray extends _TableDBArrayBase {
TableDBArray({
required super.table,
required super.crypto,
});
static Future<TableDBArray> make({
required String table,
required VeilidCrypto crypto,
}) async {
final out = TableDBArray(table: table, crypto: crypto);
await out._initWait();
return out;
}
////////////////////////////////////////////////////////////
// Public interface
Future<void> add(Uint8List value) => _add(value);
Future<void> addAll(List<Uint8List> values) => _addAll(values);
Future<void> insert(int pos, Uint8List value) => _insert(pos, value);
Future<void> insertAll(int pos, List<Uint8List> values) =>
_insertAll(pos, values);
Future<Uint8List?> get(
int pos,
) =>
get(
pos,
).then((out) => jsonDecodeOptBytes(fromJson, out));
_get(pos);
/// Convenience function:
/// Like getRange but also parses the returned elements as JSON
Future<List<T>?> getRangeJson<T>(T Function(dynamic) fromJson, int start,
[int? end]) =>
getRange(start, end ?? _length).then((out) => out.map(fromJson).toList());
Future<List<Uint8List>> getRange(int start, [int? end]) =>
_getRange(start, end);
/// Convenience function:
/// Like get but also parses the returned element as a protobuf object
Future<T?> getProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer,
int pos,
) =>
get(pos).then(fromBuffer);
Future<void> remove(int pos, {Output<Uint8List>? out}) =>
_remove(pos, out: out);
/// Convenience function:
/// Like getRange but also parses the returned elements as protobuf objects
Future<List<T>?> getRangeProtobuf<T extends GeneratedMessage>(
T Function(List<int>) fromBuffer, int start, [int? end]) =>
getRange(start, end ?? _length)
.then((out) => out.map(fromBuffer).toList());
/// Convenience function:
/// Like add but for a JSON value
Future<void> addJson<T>(T value) async => add(jsonEncodeBytes(value));
/// Convenience function:
/// Like add but for a Protobuf value
Future<void> addProtobuf<T extends GeneratedMessage>(T value) =>
add(value.writeToBuffer());
/// Convenience function:
/// Like addAll but for a JSON value
Future<void> addAllJson<T>(List<T> values) async =>
addAll(values.map(jsonEncodeBytes).toList());
/// Convenience function:
/// Like addAll but for a Protobuf value
Future<void> addAllProtobuf<T extends GeneratedMessage>(
List<T> values) async =>
addAll(values.map((x) => x.writeToBuffer()).toList());
/// Convenience function:
/// Like insert but for a JSON value
Future<void> insertJson<T>(int pos, T value) async =>
insert(pos, jsonEncodeBytes(value));
/// Convenience function:
/// Like insert but for a Protobuf value
Future<void> insertProtobuf<T extends GeneratedMessage>(
int pos, T value) async =>
insert(pos, value.writeToBuffer());
/// Convenience function:
/// Like insertAll but for a JSON value
Future<void> insertAllJson<T>(int pos, List<T> values) async =>
insertAll(pos, values.map(jsonEncodeBytes).toList());
/// Convenience function:
/// Like insertAll but for a Protobuf value
Future<void> insertAllProtobuf<T extends GeneratedMessage>(
int pos, List<T> values) async =>
insertAll(pos, values.map((x) => x.writeToBuffer()).toList());
Future<void> removeRange(int start, int end,
{Output<List<Uint8List>>? out}) =>
_removeRange(start, end, out: out);
}
//////////////////////////////////////////////////////////////////////////////
class TableDBArrayJson<T> extends _TableDBArrayBase {
TableDBArrayJson(
{required super.table,
required super.crypto,
required T Function(dynamic) fromJson})
: _fromJson = fromJson;
static Future<TableDBArrayJson<T>> make<T>(
{required String table,
required VeilidCrypto crypto,
required T Function(dynamic) fromJson}) async {
final out =
TableDBArrayJson<T>(table: table, crypto: crypto, fromJson: fromJson);
await out._initWait();
return out;
}
////////////////////////////////////////////////////////////
// Public interface
Future<void> add(T value) => _add(jsonEncodeBytes(value));
Future<void> addAll(List<T> values) async =>
_addAll(values.map(jsonEncodeBytes).toList());
Future<void> insert(int pos, T value) async =>
_insert(pos, jsonEncodeBytes(value));
Future<void> insertAll(int pos, List<T> values) async =>
_insertAll(pos, values.map(jsonEncodeBytes).toList());
Future<T?> get(
int pos,
) =>
_get(pos).then((out) => jsonDecodeOptBytes(_fromJson, out));
Future<List<T>> getRange(int start, [int? end]) =>
_getRange(start, end).then((out) => out.map(_fromJson).toList());
Future<void> remove(int pos, {Output<T>? out}) async {
final outJson = (out != null) ? Output<Uint8List>() : null;
await _remove(pos, out: outJson);
if (outJson != null && outJson.value != null) {
out!.save(jsonDecodeBytes(_fromJson, outJson.value!));
}
}
Future<void> removeRange(int start, int end, {Output<List<T>>? out}) async {
final outJson = (out != null) ? Output<List<Uint8List>>() : null;
await _removeRange(start, end, out: outJson);
if (outJson != null && outJson.value != null) {
out!.save(
outJson.value!.map((x) => jsonDecodeBytes(_fromJson, x)).toList());
}
}
////////////////////////////////////////////////////////////////////////////
final T Function(dynamic) _fromJson;
}
//////////////////////////////////////////////////////////////////////////////
class TableDBArrayProtobuf<T extends GeneratedMessage>
extends _TableDBArrayBase {
TableDBArrayProtobuf(
{required super.table,
required super.crypto,
required T Function(List<int>) fromBuffer})
: _fromBuffer = fromBuffer;
static Future<TableDBArrayProtobuf<T>> make<T extends GeneratedMessage>(
{required String table,
required VeilidCrypto crypto,
required T Function(List<int>) fromBuffer}) async {
final out = TableDBArrayProtobuf<T>(
table: table, crypto: crypto, fromBuffer: fromBuffer);
await out._initWait();
return out;
}
////////////////////////////////////////////////////////////
// Public interface
Future<void> add(T value) => _add(value.writeToBuffer());
Future<void> addAll(List<T> values) async =>
_addAll(values.map((x) => x.writeToBuffer()).toList());
Future<void> insert(int pos, T value) async =>
_insert(pos, value.writeToBuffer());
Future<void> insertAll(int pos, List<T> values) async =>
_insertAll(pos, values.map((x) => x.writeToBuffer()).toList());
Future<T?> get(
int pos,
) =>
_get(pos).then(_fromBuffer);
Future<List<T>> getRange(int start, [int? end]) =>
_getRange(start, end).then((out) => out.map(_fromBuffer).toList());
Future<void> remove(int pos, {Output<T>? out}) async {
final outProto = (out != null) ? Output<Uint8List>() : null;
await _remove(pos, out: outProto);
if (outProto != null && outProto.value != null) {
out!.save(_fromBuffer(outProto.value!));
}
}
Future<void> removeRange(int start, int end, {Output<List<T>>? out}) async {
final outProto = (out != null) ? Output<List<Uint8List>>() : null;
await _removeRange(start, end, out: outProto);
if (outProto != null && outProto.value != null) {
out!.save(outProto.value!.map(_fromBuffer).toList());
}
}
////////////////////////////////////////////////////////////////////////////
final T Function(List<int>) _fromBuffer;
}

40
packages/veilid_support/lib/src/table_db_array_cubit.dart → packages/veilid_support/lib/src/table_db_array_protobuf_cubit.dart
View File

@ -6,12 +6,14 @@ import 'package:bloc_advanced_tools/bloc_advanced_tools.dart';
import 'package:equatable/equatable.dart';
import 'package:fast_immutable_collections/fast_immutable_collections.dart';
import 'package:meta/meta.dart';
import 'package:protobuf/protobuf.dart';
import '../../../veilid_support.dart';
@immutable
class TableDBArrayStateData<T> extends Equatable {
const TableDBArrayStateData(
class TableDBArrayProtobufStateData<T extends GeneratedMessage>
extends Equatable {
const TableDBArrayProtobufStateData(
{required this.elements,
required this.tail,
required this.count,
@ -30,16 +32,17 @@ class TableDBArrayStateData<T> extends Equatable {
List<Object?> get props => [elements, tail, count, follow];
}
typedef TableDBArrayState<T> = AsyncValue<TableDBArrayStateData<T>>;
typedef TableDBArrayBusyState<T> = BlocBusyState<TableDBArrayState<T>>;
typedef TableDBArrayProtobufState<T extends GeneratedMessage>
= AsyncValue<TableDBArrayProtobufStateData<T>>;
typedef TableDBArrayProtobufBusyState<T extends GeneratedMessage>
= BlocBusyState<TableDBArrayProtobufState<T>>;
class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
with BlocBusyWrapper<TableDBArrayState<T>> {
TableDBArrayCubit({
required Future<TableDBArray> Function() open,
required T Function(List<int> data) decodeElement,
}) : _decodeElement = decodeElement,
super(const BlocBusyState(AsyncValue.loading())) {
class TableDBArrayProtobufCubit<T extends GeneratedMessage>
extends Cubit<TableDBArrayProtobufBusyState<T>>
with BlocBusyWrapper<TableDBArrayProtobufState<T>> {
TableDBArrayProtobufCubit({
required Future<TableDBArrayProtobuf<T>> Function() open,
}) : super(const BlocBusyState(AsyncValue.loading())) {
_initWait.add(() async {
// Open table db array
_array = await open();
@ -81,7 +84,7 @@ class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
busy((emit) async => _refreshInner(emit, forceRefresh: forceRefresh));
Future<void> _refreshInner(
void Function(AsyncValue<TableDBArrayStateData<T>>) emit,
void Function(AsyncValue<TableDBArrayProtobufStateData<T>>) emit,
{bool forceRefresh = false}) async {
final avElements = await _loadElements(_tail, _count);
final err = avElements.asError;
@ -95,7 +98,7 @@ class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
return;
}
final elements = avElements.asData!.value;
emit(AsyncValue.data(TableDBArrayStateData(
emit(AsyncValue.data(TableDBArrayProtobufStateData(
elements: elements, tail: _tail, count: _count, follow: _follow)));
}
@ -110,8 +113,7 @@ class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
}
final end = ((tail - 1) % length) + 1;
final start = (count < end) ? end - count : 0;
final allItems =
(await _array.getRange(start, end)).map(_decodeElement).toIList();
final allItems = (await _array.getRange(start, end)).toIList();
return AsyncValue.data(allItems);
} on Exception catch (e, st) {
return AsyncValue.error(e, st);
@ -128,7 +130,7 @@ class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
_headDelta += upd.headDelta;
_tailDelta += upd.tailDelta;
_sspUpdate.busyUpdate<T, TableDBArrayState<T>>(busy, (emit) async {
_sspUpdate.busyUpdate<T, TableDBArrayProtobufState<T>>(busy, (emit) async {
// apply follow
if (_follow) {
if (_tail <= 0) {
@ -165,14 +167,14 @@ class TableDBArrayCubit<T> extends Cubit<TableDBArrayBusyState<T>>
await super.close();
}
Future<R?> operate<R>(Future<R?> Function(TableDBArray) closure) async {
Future<R?> operate<R>(
Future<R?> Function(TableDBArrayProtobuf<T>) closure) async {
await _initWait();
return closure(_array);
}
final WaitSet<void> _initWait = WaitSet();
late final TableDBArray _array;
final T Function(List<int> data) _decodeElement;
late final TableDBArrayProtobuf<T> _array;
StreamSubscription<void>? _subscription;
bool _wantsCloseArray = false;
final _sspUpdate = SingleStatelessProcessor();

2
packages/veilid_support/lib/veilid_support.dart
View File

@ -16,6 +16,6 @@ export 'src/persistent_queue.dart';
export 'src/protobuf_tools.dart';
export 'src/table_db.dart';
export 'src/table_db_array.dart';
export 'src/table_db_array_cubit.dart';
export 'src/table_db_array_protobuf_cubit.dart';
export 'src/veilid_crypto.dart';
export 'src/veilid_log.dart' hide veilidLoggy;

14
packages/veilid_support/pubspec.lock
View File

@ -36,10 +36,9 @@ packages:
async_tools:
dependency: "direct main"
description:
name: async_tools
sha256: e783ac6ed5645c86da34240389bb3a000fc5e3ae6589c6a482eb24ece7217681
url: "https://pub.dev"
source: hosted
path: "../../../dart_async_tools"
relative: true
source: path
version: "0.1.1"
bloc:
dependency: "direct main"
@ -52,10 +51,9 @@ packages:
bloc_advanced_tools:
dependency: "direct main"
description:
name: bloc_advanced_tools
sha256: "09f8a121d950575f1f2980c8b10df46b2ac6c72c8cbe48cc145871e5882ed430"
url: "https://pub.dev"
source: hosted
path: "../../../bloc_advanced_tools"
relative: true
source: path
version: "0.1.1"
boolean_selector:
dependency: transitive

6
packages/veilid_support/pubspec.yaml
View File

@ -24,6 +24,12 @@ dependencies:
# veilid: ^0.0.1
path: ../../../veilid/veilid-flutter
dependency_overrides:
async_tools:
path: ../../../dart_async_tools
bloc_advanced_tools:
path: ../../../bloc_advanced_tools
dev_dependencies:
build_runner: ^2.4.10
freezed: ^2.5.2