fix head issue and refactor

lib/account_manager/cubits/account_record_cubit.dart

@@ -6,8 +6,8 @@ import '../../proto/proto.dart' as proto;
 
 class AccountRecordCubit extends DefaultDHTRecordCubit<proto.Account> {
   AccountRecordCubit({
-    required super.record,
+    required super.open,
-  }) : super.value(decodeState: proto.Account.fromBuffer);
+  }) : super(decodeState: proto.Account.fromBuffer);
 
   @override
   Future<void> close() async {

lib/account_manager/models/active_account_info.dart

@@ -11,7 +11,7 @@ class ActiveAccountInfo {
   const ActiveAccountInfo({
     required this.localAccount,
     required this.userLogin,
-    required this.accountRecord,
+    //required this.accountRecord,
   });
   //
 
@@ -41,5 +41,5 @@ class ActiveAccountInfo {
   //
   final LocalAccount localAccount;
   final UserLogin userLogin;
-  final DHTRecord accountRecord;
+  //final DHTRecord accountRecord;
 }

lib/account_manager/repository/account_repository/account_repository.dart

@@ -49,7 +49,6 @@ class AccountRepository {
   final TableDBValue<IList<UserLogin>> _userLogins;
   final TableDBValue<TypedKey?> _activeLocalAccount;
   final StreamController<AccountRepositoryChange> _streamController;
-  final Map<TypedKey, DHTRecord> _openedAccountRecords = {};
 
   //////////////////////////////////////////////////////////////
   /// Singleton initialization
@@ -60,11 +59,10 @@ class AccountRepository {
     await _localAccounts.get();
     await _userLogins.get();
     await _activeLocalAccount.get();
-    await _openLoggedInDHTRecords();
   }
 
   Future<void> close() async {
-    await _closeLoggedInDHTRecords();
+    // ???
   }
 
   //////////////////////////////////////////////////////////////
@@ -139,25 +137,12 @@ class AccountRepository {
           activeAccountInfo: null);
     }
 
-    // Pull the account DHT key, decode it and return it
-    final accountRecord =
-        _getAccountRecord(userLogin.accountRecordInfo.accountRecord.recordKey);
-    if (accountRecord == null) {
-      // Account could not be read or decrypted from DHT
-      return AccountInfo(
-          status: AccountInfoStatus.accountInvalid,
-          active: active,
-          activeAccountInfo: null);
-    }
-
     // Got account, decrypted and decoded
     return AccountInfo(
       status: AccountInfoStatus.accountReady,
       active: active,
-      activeAccountInfo: ActiveAccountInfo(
+      activeAccountInfo:
-          localAccount: localAccount,
+          ActiveAccountInfo(localAccount: localAccount, userLogin: userLogin),
-          userLogin: userLogin,
-          accountRecord: accountRecord),
     );
   }
 
@@ -344,9 +329,6 @@ class AccountRepository {
     await _userLogins.set(newUserLogins);
     await _activeLocalAccount.set(identityMaster.masterRecordKey);
 
-    // Ensure all logins are opened
-    await _openLoggedInDHTRecords();
-
     _streamController
       ..add(AccountRepositoryChange.userLogins)
       ..add(AccountRepositoryChange.activeLocalAccount);
@@ -395,12 +377,6 @@ class AccountRepository {
       return;
     }
 
-    // Close DHT records for this account
-    final accountRecordKey =
-        logoutUserLogin.accountRecordInfo.accountRecord.recordKey;
-    final accountRecord = _openedAccountRecords.remove(accountRecordKey);
-    await accountRecord?.close();
-
     // Remove user from active logins list
     final newUserLogins = (await _userLogins.get())
         .removeWhere((ul) => ul.accountMasterRecordKey == logoutUser);
@@ -408,32 +384,7 @@ class AccountRepository {
     _streamController.add(AccountRepositoryChange.userLogins);
   }
 
-  Future<void> _openLoggedInDHTRecords() async {
+  Future<DHTRecord> openAccountRecord(UserLogin userLogin) async {
-    // For all user logins if they arent open yet
-    final userLogins = await _userLogins.get();
-    for (final userLogin in userLogins) {
-      await _openAccountRecord(userLogin);
-    }
-  }
-
-  Future<void> _closeLoggedInDHTRecords() async {
-    final userLogins = await _userLogins.get();
-    for (final userLogin in userLogins) {
-      //// Account record key /////////////////////////////
-      final accountRecordKey =
-          userLogin.accountRecordInfo.accountRecord.recordKey;
-      await _closeAccountRecord(accountRecordKey);
-    }
-  }
-
-  Future<DHTRecord> _openAccountRecord(UserLogin userLogin) async {
-    final accountRecordKey =
-        userLogin.accountRecordInfo.accountRecord.recordKey;
-
-    final existingAccountRecord = _openedAccountRecords[accountRecordKey];
-    if (existingAccountRecord != null) {
-      return existingAccountRecord;
-    }
     final localAccount = fetchLocalAccount(userLogin.accountMasterRecordKey)!;
 
     // Record not yet open, do it
@@ -442,19 +393,6 @@ class AccountRepository {
         userLogin.accountRecordInfo.accountRecord,
         parent: localAccount.identityMaster.identityRecordKey);
 
-    _openedAccountRecords[accountRecordKey] = record;
-
-    // Watch the record's only (default) key
-    await record.watch();
-
     return record;
   }
-
-  DHTRecord? _getAccountRecord(TypedKey accountRecordKey) =>
-      _openedAccountRecords[accountRecordKey];
-
-  Future<void> _closeAccountRecord(TypedKey accountRecordKey) async {
-    final accountRecord = _openedAccountRecords.remove(accountRecordKey);
-    await accountRecord?.close();
-  }
 }

lib/chat/cubits/single_contact_messages_cubit.dart

@@ -154,7 +154,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   }
 
   Future<void> _mergeMessagesInner(
-      {required DHTShortArray reconciledMessages,
+      {required DHTShortArrayWrite reconciledMessagesWriter,
       required IList<proto.Message> messages}) async {
     // Ensure remoteMessages is sorted by timestamp
     final newMessages = messages
@@ -162,8 +162,12 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         .removeDuplicates();
 
     // Existing messages will always be sorted by timestamp so merging is easy
-    final existingMessages =
+    final existingMessages = await reconciledMessagesWriter
-        _reconciledChatMessagesCubit!.state.state.data!.value.toList();
+        .getAllItemsProtobuf(proto.Message.fromBuffer);
+    if (existingMessages == null) {
+      throw Exception(
+          'Could not load existing reconciled messages at this time');
+    }
 
     var ePos = 0;
     var nPos = 0;
@@ -180,7 +184,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         // New message belongs here
 
         // Insert into dht backing array
-        await reconciledMessages.tryInsertItem(
+        await reconciledMessagesWriter.tryInsertItem(
             ePos, newMessage.writeToBuffer());
         // Insert into local copy as well for this operation
         existingMessages.insert(ePos, newMessage);
@@ -202,7 +206,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
       final newMessage = newMessages[nPos];
 
       // Append to dht backing array
-      await reconciledMessages.tryAddItem(newMessage.writeToBuffer());
+      await reconciledMessagesWriter.tryAddItem(newMessage.writeToBuffer());
       // Insert into local copy as well for this operation
       existingMessages.add(newMessage);
 
@@ -215,16 +219,17 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
     final reconciledChatMessagesCubit = _reconciledChatMessagesCubit!;
 
     // Merge remote and local messages into the reconciled chat log
-    await reconciledChatMessagesCubit.operate((reconciledMessages) async {
+    await reconciledChatMessagesCubit
+        .operateWrite((reconciledMessagesWriter) async {
       // xxx for now, keep two lists, but can probable simplify this out soon
       if (entry.localMessages != null) {
         await _mergeMessagesInner(
-            reconciledMessages: reconciledMessages,
+            reconciledMessagesWriter: reconciledMessagesWriter,
             messages: entry.localMessages!);
       }
       if (entry.remoteMessages != null) {
         await _mergeMessagesInner(
-            reconciledMessages: reconciledMessages,
+            reconciledMessagesWriter: reconciledMessagesWriter,
             messages: entry.remoteMessages!);
       }
     });
@@ -244,8 +249,8 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   }
 
   Future<void> addMessage({required proto.Message message}) async {
-    await _localMessagesCubit!.operate(
+    await _localMessagesCubit!
-        (shortArray) => shortArray.tryAddItem(message.writeToBuffer()));
+        .operateWrite((writer) => writer.tryAddItem(message.writeToBuffer()));
   }
 
   final ActiveAccountInfo _activeAccountInfo;

lib/chat_list/cubits/chat_list_cubit.dart

@@ -41,13 +41,13 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat> {
   }) async {
     // Add Chat to account's list
     // if this fails, don't keep retrying, user can try again later
-    await operate((shortArray) async {
+    await operateWrite((writer) async {
       final remoteConversationRecordKeyProto =
           remoteConversationRecordKey.toProto();
 
       // See if we have added this chat already
-      for (var i = 0; i < shortArray.length; i++) {
+      for (var i = 0; i < writer.length; i++) {
-        final cbuf = await shortArray.getItem(i);
+        final cbuf = await writer.getItem(i);
         if (cbuf == null) {
           throw Exception('Failed to get chat');
         }
@@ -72,7 +72,7 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat> {
         ..reconciledChatRecord = reconciledChatRecord.toProto();
 
       // Add chat
-      final added = await shortArray.tryAddItem(chat.writeToBuffer());
+      final added = await writer.tryAddItem(chat.writeToBuffer());
       if (!added) {
         throw Exception('Failed to add chat');
       }
@@ -86,19 +86,19 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat> {
 
     // Remove Chat from account's list
     // if this fails, don't keep retrying, user can try again later
-    final deletedItem = await operate((shortArray) async {
+    final (deletedItem, success) = await operateWrite((writer) async {
       if (activeChatCubit.state == remoteConversationRecordKey) {
         activeChatCubit.setActiveChat(null);
       }
-      for (var i = 0; i < shortArray.length; i++) {
+      for (var i = 0; i < writer.length; i++) {
-        final cbuf = await shortArray.getItem(i);
+        final cbuf = await writer.getItem(i);
         if (cbuf == null) {
           throw Exception('Failed to get chat');
         }
         final c = proto.Chat.fromBuffer(cbuf);
         if (c.remoteConversationRecordKey == remoteConversationKey) {
           // Found the right chat
-          if (await shortArray.tryRemoveItem(i) != null) {
+          if (await writer.tryRemoveItem(i) != null) {
             return c;
           }
           return null;
@@ -106,7 +106,7 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat> {
       }
       return null;
     });
-    if (deletedItem != null) {
+    if (success && deletedItem != null) {
       try {
         await DHTRecordPool.instance
             .delete(deletedItem.reconciledChatRecord.toVeilid().recordKey);

lib/contact_invitation/cubits/contact_invitation_list_cubit.dart

@@ -139,8 +139,8 @@ class ContactInvitationListCubit
 
         // Add ContactInvitationRecord to account's list
         // if this fails, don't keep retrying, user can try again later
-        await operate((shortArray) async {
+        await operateWrite((writer) async {
-          if (await shortArray.tryAddItem(cinvrec.writeToBuffer()) == false) {
+          if (await writer.tryAddItem(cinvrec.writeToBuffer()) == false) {
             throw Exception('Failed to add contact invitation record');
           }
         });
@@ -158,16 +158,16 @@ class ContactInvitationListCubit
         _activeAccountInfo.userLogin.accountRecordInfo.accountRecord.recordKey;
 
     // Remove ContactInvitationRecord from account's list
-    final deletedItem = await operate((shortArray) async {
+    final (deletedItem, success) = await operateWrite((writer) async {
-      for (var i = 0; i < shortArray.length; i++) {
+      for (var i = 0; i < writer.length; i++) {
-        final item = await shortArray.getItemProtobuf(
+        final item = await writer.getItemProtobuf(
             proto.ContactInvitationRecord.fromBuffer, i);
         if (item == null) {
           throw Exception('Failed to get contact invitation record');
         }
         if (item.contactRequestInbox.recordKey.toVeilid() ==
             contactRequestInboxRecordKey) {
-          if (await shortArray.tryRemoveItem(i) != null) {
+          if (await writer.tryRemoveItem(i) != null) {
             return item;
           }
           return null;
@@ -176,7 +176,7 @@ class ContactInvitationListCubit
       return null;
     });
 
-    if (deletedItem != null) {
+    if (success && deletedItem != null) {
       // Delete the contact request inbox
       final contactRequestInbox = deletedItem.contactRequestInbox.toVeilid();
       await (await pool.openOwned(contactRequestInbox,

lib/contact_invitation/cubits/contact_request_inbox_cubit.dart

@@ -14,11 +14,11 @@ class ContactRequestInboxCubit
                 contactInvitationRecord: contactInvitationRecord),
             decodeState: proto.SignedContactResponse.fromBuffer);
 
-  ContactRequestInboxCubit.value(
+  // ContactRequestInboxCubit.value(
-      {required super.record,
+  //     {required super.record,
-      required this.activeAccountInfo,
+  //     required this.activeAccountInfo,
-      required this.contactInvitationRecord})
+  //     required this.contactInvitationRecord})
-      : super.value(decodeState: proto.SignedContactResponse.fromBuffer);
+  //     : super.value(decodeState: proto.SignedContactResponse.fromBuffer);
 
   static Future<DHTRecord> _open(
       {required ActiveAccountInfo activeAccountInfo,

lib/contact_invitation/cubits/waiting_invitations_bloc_map_cubit.dart

@@ -23,7 +23,7 @@ class WaitingInvitationsBlocMapCubit extends BlocMapCubit<TypedKey,
   WaitingInvitationsBlocMapCubit(
       {required this.activeAccountInfo, required this.account});
 
-  Future<void> addWaitingInvitation(
+  Future<void> _addWaitingInvitation(
           {required proto.ContactInvitationRecord
               contactInvitationRecord}) async =>
       add(() => MapEntry(
@@ -54,7 +54,7 @@ class WaitingInvitationsBlocMapCubit extends BlocMapCubit<TypedKey,
 
   @override
   Future<void> updateState(TypedKey key, proto.ContactInvitationRecord value) =>
-      addWaitingInvitation(contactInvitationRecord: value);
+      _addWaitingInvitation(contactInvitationRecord: value);
 
   ////
   final ActiveAccountInfo activeAccountInfo;

lib/contacts/cubits/contact_list_cubit.dart

@@ -52,8 +52,8 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
 
     // Add Contact to account's list
     // if this fails, don't keep retrying, user can try again later
-    await operate((shortArray) async {
+    await operateWrite((writer) async {
-      if (await shortArray.tryAddItem(contact.writeToBuffer()) == false) {
+      if (!await writer.tryAddItem(contact.writeToBuffer())) {
         throw Exception('Failed to add contact record');
       }
     });
@@ -66,16 +66,15 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
         contact.remoteConversationRecordKey.toVeilid();
 
     // Remove Contact from account's list
-    final deletedItem = await operate((shortArray) async {
+    final (deletedItem, success) = await operateWrite((writer) async {
-      for (var i = 0; i < shortArray.length; i++) {
+      for (var i = 0; i < writer.length; i++) {
-        final item =
+        final item = await writer.getItemProtobuf(proto.Contact.fromBuffer, i);
-            await shortArray.getItemProtobuf(proto.Contact.fromBuffer, i);
         if (item == null) {
           throw Exception('Failed to get contact');
         }
         if (item.remoteConversationRecordKey ==
             contact.remoteConversationRecordKey) {
-          if (await shortArray.tryRemoveItem(i) != null) {
+          if (await writer.tryRemoveItem(i) != null) {
             return item;
           }
           return null;
@@ -84,7 +83,7 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
       return null;
     });
 
-    if (deletedItem != null) {
+    if (success && deletedItem != null) {
       try {
         await pool.delete(localConversationKey);
       } on Exception catch (e) {

lib/contacts/cubits/conversation_cubit.dart

@@ -41,6 +41,7 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
         super(const AsyncValue.loading()) {
     if (_localConversationRecordKey != null) {
       Future.delayed(Duration.zero, () async {
+        await _setLocalConversation(() async {
           final accountRecordKey = _activeAccountInfo
               .userLogin.accountRecordInfo.accountRecord.recordKey;
 
@@ -51,12 +52,14 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
           final record = await pool.openWrite(
               _localConversationRecordKey!, writer,
               parent: accountRecordKey, crypto: crypto);
-        await _setLocalConversation(record);
+          return record;
+        });
       });
     }
 
     if (_remoteConversationRecordKey != null) {
       Future.delayed(Duration.zero, () async {
+        await _setRemoteConversation(() async {
           final accountRecordKey = _activeAccountInfo
               .userLogin.accountRecordInfo.accountRecord.recordKey;
 
@@ -65,7 +68,8 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
           final crypto = await _cachedConversationCrypto();
           final record = await pool.openRead(_remoteConversationRecordKey,
               parent: accountRecordKey, crypto: crypto);
-        await _setRemoteConversation(record);
+          return record;
+        });
       });
     }
   }
@@ -74,6 +78,9 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
   Future<void> close() async {
     await _localSubscription?.cancel();
     await _remoteSubscription?.cancel();
+    await _localConversationCubit?.close();
+    await _remoteConversationCubit?.close();
+
     await super.close();
   }
 
@@ -122,24 +129,21 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
   }
 
   // Open local converation key
-  Future<void> _setLocalConversation(DHTRecord localConversationRecord) async {
+  Future<void> _setLocalConversation(Future<DHTRecord> Function() open) async {
     assert(_localConversationCubit == null,
         'shoud not set local conversation twice');
-    _localConversationCubit = DefaultDHTRecordCubit.value(
+    _localConversationCubit = DefaultDHTRecordCubit(
-        record: localConversationRecord,
+        open: open, decodeState: proto.Conversation.fromBuffer);
-        decodeState: proto.Conversation.fromBuffer);
     _localSubscription =
         _localConversationCubit!.stream.listen(updateLocalConversationState);
   }
 
   // Open remote converation key
-  Future<void> _setRemoteConversation(
+  Future<void> _setRemoteConversation(Future<DHTRecord> Function() open) async {
-      DHTRecord remoteConversationRecord) async {
     assert(_remoteConversationCubit == null,
         'shoud not set remote conversation twice');
-    _remoteConversationCubit = DefaultDHTRecordCubit.value(
+    _remoteConversationCubit = DefaultDHTRecordCubit(
-        record: remoteConversationRecord,
+        open: open, decodeState: proto.Conversation.fromBuffer);
-        decodeState: proto.Conversation.fromBuffer);
     _remoteSubscription =
         _remoteConversationCubit!.stream.listen(updateRemoteConversationState);
   }
@@ -215,7 +219,7 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
 
     // If success, save the new local conversation record key in this object
     _localConversationRecordKey = localConversationRecord.key;
-    await _setLocalConversation(localConversationRecord);
+    await _setLocalConversation(() async => localConversationRecord);
 
     return out;
   }

lib/layout/home/home_account_ready/home_account_ready_shell.dart

@@ -19,14 +19,11 @@ class HomeAccountReadyShell extends StatefulWidget {
     // These must exist in order for the account to
     // be considered 'ready' for this widget subtree
     final activeLocalAccount = context.read<ActiveLocalAccountCubit>().state!;
-    final accountInfo =
+    final activeAccountInfo = context.read<ActiveAccountInfo>();
-        AccountRepository.instance.getAccountInfo(activeLocalAccount);
-    final activeAccountInfo = accountInfo.activeAccountInfo!;
     final routerCubit = context.read<RouterCubit>();
 
     return HomeAccountReadyShell._(
         activeLocalAccount: activeLocalAccount,
-        accountInfo: accountInfo,
         activeAccountInfo: activeAccountInfo,
         routerCubit: routerCubit,
         key: key,
@@ -34,7 +31,6 @@ class HomeAccountReadyShell extends StatefulWidget {
   }
   const HomeAccountReadyShell._(
       {required this.activeLocalAccount,
-      required this.accountInfo,
       required this.activeAccountInfo,
       required this.routerCubit,
       required this.child,
@@ -45,7 +41,6 @@ class HomeAccountReadyShell extends StatefulWidget {
 
   final Widget child;
   final TypedKey activeLocalAccount;
-  final AccountInfo accountInfo;
   final ActiveAccountInfo activeAccountInfo;
   final RouterCubit routerCubit;
 
@@ -55,7 +50,6 @@ class HomeAccountReadyShell extends StatefulWidget {
     properties
       ..add(DiagnosticsProperty<TypedKey>(
           'activeLocalAccount', activeLocalAccount))
-      ..add(DiagnosticsProperty<AccountInfo>('accountInfo', accountInfo))
       ..add(DiagnosticsProperty<ActiveAccountInfo>(
           'activeAccountInfo', activeAccountInfo))
       ..add(DiagnosticsProperty<RouterCubit>('routerCubit', routerCubit));
@@ -114,14 +108,8 @@ class HomeAccountReadyShellState extends State<HomeAccountReadyShell> {
   }
 
   @override
-  Widget build(BuildContext context) => Provider<ActiveAccountInfo>.value(
+  Widget build(BuildContext context) {
-      value: widget.activeAccountInfo,
+    final account = context.watch<AccountRecordCubit>().state.data?.value;
-      child: BlocProvider(
-          create: (context) => AccountRecordCubit(
-              record: widget.activeAccountInfo.accountRecord),
-          child: Builder(builder: (context) {
-            final account =
-                context.watch<AccountRecordCubit>().state.data?.value;
     if (account == null) {
       return waitingPage();
     }
@@ -155,14 +143,11 @@ class HomeAccountReadyShellState extends State<HomeAccountReadyShell> {
                   activeAccountInfo: widget.activeAccountInfo,
                   contactListCubit: context.read<ContactListCubit>(),
                   chatListCubit: context.read<ChatListCubit>())
-                        ..followBloc(
+                ..followBloc(context.read<ActiveConversationsBlocMapCubit>())),
-                            context.read<ActiveConversationsBlocMapCubit>())),
           BlocProvider(
               create: (context) => WaitingInvitationsBlocMapCubit(
-                          activeAccountInfo: widget.activeAccountInfo,
+                  activeAccountInfo: widget.activeAccountInfo, account: account)
-                          account: account)
+                ..followBloc(context.read<ContactInvitationListCubit>()))
-                        ..followBloc(
-                            context.read<ContactInvitationListCubit>()))
         ],
         child: MultiBlocListener(listeners: [
           BlocListener<WaitingInvitationsBlocMapCubit,
@@ -170,5 +155,5 @@ class HomeAccountReadyShellState extends State<HomeAccountReadyShell> {
             listener: _invitationStatusListener,
           )
         ], child: widget.child));
-          })));
+  }
 }

lib/layout/home/home_shell.dart

@@ -55,7 +55,9 @@ class HomeShellState extends State<HomeShell> {
             value: accountInfo.activeAccountInfo!,
             child: BlocProvider(
                 create: (context) => AccountRecordCubit(
-                    record: accountInfo.activeAccountInfo!.accountRecord),
+                    open: () async => AccountRepository.instance
+                        .openAccountRecord(
+                            accountInfo.activeAccountInfo!.userLogin)),
                 child: widget.accountReadyBuilder));
     }
   }

macos/Podfile.lock

@@ -72,7 +72,7 @@ SPEC CHECKSUMS:
   pasteboard: 9b69dba6fedbb04866be632205d532fe2f6b1d99
   path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c
   screen_retriever: 59634572a57080243dd1bf715e55b6c54f241a38
-  share_plus: 76dd39142738f7a68dd57b05093b5e8193f220f7
+  share_plus: 36537c04ce0c3e3f5bd297ce4318b6d5ee5fd6cf
   shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695
   smart_auth: b38e3ab4bfe089eacb1e233aca1a2340f96c28e9
   sqflite: 673a0e54cc04b7d6dba8d24fb8095b31c3a99eec

packages/async_tools/lib/src/single_stateless_processor.dart

@@ -24,7 +24,8 @@ class SingleStatelessProcessor {
     });
   }
 
-  // Like update, but with a busy wrapper that clears once the updating is finished
+  // Like update, but with a busy wrapper that
+  // clears once the updating is finished
   void busyUpdate<T, S>(
       Future<void> Function(Future<void> Function(void Function(S))) busy,
       Future<void> Function(void Function(S)) closure) {

packages/bloc_tools/lib/src/async_transformer_cubit.dart

@@ -3,6 +3,11 @@ import 'dart:async';
 import 'package:async_tools/async_tools.dart';
 import 'package:bloc/bloc.dart';
 
+// A cubit with state T that wraps another input cubit of state S and
+// produces T fro S via an asynchronous transform closure
+// The input cubit becomes 'owned' by the AsyncTransformerCubit and will
+// be closed when the AsyncTransformerCubit closes.
+
 class AsyncTransformerCubit<T, S> extends Cubit<AsyncValue<T>> {
   AsyncTransformerCubit(this.input, {required this.transform})
       : super(const AsyncValue.loading()) {

packages/veilid_support/lib/dht_support/src/dht_record/dht_record.dart

@@ -342,7 +342,7 @@ class DHTRecord {
 
   void _addValueChange(
       {required bool local,
-      required Uint8List data,
+      required Uint8List? data,
       required List<ValueSubkeyRange> subkeys}) {
     final ws = watchState;
     if (ws != null) {
@@ -378,6 +378,6 @@ class DHTRecord {
 
   void _addRemoteValueChange(VeilidUpdateValueChange update) {
     _addValueChange(
-        local: false, data: update.value.data, subkeys: update.subkeys);
+        local: false, data: update.value?.data, subkeys: update.subkeys);
   }
 }

packages/veilid_support/lib/dht_support/src/dht_record/dht_record_cubit.dart

@@ -28,19 +28,19 @@ class DHTRecordCubit<T> extends Cubit<AsyncValue<T>> {
     });
   }
 
-  DHTRecordCubit.value({
+  // DHTRecordCubit.value({
-    required DHTRecord record,
+  //   required DHTRecord record,
-    required InitialStateFunction<T> initialStateFunction,
+  //   required InitialStateFunction<T> initialStateFunction,
-    required StateFunction<T> stateFunction,
+  //   required StateFunction<T> stateFunction,
-    required WatchFunction watchFunction,
+  //   required WatchFunction watchFunction,
-  })  : _record = record,
+  // })  : _record = record,
-        _stateFunction = stateFunction,
+  //       _stateFunction = stateFunction,
-        _wantsCloseRecord = false,
+  //       _wantsCloseRecord = false,
-        super(const AsyncValue.loading()) {
+  //       super(const AsyncValue.loading()) {
-    Future.delayed(Duration.zero, () async {
+  //   Future.delayed(Duration.zero, () async {
-      await _init(initialStateFunction, stateFunction, watchFunction);
+  //     await _init(initialStateFunction, stateFunction, watchFunction);
-    });
+  //   });
-  }
+  // }
 
   Future<void> _init(
     InitialStateFunction<T> initialStateFunction,
@@ -123,13 +123,13 @@ class DefaultDHTRecordCubit<T> extends DHTRecordCubit<T> {
             stateFunction: _makeStateFunction(decodeState),
             watchFunction: _makeWatchFunction());
 
-  DefaultDHTRecordCubit.value({
+  // DefaultDHTRecordCubit.value({
-    required super.record,
+  //   required super.record,
-    required T Function(List<int> data) decodeState,
+  //   required T Function(List<int> data) decodeState,
-  }) : super.value(
+  // }) : super.value(
-            initialStateFunction: _makeInitialStateFunction(decodeState),
+  //           initialStateFunction: _makeInitialStateFunction(decodeState),
-            stateFunction: _makeStateFunction(decodeState),
+  //           stateFunction: _makeStateFunction(decodeState),
-            watchFunction: _makeWatchFunction());
+  //           watchFunction: _makeWatchFunction());
 
   static InitialStateFunction<T> _makeInitialStateFunction<T>(
           T Function(List<int> data) decodeState) =>

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array.dart

@@ -1,7 +1,6 @@
 import 'dart:async';
 import 'dart:typed_data';
 
-import 'package:async_tools/async_tools.dart';
 import 'package:mutex/mutex.dart';
 import 'package:protobuf/protobuf.dart';
 
@@ -169,90 +168,36 @@ class DHTShortArray {
   }
 
   /// Runs a closure allowing read-only access to the shortarray
-  Future<T> operate<T>(Future<T> Function(DHTShortArrayRead) closure) async =>
+  Future<T?> operate<T>(Future<T?> Function(DHTShortArrayRead) closure) async =>
       _head.operate((head) async {
         final reader = _DHTShortArrayRead._(head);
         return closure(reader);
       });
 
   /// Runs a closure allowing read-write access to the shortarray
+  /// Makes only one attempt to consistently write the changes to the DHT
   /// Returns (result, true) of the closure if the write could be performed
   /// Returns (null, false) if the write could not be performed at this time
   Future<(T?, bool)> operateWrite<T>(
-          Future<T> Function(DHTShortArrayWrite) closure) async =>
+          Future<T?> Function(DHTShortArrayWrite) closure) async =>
       _head.operateWrite((head) async {
         final writer = _DHTShortArrayWrite._(head);
         return closure(writer);
       });
 
-  /// Set an item at position 'pos' of the DHTShortArray. Retries until the
+  /// Runs a closure allowing read-write access to the shortarray
-  /// value being written is successfully made the newest value of the element.
+  /// Will execute the closure multiple times if a consistent write to the DHT
-  /// This may throw an exception if the position elements the built-in limit of
+  /// is not achieved. Timeout if specified will be thrown as a
-  /// 'maxElements = 256' entries.
+  /// TimeoutException. The closure should return true if its changes also
-  Future<void> eventualWriteItem(int pos, Uint8List newValue,
+  /// succeeded, returning false will trigger another eventual consistency
-      {Duration? timeout}) async {
+  /// attempt.
-    await _head.operateWriteEventual((head) async {
+  Future<void> operateWriteEventual(
-      bool wasSet;
+          Future<bool> Function(DHTShortArrayWrite) closure,
-      (_, wasSet) = await _tryWriteItemInner(head, pos, newValue);
+          {Duration? timeout}) async =>
-      return wasSet;
+      _head.operateWriteEventual((head) async {
+        final writer = _DHTShortArrayWrite._(head);
+        return closure(writer);
       }, timeout: timeout);
-  }
-
-  /// Change an item at position 'pos' of the DHTShortArray.
-  /// Runs with the value of the old element at that position such that it can
-  /// be changed to the returned value from tha closure. Retries until the
-  /// value being written is successfully made the newest value of the element.
-  /// This may throw an exception if the position elements the built-in limit of
-  /// 'maxElements = 256' entries.
-
-  Future<void> eventualUpdateItem(
-      int pos, Future<Uint8List> Function(Uint8List? oldValue) update,
-      {Duration? timeout}) async {
-    await _head.operateWriteEventual((head) async {
-      final oldData = await getItem(pos);
-
-      // Update the data
-      final updatedData = await update(oldData);
-
-      // Set it back
-      bool wasSet;
-      (_, wasSet) = await _tryWriteItemInner(head, pos, updatedData);
-      return wasSet;
-    }, timeout: timeout);
-  }
-
-  /// Convenience function:
-  /// Like eventualWriteItem but also encodes the input value as JSON and parses
-  /// the returned element as JSON
-  Future<void> eventualWriteItemJson<T>(int pos, T newValue,
-          {Duration? timeout}) =>
-      eventualWriteItem(pos, jsonEncodeBytes(newValue), timeout: timeout);
-
-  /// Convenience function:
-  /// Like eventualWriteItem but also encodes the input value as a protobuf
-  /// object and parses the returned element as a protobuf object
-  Future<void> eventualWriteItemProtobuf<T extends GeneratedMessage>(
-          int pos, T newValue,
-          {int subkey = -1, Duration? timeout}) =>
-      eventualWriteItem(pos, newValue.writeToBuffer(), timeout: timeout);
-
-  /// Convenience function:
-  /// Like eventualUpdateItem but also encodes the input value as JSON
-  Future<void> eventualUpdateItemJson<T>(
-          T Function(dynamic) fromJson, int pos, Future<T> Function(T?) update,
-          {Duration? timeout}) =>
-      eventualUpdateItem(pos, jsonUpdate(fromJson, update), timeout: timeout);
-
-  /// Convenience function:
-  /// Like eventualUpdateItem but also encodes the input value as a protobuf
-  /// object
-  Future<void> eventualUpdateItemProtobuf<T extends GeneratedMessage>(
-          T Function(List<int>) fromBuffer,
-          int pos,
-          Future<T> Function(T?) update,
-          {Duration? timeout}) =>
-      eventualUpdateItem(pos, protobufUpdate(fromBuffer, update),
-          timeout: timeout);
 
   Future<StreamSubscription<void>> listen(
     void Function() onChanged,

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_cubit.dart

@@ -4,7 +4,6 @@ import 'package:async_tools/async_tools.dart';
 import 'package:bloc/bloc.dart';
 import 'package:bloc_tools/bloc_tools.dart';
 import 'package:fast_immutable_collections/fast_immutable_collections.dart';
-import 'package:mutex/mutex.dart';
 
 import '../../../veilid_support.dart';
 
@@ -29,18 +28,18 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
     });
   }
 
-  DHTShortArrayCubit.value({ 
+  // DHTShortArrayCubit.value({
-    required DHTShortArray shortArray,
+  //   required DHTShortArray shortArray,
-    required T Function(List<int> data) decodeElement,
+  //   required T Function(List<int> data) decodeElement,
-  })  : _shortArray = shortArray,
+  // })  : _shortArray = shortArray,
-        _decodeElement = decodeElement,
+  //       _decodeElement = decodeElement,
-        super(const BlocBusyState(AsyncValue.loading())) {
+  //       super(const BlocBusyState(AsyncValue.loading())) {
-    _initFuture = Future(() async {
+  //   _initFuture = Future(() async {
-      // Make initial state update
+  //     // Make initial state update
-      unawaited(_refreshNoWait());
+  //     unawaited(_refreshNoWait());
-      _subscription = await shortArray.listen(_update);
+  //     _subscription = await shortArray.listen(_update);
-    });
+  //   });
-  }
+  // }
 
   Future<void> refresh({bool forceRefresh = false}) async {
     await _initFuture;
@@ -48,14 +47,13 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
   }
 
   Future<void> _refreshNoWait({bool forceRefresh = false}) async =>
-      busy((emit) async => _operateMutex.protect(
+      busy((emit) async => _refreshInner(emit, forceRefresh: forceRefresh));
-          () async => _refreshInner(emit, forceRefresh: forceRefresh)));
 
   Future<void> _refreshInner(void Function(AsyncValue<IList<T>>) emit,
       {bool forceRefresh = false}) async {
     try {
-      final newState =
+      final newState = (await _shortArray.operate(
-          (await _shortArray.getAllItems(forceRefresh: forceRefresh))
+              (reader) => reader.getAllItems(forceRefresh: forceRefresh)))
           ?.map(_decodeElement)
           .toIList();
       if (newState != null) {
@@ -71,8 +69,8 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
     // Because this is async, we could get an update while we're
     // still processing the last one. Only called after init future has run
     // so we dont have to wait for that here.
-    _sspUpdate.busyUpdate<T, AsyncValue<IList<T>>>(busy,
+    _sspUpdate.busyUpdate<T, AsyncValue<IList<T>>>(
-        (emit) async => _operateMutex.protect(() async => _refreshInner(emit)));
+        busy, (emit) async => _refreshInner(emit));
   }
 
   @override
@@ -86,12 +84,24 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
     await super.close();
   }
 
-  Future<R?> operate<R>(Future<R?> Function(DHTShortArray) closure) async {
+  Future<R?> operate<R>(Future<R?> Function(DHTShortArrayRead) closure) async {
     await _initFuture;
-    return _operateMutex.protect(() async => closure(_shortArray));
+    return _shortArray.operate(closure);
+  }
+
+  Future<(R?, bool)> operateWrite<R>(
+      Future<R?> Function(DHTShortArrayWrite) closure) async {
+    await _initFuture;
+    return _shortArray.operateWrite(closure);
+  }
+
+  Future<void> operateWriteEventual(
+      Future<bool> Function(DHTShortArrayWrite) closure,
+      {Duration? timeout}) async {
+    await _initFuture;
+    return _shortArray.operateWriteEventual(closure, timeout: timeout);
   }
 
-  final _operateMutex = Mutex();
   late final Future<void> _initFuture;
   late final DHTShortArray _shortArray;
   final T Function(List<int> data) _decodeElement;

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_head.dart

@@ -32,7 +32,7 @@ class _DHTShortArrayHead {
 
     final head = proto.DHTShortArray();
     head.keys.addAll(_linkedRecords.map((lr) => lr.key.toProto()));
-    head.index.addAll(_index);
+    head.index = List.of(_index);
     head.seqs.addAll(_seqs);
     // Do not serialize free list, it gets recreated
     // Do not serialize local seqs, they are only locally relevant
@@ -58,7 +58,7 @@ class _DHTShortArrayHead {
       });
 
   Future<(T?, bool)> operateWrite<T>(
-          Future<T> Function(_DHTShortArrayHead) closure) async =>
+          Future<T?> Function(_DHTShortArrayHead) closure) async =>
       _headMutex.protect(() async {
         final oldLinkedRecords = List.of(_linkedRecords);
         final oldIndex = List.of(_index);
@@ -111,14 +111,22 @@ class _DHTShortArrayHead {
           oldSeqs = List.of(_seqs);
 
           // Try to do the element write
-          do {
+          while (true) {
             if (timeoutTs != null) {
               final now = Veilid.instance.now();
               if (now >= timeoutTs) {
                 throw TimeoutException('timeout reached');
               }
             }
-          } while (!await closure(this));
+            if (await closure(this)) {
+              break;
+            }
+            // Failed to write in closure resets state
+            _linkedRecords = List.of(oldLinkedRecords);
+            _index = List.of(oldIndex);
+            _free = List.of(oldFree);
+            _seqs = List.of(oldSeqs);
+          }
 
           // Try to do the head write
         } while (!await _writeHead());