Merge branch 'table-db-array' into 'main'

Long conversation support

See merge request veilid/veilidchat!28

README.md

@@ -12,17 +12,17 @@ While this is still in development, you must have a clone of the Veilid source c
 
 ### For Linux Systems:
 ```
-./setup_linux.sh
+./dev-setup/setup_linux.sh
 ```
 
 ### For Mac Systems:
 ```
-./setup_macos.sh
+./dev-setup/setup_macos.sh
 ```
 
 ## Updating Code
 
 ### To update the WASM binary from `veilid-wasm`:
-* Debug WASM: run `./wasm_update.sh`
-* Release WASM: run `/wasm_update.sh release`
+* Debug WASM: run `./dev-setup/wasm_update.sh`
+* Release WASM: run `./dev-setup/wasm_update.sh release`
 

assets/i18n/en.json

@@ -67,6 +67,7 @@
         "new_chat": "New Chat"
     },
     "chat": {
+        "start_a_conversation": "Start A Conversation",
         "say_something": "Say Something"
     },
     "create_invitation_dialog": {

lib/account_manager/models/active_account_info.dart

@@ -11,7 +11,6 @@ class ActiveAccountInfo {
   const ActiveAccountInfo({
     required this.localAccount,
     required this.userLogin,
-    //required this.accountRecord,
   });
   //
 
@@ -24,7 +23,7 @@ class ActiveAccountInfo {
     return KeyPair(key: identityKey, secret: identitySecret.value);
   }
 
-  Future<DHTRecordCrypto> makeConversationCrypto(
+  Future<VeilidCrypto> makeConversationCrypto(
       TypedKey remoteIdentityPublicKey) async {
     final identitySecret = userLogin.identitySecret;
     final cs = await Veilid.instance.getCryptoSystem(identitySecret.kind);
@@ -33,7 +32,7 @@ class ActiveAccountInfo {
         identitySecret.value,
         utf8.encode('VeilidChat Conversation'));
 
-    final messagesCrypto = await DHTRecordCryptoPrivate.fromSecret(
+    final messagesCrypto = await VeilidCryptoPrivate.fromSharedSecret(
         identitySecret.kind, sharedSecret);
     return messagesCrypto;
   }
@@ -41,5 +40,4 @@ class ActiveAccountInfo {
   //
   final LocalAccount localAccount;
   final UserLogin userLogin;
-  //final DHTRecord accountRecord;
 }

lib/chat/cubits/active_chat_cubit.dart

@@ -1,10 +1,13 @@
 import 'package:flutter_bloc/flutter_bloc.dart';
 import 'package:veilid_support/veilid_support.dart';
 
+// XXX: if we ever want to have more than one chat 'open', we should put the
+// operations and state for that here.
+
 class ActiveChatCubit extends Cubit<TypedKey?> {
   ActiveChatCubit(super.initialState);
 
-  void setActiveChat(TypedKey? activeChatRemoteConversationRecordKey) {
-    emit(activeChatRemoteConversationRecordKey);
+  void setActiveChat(TypedKey? activeChatLocalConversationRecordKey) {
+    emit(activeChatLocalConversationRecordKey);
   }
 }

lib/chat/cubits/chat_component_cubit.dart

@@ -0,0 +1,272 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:fixnum/fixnum.dart';
+import 'package:flutter/widgets.dart';
+import 'package:flutter_bloc/flutter_bloc.dart';
+import 'package:flutter_chat_types/flutter_chat_types.dart' as types;
+import 'package:flutter_chat_ui/flutter_chat_ui.dart';
+import 'package:scroll_to_index/scroll_to_index.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+import '../../account_manager/account_manager.dart';
+import '../../chat_list/chat_list.dart';
+import '../../proto/proto.dart' as proto;
+import '../models/chat_component_state.dart';
+import '../models/message_state.dart';
+import '../models/window_state.dart';
+import 'cubits.dart';
+
+const metadataKeyIdentityPublicKey = 'identityPublicKey';
+const metadataKeyExpirationDuration = 'expiration';
+const metadataKeyViewLimit = 'view_limit';
+const metadataKeyAttachments = 'attachments';
+
+class ChatComponentCubit extends Cubit<ChatComponentState> {
+  ChatComponentCubit._({
+    required SingleContactMessagesCubit messagesCubit,
+    required types.User localUser,
+    required IMap<TypedKey, types.User> remoteUsers,
+  })  : _messagesCubit = messagesCubit,
+        super(ChatComponentState(
+          chatKey: GlobalKey<ChatState>(),
+          scrollController: AutoScrollController(),
+          localUser: localUser,
+          remoteUsers: remoteUsers,
+          messageWindow: const AsyncLoading(),
+          title: '',
+        )) {
+    // Async Init
+    _initWait.add(_init);
+  }
+
+  // ignore: prefer_constructors_over_static_methods
+  static ChatComponentCubit singleContact(
+      {required ActiveAccountInfo activeAccountInfo,
+      required proto.Account accountRecordInfo,
+      required ActiveConversationState activeConversationState,
+      required SingleContactMessagesCubit messagesCubit}) {
+    // Make local 'User'
+    final localUserIdentityKey =
+        activeAccountInfo.localAccount.identityMaster.identityPublicTypedKey();
+    final localUser = types.User(
+        id: localUserIdentityKey.toString(),
+        firstName: accountRecordInfo.profile.name,
+        metadata: {metadataKeyIdentityPublicKey: localUserIdentityKey});
+    // Make remote 'User's
+    final remoteUsers = {
+      activeConversationState.contact.identityPublicKey.toVeilid(): types.User(
+          id: activeConversationState.contact.identityPublicKey
+              .toVeilid()
+              .toString(),
+          firstName: activeConversationState.contact.editedProfile.name,
+          metadata: {
+            metadataKeyIdentityPublicKey:
+                activeConversationState.contact.identityPublicKey.toVeilid()
+          })
+    }.toIMap();
+
+    return ChatComponentCubit._(
+      messagesCubit: messagesCubit,
+      localUser: localUser,
+      remoteUsers: remoteUsers,
+    );
+  }
+
+  Future<void> _init() async {
+    _messagesSubscription = _messagesCubit.stream.listen((messagesState) {
+      emit(state.copyWith(
+        messageWindow: _convertMessages(messagesState),
+      ));
+    });
+    emit(state.copyWith(
+      messageWindow: _convertMessages(_messagesCubit.state),
+      title: _getTitle(),
+    ));
+  }
+
+  @override
+  Future<void> close() async {
+    await _initWait();
+    await _messagesSubscription.cancel();
+    await super.close();
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Public Interface
+
+  // Set the tail position of the log for pagination.
+  // If tail is 0, the end of the log is used.
+  // If tail is negative, the position is subtracted from the current log
+  // length.
+  // If tail is positive, the position is absolute from the head of the log
+  // If follow is enabled, the tail offset will update when the log changes
+  Future<void> setWindow(
+      {int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
+    //await _initWait();
+    await _messagesCubit.setWindow(
+        tail: tail, count: count, follow: follow, forceRefresh: forceRefresh);
+  }
+
+  // Send a message
+  void sendMessage(types.PartialText message) {
+    final text = message.text;
+
+    final replyId = (message.repliedMessage != null)
+        ? base64UrlNoPadDecode(message.repliedMessage!.id)
+        : null;
+    Timestamp? expiration;
+    int? viewLimit;
+    List<proto.Attachment>? attachments;
+    final metadata = message.metadata;
+    if (metadata != null) {
+      final expirationValue =
+          metadata[metadataKeyExpirationDuration] as TimestampDuration?;
+      if (expirationValue != null) {
+        expiration = Veilid.instance.now().offset(expirationValue);
+      }
+      final viewLimitValue = metadata[metadataKeyViewLimit] as int?;
+      if (viewLimitValue != null) {
+        viewLimit = viewLimitValue;
+      }
+      final attachmentsValue =
+          metadata[metadataKeyAttachments] as List<proto.Attachment>?;
+      if (attachmentsValue != null) {
+        attachments = attachmentsValue;
+      }
+    }
+
+    _addTextMessage(
+        text: text,
+        replyId: replyId,
+        expiration: expiration,
+        viewLimit: viewLimit,
+        attachments: attachments ?? []);
+  }
+
+  // Run a chat command
+  void runCommand(String command) {
+    _messagesCubit.runCommand(command);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Private Implementation
+
+  String _getTitle() {
+    if (state.remoteUsers.length == 1) {
+      final remoteUser = state.remoteUsers.values.first;
+      return remoteUser.firstName ?? '<unnamed>';
+    } else {
+      return '<group chat with ${state.remoteUsers.length} users>';
+    }
+  }
+
+  types.Message? _messageStateToChatMessage(MessageState message) {
+    final authorIdentityPublicKey = message.content.author.toVeilid();
+    final author =
+        state.remoteUsers[authorIdentityPublicKey] ?? state.localUser;
+
+    types.Status? status;
+    if (message.sendState != null) {
+      assert(author == state.localUser,
+          'send state should only be on sent messages');
+      switch (message.sendState!) {
+        case MessageSendState.sending:
+          status = types.Status.sending;
+        case MessageSendState.sent:
+          status = types.Status.sent;
+        case MessageSendState.delivered:
+          status = types.Status.delivered;
+      }
+    }
+
+    switch (message.content.whichKind()) {
+      case proto.Message_Kind.text:
+        final contextText = message.content.text;
+        final textMessage = types.TextMessage(
+            author: author,
+            createdAt:
+                (message.sentTimestamp.value ~/ BigInt.from(1000)).toInt(),
+            id: message.content.authorUniqueIdString,
+            text: contextText.text,
+            showStatus: status != null,
+            status: status);
+        return textMessage;
+      case proto.Message_Kind.secret:
+      case proto.Message_Kind.delete:
+      case proto.Message_Kind.erase:
+      case proto.Message_Kind.settings:
+      case proto.Message_Kind.permissions:
+      case proto.Message_Kind.membership:
+      case proto.Message_Kind.moderation:
+      case proto.Message_Kind.notSet:
+        return null;
+    }
+  }
+
+  AsyncValue<WindowState<types.Message>> _convertMessages(
+      AsyncValue<WindowState<MessageState>> avMessagesState) {
+    final asError = avMessagesState.asError;
+    if (asError != null) {
+      return AsyncValue.error(asError.error, asError.stackTrace);
+    } else if (avMessagesState.asLoading != null) {
+      return const AsyncValue.loading();
+    }
+    final messagesState = avMessagesState.asData!.value;
+
+    // Convert protobuf messages to chat messages
+    final chatMessages = <types.Message>[];
+    final tsSet = <String>{};
+    for (final message in messagesState.window) {
+      final chatMessage = _messageStateToChatMessage(message);
+      if (chatMessage == null) {
+        continue;
+      }
+      chatMessages.insert(0, chatMessage);
+      if (!tsSet.add(chatMessage.id)) {
+        // ignore: avoid_print
+        print('duplicate id found: ${chatMessage.id}:\n'
+            'Messages:\n${messagesState.window}\n'
+            'ChatMessages:\n$chatMessages');
+        assert(false, 'should not have duplicate id');
+      }
+    }
+    return AsyncValue.data(WindowState<types.Message>(
+        window: chatMessages.toIList(),
+        length: messagesState.length,
+        windowTail: messagesState.windowTail,
+        windowCount: messagesState.windowCount,
+        follow: messagesState.follow));
+  }
+
+  void _addTextMessage(
+      {required String text,
+      String? topic,
+      Uint8List? replyId,
+      Timestamp? expiration,
+      int? viewLimit,
+      List<proto.Attachment> attachments = const []}) {
+    final protoMessageText = proto.Message_Text()..text = text;
+    if (topic != null) {
+      protoMessageText.topic = topic;
+    }
+    if (replyId != null) {
+      protoMessageText.replyId = replyId;
+    }
+    protoMessageText
+      ..expiration = expiration?.toInt64() ?? Int64.ZERO
+      ..viewLimit = viewLimit ?? 0;
+    protoMessageText.attachments.addAll(attachments);
+
+    _messagesCubit.sendTextMessage(messageText: protoMessageText);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  final _initWait = WaitSet<void>();
+  final SingleContactMessagesCubit _messagesCubit;
+  late StreamSubscription<SingleContactMessagesState> _messagesSubscription;
+  double scrollOffset = 0;
+}

lib/chat/cubits/cubits.dart

@@ -1,2 +1,3 @@
 export 'active_chat_cubit.dart';
+export 'chat_component_cubit.dart';
 export 'single_contact_messages_cubit.dart';

lib/chat/cubits/reconciliation/author_input_queue.dart

@@ -0,0 +1,213 @@
+import 'dart:async';
+import 'package:veilid_support/veilid_support.dart';
+
+import '../../../proto/proto.dart' as proto;
+
+import 'author_input_source.dart';
+import 'message_integrity.dart';
+import 'output_position.dart';
+
+class AuthorInputQueue {
+  AuthorInputQueue._({
+    required TypedKey author,
+    required AuthorInputSource inputSource,
+    required OutputPosition? outputPosition,
+    required void Function(Object, StackTrace?) onError,
+    required MessageIntegrity messageIntegrity,
+  })  : _author = author,
+        _onError = onError,
+        _inputSource = inputSource,
+        _outputPosition = outputPosition,
+        _lastMessage = outputPosition?.message.content,
+        _messageIntegrity = messageIntegrity,
+        _currentPosition = inputSource.currentWindow.last;
+
+  static Future<AuthorInputQueue?> create({
+    required TypedKey author,
+    required AuthorInputSource inputSource,
+    required OutputPosition? outputPosition,
+    required void Function(Object, StackTrace?) onError,
+  }) async {
+    final queue = AuthorInputQueue._(
+        author: author,
+        inputSource: inputSource,
+        outputPosition: outputPosition,
+        onError: onError,
+        messageIntegrity: await MessageIntegrity.create(author: author));
+    if (!await queue._findStartOfWork()) {
+      return null;
+    }
+    return queue;
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Public interface
+
+  // Check if there are no messages left in this queue to reconcile
+  bool get isDone => _isDone;
+
+  // Get the current message that needs reconciliation
+  proto.Message? get current => _currentMessage;
+
+  // Get the earliest output position to start inserting
+  OutputPosition? get outputPosition => _outputPosition;
+
+  // Get the author of this queue
+  TypedKey get author => _author;
+
+  // Remove a reconciled message and move to the next message
+  // Returns true if there is more work to do
+  Future<bool> consume() async {
+    if (_isDone) {
+      return false;
+    }
+    while (true) {
+      _lastMessage = _currentMessage;
+
+      _currentPosition++;
+
+      // Get more window if we need to
+      if (!await _updateWindow()) {
+        // Window is not available so this queue can't work right now
+        _isDone = true;
+        return false;
+      }
+      final nextMessage = _inputSource.currentWindow
+          .elements[_currentPosition - _inputSource.currentWindow.first];
+
+      // Drop the 'offline' elements because we don't reconcile
+      // anything until it has been confirmed to be committed to the DHT
+      // if (nextMessage.isOffline) {
+      //   continue;
+      // }
+
+      if (_lastMessage != null) {
+        // Ensure the timestamp is not moving backward
+        if (nextMessage.value.timestamp < _lastMessage!.timestamp) {
+          continue;
+        }
+      }
+
+      // Verify the id chain for the message
+      final matchId = await _messageIntegrity.generateMessageId(_lastMessage);
+      if (matchId.compare(nextMessage.value.idBytes) != 0) {
+        continue;
+      }
+
+      // Verify the signature for the message
+      if (!await _messageIntegrity.verifyMessage(nextMessage.value)) {
+        continue;
+      }
+
+      _currentMessage = nextMessage.value;
+      break;
+    }
+    return true;
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Internal implementation
+
+  // Walk backward from the tail of the input queue to find the first
+  // message newer than our last reconciled message from this author
+  // Returns false if no work is needed
+  Future<bool> _findStartOfWork() async {
+    // Iterate windows over the inputSource
+    outer:
+    while (true) {
+      // Iterate through current window backward
+      for (var i = _inputSource.currentWindow.elements.length - 1;
+          i >= 0 && _currentPosition >= 0;
+          i--, _currentPosition--) {
+        final elem = _inputSource.currentWindow.elements[i];
+
+        // If we've found an input element that is older or same time as our
+        // last reconciled message for this author, or we find the message
+        // itself then we stop
+        if (_lastMessage != null) {
+          if (elem.value.authorUniqueIdBytes
+                      .compare(_lastMessage!.authorUniqueIdBytes) ==
+                  0 ||
+              elem.value.timestamp <= _lastMessage!.timestamp) {
+            break outer;
+          }
+        }
+      }
+      // If we're at the beginning of the inputSource then we stop
+      if (_currentPosition < 0) {
+        break;
+      }
+
+      // Get more window if we need to
+      if (!await _updateWindow()) {
+        // Window is not available or things are empty so this
+        // queue can't work right now
+        _isDone = true;
+        return false;
+      }
+    }
+
+    // _currentPosition points to either before the input source starts
+    // or the position of the previous element. We still need to set the
+    // _currentMessage to the previous element so consume() can compare
+    // against it if we can.
+    if (_currentPosition >= 0) {
+      _currentMessage = _inputSource.currentWindow
+          .elements[_currentPosition - _inputSource.currentWindow.first].value;
+    }
+
+    // After this consume(), the currentPosition and _currentMessage should
+    // be equal to the first message to process and the current window to
+    // process should not be empty
+    return consume();
+  }
+
+  // Slide the window toward the current position and load the batch around it
+  Future<bool> _updateWindow() async {
+    // Check if we are still in the window
+    if (_currentPosition >= _inputSource.currentWindow.first &&
+        _currentPosition <= _inputSource.currentWindow.last) {
+      return true;
+    }
+
+    // Get another input batch futher back
+    final avOk =
+        await _inputSource.updateWindow(_currentPosition, _maxWindowLength);
+
+    final asErr = avOk.asError;
+    if (asErr != null) {
+      _onError(asErr.error, asErr.stackTrace);
+      return false;
+    }
+    final asLoading = avOk.asLoading;
+    if (asLoading != null) {
+      // xxx: no need to block the cubit here for this
+      // xxx: might want to switch to a 'busy' state though
+      // xxx: to let the messages view show a spinner at the bottom
+      // xxx: while we reconcile...
+      // emit(const AsyncValue.loading());
+      return false;
+    }
+    return avOk.asData!.value;
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  final TypedKey _author;
+  final AuthorInputSource _inputSource;
+  final OutputPosition? _outputPosition;
+  final void Function(Object, StackTrace?) _onError;
+  final MessageIntegrity _messageIntegrity;
+
+  // The last message we've consumed
+  proto.Message? _lastMessage;
+  // The current position in the input log that we are looking at
+  int _currentPosition;
+  // The current message we're looking at
+  proto.Message? _currentMessage;
+  // If we have reached the end
+  bool _isDone = false;
+
+  // Desired maximum window length
+  static const int _maxWindowLength = 256;
+}

lib/chat/cubits/reconciliation/author_input_source.dart

@@ -0,0 +1,77 @@
+import 'dart:math';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:meta/meta.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+import '../../../proto/proto.dart' as proto;
+
+@immutable
+class InputWindow {
+  const InputWindow(
+      {required this.elements, required this.first, required this.last});
+  final IList<OnlineElementState<proto.Message>> elements;
+  final int first;
+  final int last;
+}
+
+class AuthorInputSource {
+  AuthorInputSource.fromCubit(
+      {required DHTLogStateData<proto.Message> cubitState,
+      required this.cubit}) {
+    _currentWindow = InputWindow(
+        elements: cubitState.window,
+        first: (cubitState.windowTail - cubitState.window.length) %
+            cubitState.length,
+        last: (cubitState.windowTail - 1) % cubitState.length);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  InputWindow get currentWindow => _currentWindow;
+
+  Future<AsyncValue<bool>> updateWindow(
+          int currentPosition, int windowLength) async =>
+      cubit.operate((reader) async {
+        // See if we're beyond the input source
+        if (currentPosition < 0 || currentPosition >= reader.length) {
+          return const AsyncValue.data(false);
+        }
+
+        // Slide the window if we need to
+        var first = _currentWindow.first;
+        var last = _currentWindow.last;
+        if (currentPosition < first) {
+          // Slide it backward, current position is now last
+          first = max((currentPosition - windowLength) + 1, 0);
+          last = currentPosition;
+        } else if (currentPosition > last) {
+          // Slide it forward, current position is now first
+          first = currentPosition;
+          last = min((currentPosition + windowLength) - 1, reader.length - 1);
+        } else {
+          return const AsyncValue.data(true);
+        }
+
+        // Get another input batch futher back
+        final nextWindow = await cubit.loadElementsFromReader(
+            reader, last + 1, (last + 1) - first);
+        final asErr = nextWindow.asError;
+        if (asErr != null) {
+          return AsyncValue.error(asErr.error, asErr.stackTrace);
+        }
+        final asLoading = nextWindow.asLoading;
+        if (asLoading != null) {
+          return const AsyncValue.loading();
+        }
+        _currentWindow = InputWindow(
+            elements: nextWindow.asData!.value, first: first, last: last);
+        return const AsyncValue.data(true);
+      });
+
+  ////////////////////////////////////////////////////////////////////////////
+  final DHTLogCubit<proto.Message> cubit;
+
+  late InputWindow _currentWindow;
+}

lib/chat/cubits/reconciliation/message_integrity.dart

@@ -0,0 +1,74 @@
+import 'dart:convert';
+import 'dart:typed_data';
+
+import 'package:protobuf/protobuf.dart';
+import 'package:veilid_support/veilid_support.dart';
+import '../../../proto/proto.dart' as proto;
+
+class MessageIntegrity {
+  MessageIntegrity._({
+    required TypedKey author,
+    required VeilidCryptoSystem crypto,
+  })  : _author = author,
+        _crypto = crypto;
+  static Future<MessageIntegrity> create({required TypedKey author}) async {
+    final crypto = await Veilid.instance.getCryptoSystem(author.kind);
+    return MessageIntegrity._(author: author, crypto: crypto);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Public interface
+
+  Future<Uint8List> generateMessageId(proto.Message? previous) async {
+    if (previous == null) {
+      // If there's no last sent message,
+      // we start at a hash of the identity public key
+      return _generateInitialId();
+    } else {
+      // If there is a last message, we generate the hash
+      // of the last message's signature and use it as our next id
+      return _hashSignature(previous.signature);
+    }
+  }
+
+  Future<void> signMessage(
+    proto.Message message,
+    SecretKey authorSecret,
+  ) async {
+    // Ensure this message is not already signed
+    assert(!message.hasSignature(), 'should not sign message twice');
+    // Generate data to sign
+    final data = Uint8List.fromList(utf8.encode(message.writeToJson()));
+
+    // Sign with our identity
+    final signature = await _crypto.sign(_author.value, authorSecret, data);
+
+    // Add to the message
+    message.signature = signature.toProto();
+  }
+
+  Future<bool> verifyMessage(proto.Message message) async {
+    // Ensure the message is signed
+    assert(message.hasSignature(), 'should not verify unsigned message');
+    final signature = message.signature.toVeilid();
+
+    // Generate data to sign
+    final messageNoSig = message.deepCopy()..clearSignature();
+    final data = Uint8List.fromList(utf8.encode(messageNoSig.writeToJson()));
+
+    // Verify signature
+    return _crypto.verify(_author.value, data, signature);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Private implementation
+
+  Future<Uint8List> _generateInitialId() async =>
+      (await _crypto.generateHash(_author.decode())).decode();
+
+  Future<Uint8List> _hashSignature(proto.Signature signature) async =>
+      (await _crypto.generateHash(signature.toVeilid().decode())).decode();
+  ////////////////////////////////////////////////////////////////////////////
+  final TypedKey _author;
+  final VeilidCryptoSystem _crypto;
+}

lib/chat/cubits/reconciliation/message_reconciliation.dart

@@ -0,0 +1,195 @@
+import 'dart:async';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:sorted_list/sorted_list.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+import '../../../proto/proto.dart' as proto;
+import 'author_input_queue.dart';
+import 'author_input_source.dart';
+import 'output_position.dart';
+
+class MessageReconciliation {
+  MessageReconciliation(
+      {required TableDBArrayProtobufCubit<proto.ReconciledMessage> output,
+      required void Function(Object, StackTrace?) onError})
+      : _outputCubit = output,
+        _onError = onError;
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  void reconcileMessages(
+      TypedKey author,
+      DHTLogStateData<proto.Message> inputMessagesCubitState,
+      DHTLogCubit<proto.Message> inputMessagesCubit) {
+    if (inputMessagesCubitState.window.isEmpty) {
+      return;
+    }
+
+    _inputSources[author] = AuthorInputSource.fromCubit(
+        cubitState: inputMessagesCubitState, cubit: inputMessagesCubit);
+
+    singleFuture(this, onError: _onError, () async {
+      // Take entire list of input sources we have currently and process them
+      final inputSources = _inputSources;
+      _inputSources = {};
+
+      final inputFuts = <Future<AuthorInputQueue?>>[];
+      for (final kv in inputSources.entries) {
+        final author = kv.key;
+        final inputSource = kv.value;
+        inputFuts
+            .add(_enqueueAuthorInput(author: author, inputSource: inputSource));
+      }
+      final inputQueues = await inputFuts.wait;
+
+      // Make this safe to cast by removing inputs that were rejected or empty
+      inputQueues.removeNulls();
+
+      // Process all input queues together
+      await _outputCubit
+          .operate((reconciledArray) async => _reconcileInputQueues(
+                reconciledArray: reconciledArray,
+                inputQueues: inputQueues.cast<AuthorInputQueue>(),
+              ));
+    });
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  // Set up a single author's message reconciliation
+  Future<AuthorInputQueue?> _enqueueAuthorInput(
+      {required TypedKey author,
+      required AuthorInputSource inputSource}) async {
+    // Get the position of our most recent reconciled message from this author
+    final outputPosition = await _findLastOutputPosition(author: author);
+
+    // Find oldest message we have not yet reconciled
+    final inputQueue = await AuthorInputQueue.create(
+      author: author,
+      inputSource: inputSource,
+      outputPosition: outputPosition,
+      onError: _onError,
+    );
+    return inputQueue;
+  }
+
+  // Get the position of our most recent reconciled message from this author
+  // XXX: For a group chat, this should find when the author
+  // was added to the membership so we don't just go back in time forever
+  Future<OutputPosition?> _findLastOutputPosition(
+          {required TypedKey author}) async =>
+      _outputCubit.operate((arr) async {
+        var pos = arr.length - 1;
+        while (pos >= 0) {
+          final message = await arr.get(pos);
+          if (message.content.author.toVeilid() == author) {
+            return OutputPosition(message, pos);
+          }
+          pos--;
+        }
+        return null;
+      });
+
+  // Process a list of author input queues and insert their messages
+  // into the output array, performing validation steps along the way
+  Future<void> _reconcileInputQueues({
+    required TableDBArrayProtobuf<proto.ReconciledMessage> reconciledArray,
+    required List<AuthorInputQueue> inputQueues,
+  }) async {
+    // Ensure queues all have something to do
+    inputQueues.removeWhere((q) => q.isDone);
+    if (inputQueues.isEmpty) {
+      return;
+    }
+
+    // Sort queues from earliest to latest and then by author
+    // to ensure a deterministic insert order
+    inputQueues.sort((a, b) {
+      final acmp = a.outputPosition?.pos ?? -1;
+      final bcmp = b.outputPosition?.pos ?? -1;
+      if (acmp == bcmp) {
+        return a.author.toString().compareTo(b.author.toString());
+      }
+      return acmp.compareTo(bcmp);
+    });
+
+    // Start at the earliest position we know about in all the queues
+    var currentOutputPosition = inputQueues.first.outputPosition;
+
+    final toInsert =
+        SortedList<proto.Message>(proto.MessageExt.compareTimestamp);
+
+    while (inputQueues.isNotEmpty) {
+      // Get up to '_maxReconcileChunk' of the items from the queues
+      // that we can insert at this location
+
+      bool added;
+      do {
+        added = false;
+        var someQueueEmpty = false;
+        for (final inputQueue in inputQueues) {
+          final inputCurrent = inputQueue.current!;
+          if (currentOutputPosition == null ||
+              inputCurrent.timestamp <
+                  currentOutputPosition.message.content.timestamp) {
+            toInsert.add(inputCurrent);
+            added = true;
+
+            // Advance this queue
+            if (!await inputQueue.consume()) {
+              // Queue is empty now, run a queue purge
+              someQueueEmpty = true;
+            }
+          }
+        }
+        // Remove empty queues now that we're done iterating
+        if (someQueueEmpty) {
+          inputQueues.removeWhere((q) => q.isDone);
+        }
+
+        if (toInsert.length >= _maxReconcileChunk) {
+          break;
+        }
+      } while (added);
+
+      // Perform insertions in bulk
+      if (toInsert.isNotEmpty) {
+        final reconciledTime = Veilid.instance.now().toInt64();
+
+        // Add reconciled timestamps
+        final reconciledInserts = toInsert
+            .map((message) => proto.ReconciledMessage()
+              ..reconciledTime = reconciledTime
+              ..content = message)
+            .toList();
+
+        await reconciledArray.insertAll(
+            currentOutputPosition?.pos ?? reconciledArray.length,
+            reconciledInserts);
+
+        toInsert.clear();
+      } else {
+        // If there's nothing to insert at this position move to the next one
+        final nextOutputPos = (currentOutputPosition != null)
+            ? currentOutputPosition.pos + 1
+            : reconciledArray.length;
+        if (nextOutputPos == reconciledArray.length) {
+          currentOutputPosition = null;
+        } else {
+          currentOutputPosition = OutputPosition(
+              await reconciledArray.get(nextOutputPos), nextOutputPos);
+        }
+      }
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  Map<TypedKey, AuthorInputSource> _inputSources = {};
+  final TableDBArrayProtobufCubit<proto.ReconciledMessage> _outputCubit;
+  final void Function(Object, StackTrace?) _onError;
+
+  static const int _maxReconcileChunk = 65536;
+}

lib/chat/cubits/reconciliation/output_position.dart

@@ -0,0 +1,13 @@
+import 'package:equatable/equatable.dart';
+import 'package:meta/meta.dart';
+
+import '../../../proto/proto.dart' as proto;
+
+@immutable
+class OutputPosition extends Equatable {
+  const OutputPosition(this.message, this.pos);
+  final proto.ReconciledMessage message;
+  final int pos;
+  @override
+  List<Object?> get props => [message, pos];
+}

lib/chat/cubits/reconciliation/reconciliation.dart

@@ -0,0 +1,2 @@
+export 'message_integrity.dart';
+export 'message_reconciliation.dart';

lib/chat/cubits/single_contact_messages_cubit.dart

@@ -2,20 +2,21 @@ import 'dart:async';
 
 import 'package:async_tools/async_tools.dart';
 import 'package:fast_immutable_collections/fast_immutable_collections.dart';
-import 'package:fixnum/fixnum.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter_bloc/flutter_bloc.dart';
 import 'package:veilid_support/veilid_support.dart';
 
 import '../../account_manager/account_manager.dart';
 import '../../proto/proto.dart' as proto;
+import '../../tools/tools.dart';
 import '../models/models.dart';
+import 'reconciliation/reconciliation.dart';
 
 class RenderStateElement {
   RenderStateElement(
       {required this.message,
       required this.isLocal,
-      this.reconciled = false,
-      this.reconciledOffline = false,
+      this.reconciledTimestamp,
       this.sent = false,
       this.sentOffline = false});
 
@@ -23,25 +24,27 @@ class RenderStateElement {
     if (!isLocal) {
       return null;
     }
-
-    if (sent && !sentOffline) {
+    if (reconciledTimestamp != null) {
       return MessageSendState.delivered;
     }
-    if (reconciled && !reconciledOffline) {
+    if (sent) {
+      if (!sentOffline) {
         return MessageSendState.sent;
-    }
+      } else {
         return MessageSendState.sending;
       }
+    }
+    return null;
+  }
 
   proto.Message message;
   bool isLocal;
-  bool reconciled;
-  bool reconciledOffline;
+  Timestamp? reconciledTimestamp;
   bool sent;
   bool sentOffline;
 }
 
-typedef SingleContactMessagesState = AsyncValue<IList<MessageState>>;
+typedef SingleContactMessagesState = AsyncValue<WindowState<MessageState>>;
 
 // Cubit that processes single-contact chats
 // Builds the reconciled chat record from the local and remote conversation keys
@@ -53,14 +56,13 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
     required TypedKey localMessagesRecordKey,
     required TypedKey remoteConversationRecordKey,
     required TypedKey remoteMessagesRecordKey,
-    required OwnedDHTRecordPointer reconciledChatRecord,
   })  : _activeAccountInfo = activeAccountInfo,
         _remoteIdentityPublicKey = remoteIdentityPublicKey,
         _localConversationRecordKey = localConversationRecordKey,
         _localMessagesRecordKey = localMessagesRecordKey,
         _remoteConversationRecordKey = remoteConversationRecordKey,
         _remoteMessagesRecordKey = remoteMessagesRecordKey,
-        _reconciledChatRecord = reconciledChatRecord,
+        _commandController = StreamController(),
         super(const AsyncValue.loading()) {
     // Async Init
     _initWait.add(_init);
@@ -70,8 +72,9 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   Future<void> close() async {
     await _initWait();
 
-    await _unreconciledMessagesQueue.close();
-    await _sendingMessagesQueue.close();
+    await _commandController.close();
+    await _commandRunnerFut;
+    await _unsentMessagesQueue.close();
     await _sentSubscription?.cancel();
     await _rcvdSubscription?.cancel();
     await _reconciledSubscription?.cancel();
@@ -83,22 +86,15 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
   // Initialize everything
   Future<void> _init() async {
-    // Late initialization of queues with closures
-    _unreconciledMessagesQueue = PersistentQueue<proto.Message>(
-      table: 'SingleContactUnreconciledMessages',
+    _unsentMessagesQueue = PersistentQueue<proto.Message>(
+      table: 'SingleContactUnsentMessages',
       key: _remoteConversationRecordKey.toString(),
       fromBuffer: proto.Message.fromBuffer,
-      closure: _processUnreconciledMessages,
-    );
-    _sendingMessagesQueue = PersistentQueue<proto.Message>(
-      table: 'SingleContactSendingMessages',
-      key: _remoteConversationRecordKey.toString(),
-      fromBuffer: proto.Message.fromBuffer,
-      closure: _processSendingMessages,
+      closure: _processUnsentMessages,
     );
 
     // Make crypto
-    await _initMessagesCrypto();
+    await _initCrypto();
 
     // Reconciled messages key
     await _initReconciledMessagesCubit();
@@ -108,25 +104,30 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
     // Remote messages key
     await _initRcvdMessagesCubit();
+
+    // Command execution background process
+    _commandRunnerFut = Future.delayed(Duration.zero, _commandRunner);
   }
 
   // Make crypto
-  Future<void> _initMessagesCrypto() async {
-    _messagesCrypto = await _activeAccountInfo
+  Future<void> _initCrypto() async {
+    _conversationCrypto = await _activeAccountInfo
         .makeConversationCrypto(_remoteIdentityPublicKey);
+    _senderMessageIntegrity = await MessageIntegrity.create(
+        author: _activeAccountInfo.localAccount.identityMaster
+            .identityPublicTypedKey());
   }
 
   // Open local messages key
   Future<void> _initSentMessagesCubit() async {
     final writer = _activeAccountInfo.conversationWriter;
 
-    _sentMessagesCubit = DHTShortArrayCubit(
-        open: () async => DHTShortArray.openWrite(
-            _localMessagesRecordKey, writer,
+    _sentMessagesCubit = DHTLogCubit(
+        open: () async => DHTLog.openWrite(_localMessagesRecordKey, writer,
             debugName: 'SingleContactMessagesCubit::_initSentMessagesCubit::'
                 'SentMessages',
             parent: _localConversationRecordKey,
-            crypto: _messagesCrypto),
+            crypto: _conversationCrypto),
         decodeElement: proto.Message.fromBuffer);
     _sentSubscription =
         _sentMessagesCubit!.stream.listen(_updateSentMessagesState);
@@ -135,156 +136,166 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
   // Open remote messages key
   Future<void> _initRcvdMessagesCubit() async {
-    _rcvdMessagesCubit = DHTShortArrayCubit(
-        open: () async => DHTShortArray.openRead(_remoteMessagesRecordKey,
+    _rcvdMessagesCubit = DHTLogCubit(
+        open: () async => DHTLog.openRead(_remoteMessagesRecordKey,
             debugName: 'SingleContactMessagesCubit::_initRcvdMessagesCubit::'
                 'RcvdMessages',
             parent: _remoteConversationRecordKey,
-            crypto: _messagesCrypto),
+            crypto: _conversationCrypto),
         decodeElement: proto.Message.fromBuffer);
     _rcvdSubscription =
         _rcvdMessagesCubit!.stream.listen(_updateRcvdMessagesState);
     _updateRcvdMessagesState(_rcvdMessagesCubit!.state);
   }
 
+  Future<VeilidCrypto> _makeLocalMessagesCrypto() async =>
+      VeilidCryptoPrivate.fromTypedKey(
+          _activeAccountInfo.userLogin.identitySecret, 'tabledb');
+
   // Open reconciled chat record key
   Future<void> _initReconciledMessagesCubit() async {
-    final accountRecordKey =
-        _activeAccountInfo.userLogin.accountRecordInfo.accountRecord.recordKey;
+    final tableName =
+        _reconciledMessagesTableDBName(_localConversationRecordKey);
+
+    final crypto = await _makeLocalMessagesCrypto();
+
+    _reconciledMessagesCubit = TableDBArrayProtobufCubit(
+      open: () async => TableDBArrayProtobuf.make(
+          table: tableName,
+          crypto: crypto,
+          fromBuffer: proto.ReconciledMessage.fromBuffer),
+    );
+
+    _reconciliation = MessageReconciliation(
+        output: _reconciledMessagesCubit!,
+        onError: (e, st) {
+          emit(AsyncValue.error(e, st));
+        });
 
-    _reconciledMessagesCubit = DHTShortArrayCubit(
-        open: () async => DHTShortArray.openOwned(_reconciledChatRecord,
-            debugName:
-                'SingleContactMessagesCubit::_initReconciledMessagesCubit::'
-                'ReconciledMessages',
-            parent: accountRecordKey),
-        decodeElement: proto.Message.fromBuffer);
     _reconciledSubscription =
         _reconciledMessagesCubit!.stream.listen(_updateReconciledMessagesState);
     _updateReconciledMessagesState(_reconciledMessagesCubit!.state);
   }
 
   ////////////////////////////////////////////////////////////////////////////
+  // Public interface
+
+  // Set the tail position of the log for pagination.
+  // If tail is 0, the end of the log is used.
+  // If tail is negative, the position is subtracted from the current log
+  // length.
+  // If tail is positive, the position is absolute from the head of the log
+  // If follow is enabled, the tail offset will update when the log changes
+  Future<void> setWindow(
+      {int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
+    await _initWait();
+
+    print('setWindow: tail=$tail count=$count, follow=$follow');
+
+    await _reconciledMessagesCubit!.setWindow(
+        tail: tail, count: count, follow: follow, forceRefresh: forceRefresh);
+  }
+
+  // Set a user-visible 'text' message with possible attachments
+  void sendTextMessage({required proto.Message_Text messageText}) {
+    final message = proto.Message()..text = messageText;
+    _sendMessage(message: message);
+  }
+
+  // Run a chat command
+  void runCommand(String command) {
+    final (cmd, rest) = command.splitOnce(' ');
+
+    if (kDebugMode) {
+      if (cmd == '/repeat' && rest != null) {
+        final (countStr, text) = rest.splitOnce(' ');
+        final count = int.tryParse(countStr);
+        if (count != null) {
+          runCommandRepeat(count, text ?? '');
+        }
+      }
+    }
+  }
+
+  // Run a repeat command
+  void runCommandRepeat(int count, String text) {
+    _commandController.sink.add(() async {
+      for (var i = 0; i < count; i++) {
+        final protoMessageText = proto.Message_Text()
+          ..text = text.replaceAll(RegExp(r'\$n\b'), i.toString());
+        final message = proto.Message()..text = protoMessageText;
+        _sendMessage(message: message);
+        await Future<void>.delayed(const Duration(milliseconds: 50));
+      }
+    });
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Internal implementation
 
   // Called when the sent messages cubit gets a change
   // This will re-render when messages are sent from another machine
-  void _updateSentMessagesState(
-      DHTShortArrayBusyState<proto.Message> avmessages) {
+  void _updateSentMessagesState(DHTLogBusyState<proto.Message> avmessages) {
     final sentMessages = avmessages.state.asData?.value;
     if (sentMessages == null) {
       return;
     }
-    // Don't reconcile, the sending machine will have already added
-    // to the reconciliation queue on that machine
+
+    _reconciliation.reconcileMessages(
+        _activeAccountInfo.localAccount.identityMaster.identityPublicTypedKey(),
+        sentMessages,
+        _sentMessagesCubit!);
 
     // Update the view
     _renderState();
   }
 
   // Called when the received messages cubit gets a change
-  void _updateRcvdMessagesState(
-      DHTShortArrayBusyState<proto.Message> avmessages) {
+  void _updateRcvdMessagesState(DHTLogBusyState<proto.Message> avmessages) {
     final rcvdMessages = avmessages.state.asData?.value;
     if (rcvdMessages == null) {
       return;
     }
 
-    // Add remote messages updates to queue to process asynchronously
-    // Ignore offline state because remote messages are always fully delivered
-    // This may happen once per client but should be idempotent
-    _unreconciledMessagesQueue.addAllSync(rcvdMessages.map((x) => x.value));
-
-    // Update the view
-    _renderState();
+    _reconciliation.reconcileMessages(
+        _remoteIdentityPublicKey, rcvdMessages, _rcvdMessagesCubit!);
   }
 
-  // Called when the reconciled messages list gets a change
-  // This can happen when multiple clients for the same identity are
-  // reading and reconciling the same remote chat
+  // Called when the reconciled messages window gets a change
   void _updateReconciledMessagesState(
-      DHTShortArrayBusyState<proto.Message> avmessages) {
+      TableDBArrayProtobufBusyState<proto.ReconciledMessage> avmessages) {
     // Update the view
     _renderState();
   }
 
-  // Async process to reconcile messages sent or received in the background
-  Future<void> _processUnreconciledMessages(
-      IList<proto.Message> messages) async {
-    await _reconciledMessagesCubit!
-        .operateWrite((reconciledMessagesWriter) async {
-      await _reconcileMessagesInner(
-          reconciledMessagesWriter: reconciledMessagesWriter,
-          messages: messages);
-    });
+  Future<void> _processMessageToSend(
+      proto.Message message, proto.Message? previousMessage) async {
+    // Get the previous message if we don't have one
+    previousMessage ??= await _sentMessagesCubit!.operate((r) async =>
+        r.length == 0
+            ? null
+            : await r.getProtobuf(proto.Message.fromBuffer, r.length - 1));
+
+    message.id =
+        await _senderMessageIntegrity.generateMessageId(previousMessage);
+
+    // Now sign it
+    await _senderMessageIntegrity.signMessage(
+        message, _activeAccountInfo.userLogin.identitySecret.value);
   }
 
   // Async process to send messages in the background
-  Future<void> _processSendingMessages(IList<proto.Message> messages) async {
-    for (final message in messages) {
-      await _sentMessagesCubit!.operateWriteEventual(
-          (writer) => writer.tryAddItem(message.writeToBuffer()));
-    }
+  Future<void> _processUnsentMessages(IList<proto.Message> messages) async {
+    // Go through and assign ids to all the messages in order
+    proto.Message? previousMessage;
+    final processedMessages = messages.toList();
+    for (final message in processedMessages) {
+      await _processMessageToSend(message, previousMessage);
+      previousMessage = message;
     }
 
-  Future<void> _reconcileMessagesInner(
-      {required DHTRandomReadWrite reconciledMessagesWriter,
-      required IList<proto.Message> messages}) async {
-    // Ensure remoteMessages is sorted by timestamp
-    final newMessages = messages
-        .sort((a, b) => a.timestamp.compareTo(b.timestamp))
-        .removeDuplicates();
-
-    // Existing messages will always be sorted by timestamp so merging is easy
-    final existingMessages = await reconciledMessagesWriter
-        .getItemRangeProtobuf(proto.Message.fromBuffer, 0);
-    if (existingMessages == null) {
-      throw Exception(
-          'Could not load existing reconciled messages at this time');
-    }
-
-    var ePos = 0;
-    var nPos = 0;
-    while (ePos < existingMessages.length && nPos < newMessages.length) {
-      final existingMessage = existingMessages[ePos];
-      final newMessage = newMessages[nPos];
-
-      // If timestamp to insert is less than
-      // the current position, insert it here
-      final newTs = Timestamp.fromInt64(newMessage.timestamp);
-      final existingTs = Timestamp.fromInt64(existingMessage.timestamp);
-      final cmp = newTs.compareTo(existingTs);
-      if (cmp < 0) {
-        // New message belongs here
-
-        // Insert into dht backing array
-        await reconciledMessagesWriter.tryInsertItem(
-            ePos, newMessage.writeToBuffer());
-        // Insert into local copy as well for this operation
-        existingMessages.insert(ePos, newMessage);
-
-        // Next message
-        nPos++;
-        ePos++;
-      } else if (cmp == 0) {
-        // Duplicate, skip
-        nPos++;
-        ePos++;
-      } else if (cmp > 0) {
-        // New message belongs later
-        ePos++;
-      }
-    }
-    // If there are any new messages left, append them all
-    while (nPos < newMessages.length) {
-      final newMessage = newMessages[nPos];
-
-      // Append to dht backing array
-      await reconciledMessagesWriter.tryAddItem(newMessage.writeToBuffer());
-      // Insert into local copy as well for this operation
-      existingMessages.add(newMessage);
-
-      nPos++;
-    }
+    await _sentMessagesCubit!.operateAppendEventual((writer) =>
+        writer.tryAddAll(messages.map((m) => m.writeToBuffer()).toList()));
   }
 
   // Produce a state for this cubit from the input cubits and queues
@@ -294,10 +305,8 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         _reconciledMessagesCubit?.state.state.asData?.value;
     // Get all sent messages
     final sentMessages = _sentMessagesCubit?.state.state.asData?.value;
-    // Get all items in the unreconciled queue
-    final unreconciledMessages = _unreconciledMessagesQueue.queue;
     // Get all items in the unsent queue
-    final sendingMessages = _sendingMessagesQueue.queue;
+    // final unsentMessages = _unsentMessagesQueue.queue;
 
     // If we aren't ready to render a state, say we're loading
     if (reconciledMessages == null || sentMessages == null) {
@@ -306,91 +315,98 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
     }
 
     // Generate state for each message
+    // final reconciledMessagesMap =
+    //     IMap<String, proto.ReconciledMessage>.fromValues(
+    //   keyMapper: (x) => x.content.authorUniqueIdString,
+    //   values: reconciledMessages.elements,
+    // );
     final sentMessagesMap =
-        IMap<Int64, DHTShortArrayElementState<proto.Message>>.fromValues(
-      keyMapper: (x) => x.value.timestamp,
-      values: sentMessages,
-    );
-    final reconciledMessagesMap =
-        IMap<Int64, DHTShortArrayElementState<proto.Message>>.fromValues(
-      keyMapper: (x) => x.value.timestamp,
-      values: reconciledMessages,
-    );
-    final sendingMessagesMap = IMap<Int64, proto.Message>.fromValues(
-      keyMapper: (x) => x.timestamp,
-      values: sendingMessages,
-    );
-    final unreconciledMessagesMap = IMap<Int64, proto.Message>.fromValues(
-      keyMapper: (x) => x.timestamp,
-      values: unreconciledMessages,
+        IMap<String, OnlineElementState<proto.Message>>.fromValues(
+      keyMapper: (x) => x.value.authorUniqueIdString,
+      values: sentMessages.window,
     );
+    // final unsentMessagesMap = IMap<String, proto.Message>.fromValues(
+    //   keyMapper: (x) => x.authorUniqueIdString,
+    //   values: unsentMessages,
+    // );
 
-    final renderedElements = <Int64, RenderStateElement>{};
+    final renderedElements = <RenderStateElement>[];
 
-    for (final m in reconciledMessagesMap.entries) {
-      renderedElements[m.key] = RenderStateElement(
-          message: m.value.value,
-          isLocal: m.value.value.author.toVeilid() != _remoteIdentityPublicKey,
-          reconciled: true,
-          reconciledOffline: m.value.isOffline);
-    }
-    for (final m in sentMessagesMap.entries) {
-      renderedElements.putIfAbsent(
-          m.key,
-          () => RenderStateElement(
-                message: m.value.value,
-                isLocal: true,
-              ))
-        ..sent = true
-        ..sentOffline = m.value.isOffline;
-    }
-    for (final m in unreconciledMessagesMap.entries) {
-      renderedElements
-          .putIfAbsent(
-              m.key,
-              () => RenderStateElement(
-                    message: m.value,
-                    isLocal:
-                        m.value.author.toVeilid() != _remoteIdentityPublicKey,
-                  ))
-          .reconciled = false;
-    }
-    for (final m in sendingMessagesMap.entries) {
-      renderedElements
-          .putIfAbsent(
-              m.key,
-              () => RenderStateElement(
-                    message: m.value,
-                    isLocal: true,
-                  ))
-          .sent = false;
+    for (final m in reconciledMessages.windowElements) {
+      final isLocal = m.content.author.toVeilid() ==
+          _activeAccountInfo.localAccount.identityMaster
+              .identityPublicTypedKey();
+      final reconciledTimestamp = Timestamp.fromInt64(m.reconciledTime);
+      final sm =
+          isLocal ? sentMessagesMap[m.content.authorUniqueIdString] : null;
+      final sent = isLocal && sm != null;
+      final sentOffline = isLocal && sm != null && sm.isOffline;
+
+      renderedElements.add(RenderStateElement(
+        message: m.content,
+        isLocal: isLocal,
+        reconciledTimestamp: reconciledTimestamp,
+        sent: sent,
+        sentOffline: sentOffline,
+      ));
     }
 
     // Render the state
-    final messageKeys = renderedElements.entries
-        .toIList()
-        .sort((x, y) => x.key.compareTo(y.key));
-    final renderedState = messageKeys
+    final messages = renderedElements
         .map((x) => MessageState(
-            author: x.value.message.author.toVeilid(),
-            timestamp: Timestamp.fromInt64(x.key),
-            text: x.value.message.text,
-            sendState: x.value.sendState))
+            content: x.message,
+            sentTimestamp: Timestamp.fromInt64(x.message.timestamp),
+            reconciledTimestamp: x.reconciledTimestamp,
+            sendState: x.sendState))
         .toIList();
 
     // Emit the rendered state
-
-    emit(AsyncValue.data(renderedState));
+    emit(AsyncValue.data(WindowState<MessageState>(
+        window: messages,
+        length: reconciledMessages.length,
+        windowTail: reconciledMessages.windowTail,
+        windowCount: reconciledMessages.windowCount,
+        follow: reconciledMessages.follow)));
   }
 
-  void addMessage({required proto.Message message}) {
-    _unreconciledMessagesQueue.addSync(message);
-    _sendingMessagesQueue.addSync(message);
+  void _sendMessage({required proto.Message message}) {
+    // Add common fields
+    // id and signature will get set by _processMessageToSend
+    message
+      ..author = _activeAccountInfo.localAccount.identityMaster
+          .identityPublicTypedKey()
+          .toProto()
+      ..timestamp = Veilid.instance.now().toInt64();
+
+    // Put in the queue
+    _unsentMessagesQueue.addSync(message);
 
     // Update the view
     _renderState();
   }
 
+  Future<void> _commandRunner() async {
+    await for (final command in _commandController.stream) {
+      await command();
+    }
+  }
+
+  /////////////////////////////////////////////////////////////////////////
+  // Static utility functions
+
+  static Future<void> cleanupAndDeleteMessages(
+      {required TypedKey localConversationRecordKey}) async {
+    final recmsgdbname =
+        _reconciledMessagesTableDBName(localConversationRecordKey);
+    await Veilid.instance.deleteTableDB(recmsgdbname);
+  }
+
+  static String _reconciledMessagesTableDBName(
+          TypedKey localConversationRecordKey) =>
+      'msg_${localConversationRecordKey.toString().replaceAll(':', '_')}';
+
+  /////////////////////////////////////////////////////////////////////////
+
   final WaitSet<void> _initWait = WaitSet();
   final ActiveAccountInfo _activeAccountInfo;
   final TypedKey _remoteIdentityPublicKey;
@@ -398,19 +414,22 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   final TypedKey _localMessagesRecordKey;
   final TypedKey _remoteConversationRecordKey;
   final TypedKey _remoteMessagesRecordKey;
-  final OwnedDHTRecordPointer _reconciledChatRecord;
 
-  late final DHTRecordCrypto _messagesCrypto;
+  late final VeilidCrypto _conversationCrypto;
+  late final MessageIntegrity _senderMessageIntegrity;
 
-  DHTShortArrayCubit<proto.Message>? _sentMessagesCubit;
-  DHTShortArrayCubit<proto.Message>? _rcvdMessagesCubit;
-  DHTShortArrayCubit<proto.Message>? _reconciledMessagesCubit;
+  DHTLogCubit<proto.Message>? _sentMessagesCubit;
+  DHTLogCubit<proto.Message>? _rcvdMessagesCubit;
+  TableDBArrayProtobufCubit<proto.ReconciledMessage>? _reconciledMessagesCubit;
 
-  late final PersistentQueue<proto.Message> _unreconciledMessagesQueue;
-  late final PersistentQueue<proto.Message> _sendingMessagesQueue;
+  late final MessageReconciliation _reconciliation;
 
-  StreamSubscription<DHTShortArrayBusyState<proto.Message>>? _sentSubscription;
-  StreamSubscription<DHTShortArrayBusyState<proto.Message>>? _rcvdSubscription;
-  StreamSubscription<DHTShortArrayBusyState<proto.Message>>?
+  late final PersistentQueue<proto.Message> _unsentMessagesQueue;
+
+  StreamSubscription<DHTLogBusyState<proto.Message>>? _sentSubscription;
+  StreamSubscription<DHTLogBusyState<proto.Message>>? _rcvdSubscription;
+  StreamSubscription<TableDBArrayProtobufBusyState<proto.ReconciledMessage>>?
       _reconciledSubscription;
+  final StreamController<Future<void> Function()> _commandController;
+  late final Future<void> _commandRunnerFut;
 }

lib/chat/models/chat_component_state.dart

@@ -0,0 +1,34 @@
+import 'package:async_tools/async_tools.dart';
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_chat_types/flutter_chat_types.dart' show Message, User;
+import 'package:flutter_chat_ui/flutter_chat_ui.dart' show ChatState;
+import 'package:freezed_annotation/freezed_annotation.dart';
+import 'package:scroll_to_index/scroll_to_index.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+import 'window_state.dart';
+
+part 'chat_component_state.freezed.dart';
+
+@freezed
+class ChatComponentState with _$ChatComponentState {
+  const factory ChatComponentState(
+      {
+      // GlobalKey for the chat
+      required GlobalKey<ChatState> chatKey,
+      // ScrollController for the chat
+      required AutoScrollController scrollController,
+      // Local user
+      required User localUser,
+      // Remote users
+      required IMap<TypedKey, User> remoteUsers,
+      // Messages state
+      required AsyncValue<WindowState<Message>> messageWindow,
+      // Title of the chat
+      required String title}) = _ChatComponentState;
+}
+
+extension ChatComponentStateExt on ChatComponentState {
+  //
+}

lib/chat/models/chat_component_state.freezed.dart

@@ -0,0 +1,267 @@
+// coverage:ignore-file
+// GENERATED CODE - DO NOT MODIFY BY HAND
+// ignore_for_file: type=lint
+// ignore_for_file: unused_element, deprecated_member_use, deprecated_member_use_from_same_package, use_function_type_syntax_for_parameters, unnecessary_const, avoid_init_to_null, invalid_override_different_default_values_named, prefer_expression_function_bodies, annotate_overrides, invalid_annotation_target, unnecessary_question_mark
+
+part of 'chat_component_state.dart';
+
+// **************************************************************************
+// FreezedGenerator
+// **************************************************************************
+
+T _$identity<T>(T value) => value;
+
+final _privateConstructorUsedError = UnsupportedError(
+    'It seems like you constructed your class using `MyClass._()`. This constructor is only meant to be used by freezed and you are not supposed to need it nor use it.\nPlease check the documentation here for more information: https://github.com/rrousselGit/freezed#adding-getters-and-methods-to-our-models');
+
+/// @nodoc
+mixin _$ChatComponentState {
+// GlobalKey for the chat
+  GlobalKey<ChatState> get chatKey =>
+      throw _privateConstructorUsedError; // ScrollController for the chat
+  AutoScrollController get scrollController =>
+      throw _privateConstructorUsedError; // Local user
+  User get localUser => throw _privateConstructorUsedError; // Remote users
+  IMap<Typed<FixedEncodedString43>, User> get remoteUsers =>
+      throw _privateConstructorUsedError; // Messages state
+  AsyncValue<WindowState<Message>> get messageWindow =>
+      throw _privateConstructorUsedError; // Title of the chat
+  String get title => throw _privateConstructorUsedError;
+
+  @JsonKey(ignore: true)
+  $ChatComponentStateCopyWith<ChatComponentState> get copyWith =>
+      throw _privateConstructorUsedError;
+}
+
+/// @nodoc
+abstract class $ChatComponentStateCopyWith<$Res> {
+  factory $ChatComponentStateCopyWith(
+          ChatComponentState value, $Res Function(ChatComponentState) then) =
+      _$ChatComponentStateCopyWithImpl<$Res, ChatComponentState>;
+  @useResult
+  $Res call(
+      {GlobalKey<ChatState> chatKey,
+      AutoScrollController scrollController,
+      User localUser,
+      IMap<Typed<FixedEncodedString43>, User> remoteUsers,
+      AsyncValue<WindowState<Message>> messageWindow,
+      String title});
+
+  $AsyncValueCopyWith<WindowState<Message>, $Res> get messageWindow;
+}
+
+/// @nodoc
+class _$ChatComponentStateCopyWithImpl<$Res, $Val extends ChatComponentState>
+    implements $ChatComponentStateCopyWith<$Res> {
+  _$ChatComponentStateCopyWithImpl(this._value, this._then);
+
+  // ignore: unused_field
+  final $Val _value;
+  // ignore: unused_field
+  final $Res Function($Val) _then;
+
+  @pragma('vm:prefer-inline')
+  @override
+  $Res call({
+    Object? chatKey = null,
+    Object? scrollController = null,
+    Object? localUser = null,
+    Object? remoteUsers = null,
+    Object? messageWindow = null,
+    Object? title = null,
+  }) {
+    return _then(_value.copyWith(
+      chatKey: null == chatKey
+          ? _value.chatKey
+          : chatKey // ignore: cast_nullable_to_non_nullable
+              as GlobalKey<ChatState>,
+      scrollController: null == scrollController
+          ? _value.scrollController
+          : scrollController // ignore: cast_nullable_to_non_nullable
+              as AutoScrollController,
+      localUser: null == localUser
+          ? _value.localUser
+          : localUser // ignore: cast_nullable_to_non_nullable
+              as User,
+      remoteUsers: null == remoteUsers
+          ? _value.remoteUsers
+          : remoteUsers // ignore: cast_nullable_to_non_nullable
+              as IMap<Typed<FixedEncodedString43>, User>,
+      messageWindow: null == messageWindow
+          ? _value.messageWindow
+          : messageWindow // ignore: cast_nullable_to_non_nullable
+              as AsyncValue<WindowState<Message>>,
+      title: null == title
+          ? _value.title
+          : title // ignore: cast_nullable_to_non_nullable
+              as String,
+    ) as $Val);
+  }
+
+  @override
+  @pragma('vm:prefer-inline')
+  $AsyncValueCopyWith<WindowState<Message>, $Res> get messageWindow {
+    return $AsyncValueCopyWith<WindowState<Message>, $Res>(_value.messageWindow,
+        (value) {
+      return _then(_value.copyWith(messageWindow: value) as $Val);
+    });
+  }
+}
+
+/// @nodoc
+abstract class _$$ChatComponentStateImplCopyWith<$Res>
+    implements $ChatComponentStateCopyWith<$Res> {
+  factory _$$ChatComponentStateImplCopyWith(_$ChatComponentStateImpl value,
+          $Res Function(_$ChatComponentStateImpl) then) =
+      __$$ChatComponentStateImplCopyWithImpl<$Res>;
+  @override
+  @useResult
+  $Res call(
+      {GlobalKey<ChatState> chatKey,
+      AutoScrollController scrollController,
+      User localUser,
+      IMap<Typed<FixedEncodedString43>, User> remoteUsers,
+      AsyncValue<WindowState<Message>> messageWindow,
+      String title});
+
+  @override
+  $AsyncValueCopyWith<WindowState<Message>, $Res> get messageWindow;
+}
+
+/// @nodoc
+class __$$ChatComponentStateImplCopyWithImpl<$Res>
+    extends _$ChatComponentStateCopyWithImpl<$Res, _$ChatComponentStateImpl>
+    implements _$$ChatComponentStateImplCopyWith<$Res> {
+  __$$ChatComponentStateImplCopyWithImpl(_$ChatComponentStateImpl _value,
+      $Res Function(_$ChatComponentStateImpl) _then)
+      : super(_value, _then);
+
+  @pragma('vm:prefer-inline')
+  @override
+  $Res call({
+    Object? chatKey = null,
+    Object? scrollController = null,
+    Object? localUser = null,
+    Object? remoteUsers = null,
+    Object? messageWindow = null,
+    Object? title = null,
+  }) {
+    return _then(_$ChatComponentStateImpl(
+      chatKey: null == chatKey
+          ? _value.chatKey
+          : chatKey // ignore: cast_nullable_to_non_nullable
+              as GlobalKey<ChatState>,
+      scrollController: null == scrollController
+          ? _value.scrollController
+          : scrollController // ignore: cast_nullable_to_non_nullable
+              as AutoScrollController,
+      localUser: null == localUser
+          ? _value.localUser
+          : localUser // ignore: cast_nullable_to_non_nullable
+              as User,
+      remoteUsers: null == remoteUsers
+          ? _value.remoteUsers
+          : remoteUsers // ignore: cast_nullable_to_non_nullable
+              as IMap<Typed<FixedEncodedString43>, User>,
+      messageWindow: null == messageWindow
+          ? _value.messageWindow
+          : messageWindow // ignore: cast_nullable_to_non_nullable
+              as AsyncValue<WindowState<Message>>,
+      title: null == title
+          ? _value.title
+          : title // ignore: cast_nullable_to_non_nullable
+              as String,
+    ));
+  }
+}
+
+/// @nodoc
+
+class _$ChatComponentStateImpl implements _ChatComponentState {
+  const _$ChatComponentStateImpl(
+      {required this.chatKey,
+      required this.scrollController,
+      required this.localUser,
+      required this.remoteUsers,
+      required this.messageWindow,
+      required this.title});
+
+// GlobalKey for the chat
+  @override
+  final GlobalKey<ChatState> chatKey;
+// ScrollController for the chat
+  @override
+  final AutoScrollController scrollController;
+// Local user
+  @override
+  final User localUser;
+// Remote users
+  @override
+  final IMap<Typed<FixedEncodedString43>, User> remoteUsers;
+// Messages state
+  @override
+  final AsyncValue<WindowState<Message>> messageWindow;
+// Title of the chat
+  @override
+  final String title;
+
+  @override
+  String toString() {
+    return 'ChatComponentState(chatKey: $chatKey, scrollController: $scrollController, localUser: $localUser, remoteUsers: $remoteUsers, messageWindow: $messageWindow, title: $title)';
+  }
+
+  @override
+  bool operator ==(Object other) {
+    return identical(this, other) ||
+        (other.runtimeType == runtimeType &&
+            other is _$ChatComponentStateImpl &&
+            (identical(other.chatKey, chatKey) || other.chatKey == chatKey) &&
+            (identical(other.scrollController, scrollController) ||
+                other.scrollController == scrollController) &&
+            (identical(other.localUser, localUser) ||
+                other.localUser == localUser) &&
+            (identical(other.remoteUsers, remoteUsers) ||
+                other.remoteUsers == remoteUsers) &&
+            (identical(other.messageWindow, messageWindow) ||
+                other.messageWindow == messageWindow) &&
+            (identical(other.title, title) || other.title == title));
+  }
+
+  @override
+  int get hashCode => Object.hash(runtimeType, chatKey, scrollController,
+      localUser, remoteUsers, messageWindow, title);
+
+  @JsonKey(ignore: true)
+  @override
+  @pragma('vm:prefer-inline')
+  _$$ChatComponentStateImplCopyWith<_$ChatComponentStateImpl> get copyWith =>
+      __$$ChatComponentStateImplCopyWithImpl<_$ChatComponentStateImpl>(
+          this, _$identity);
+}
+
+abstract class _ChatComponentState implements ChatComponentState {
+  const factory _ChatComponentState(
+      {required final GlobalKey<ChatState> chatKey,
+      required final AutoScrollController scrollController,
+      required final User localUser,
+      required final IMap<Typed<FixedEncodedString43>, User> remoteUsers,
+      required final AsyncValue<WindowState<Message>> messageWindow,
+      required final String title}) = _$ChatComponentStateImpl;
+
+  @override // GlobalKey for the chat
+  GlobalKey<ChatState> get chatKey;
+  @override // ScrollController for the chat
+  AutoScrollController get scrollController;
+  @override // Local user
+  User get localUser;
+  @override // Remote users
+  IMap<Typed<FixedEncodedString43>, User> get remoteUsers;
+  @override // Messages state
+  AsyncValue<WindowState<Message>> get messageWindow;
+  @override // Title of the chat
+  String get title;
+  @override
+  @JsonKey(ignore: true)
+  _$$ChatComponentStateImplCopyWith<_$ChatComponentStateImpl> get copyWith =>
+      throw _privateConstructorUsedError;
+}

lib/chat/models/message_state.dart

@@ -3,6 +3,9 @@ import 'package:flutter/foundation.dart';
 import 'package:freezed_annotation/freezed_annotation.dart';
 import 'package:veilid_support/veilid_support.dart';
 
+import '../../proto/proto.dart' as proto;
+import '../../proto/proto.dart' show messageFromJson, messageToJson;
+
 part 'message_state.freezed.dart';
 part 'message_state.g.dart';
 
@@ -23,9 +26,14 @@ enum MessageSendState {
 @freezed
 class MessageState with _$MessageState {
   const factory MessageState({
-    required TypedKey author,
-    required Timestamp timestamp,
-    required String text,
+    // Content of the message
+    @JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+    required proto.Message content,
+    // Sent timestamp
+    required Timestamp sentTimestamp,
+    // Reconciled timestamp
+    required Timestamp? reconciledTimestamp,
+    // The state of the message
     required MessageSendState? sendState,
   }) = _MessageState;
 

lib/chat/models/message_state.freezed.dart

@@ -20,9 +20,14 @@ MessageState _$MessageStateFromJson(Map<String, dynamic> json) {
 
 /// @nodoc
 mixin _$MessageState {
-  Typed<FixedEncodedString43> get author => throw _privateConstructorUsedError;
-  Timestamp get timestamp => throw _privateConstructorUsedError;
-  String get text => throw _privateConstructorUsedError;
+// Content of the message
+  @JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+  proto.Message get content =>
+      throw _privateConstructorUsedError; // Sent timestamp
+  Timestamp get sentTimestamp =>
+      throw _privateConstructorUsedError; // Reconciled timestamp
+  Timestamp? get reconciledTimestamp =>
+      throw _privateConstructorUsedError; // The state of the message
   MessageSendState? get sendState => throw _privateConstructorUsedError;
 
   Map<String, dynamic> toJson() => throw _privateConstructorUsedError;
@@ -38,9 +43,10 @@ abstract class $MessageStateCopyWith<$Res> {
       _$MessageStateCopyWithImpl<$Res, MessageState>;
   @useResult
   $Res call(
-      {Typed<FixedEncodedString43> author,
-      Timestamp timestamp,
-      String text,
+      {@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+      proto.Message content,
+      Timestamp sentTimestamp,
+      Timestamp? reconciledTimestamp,
       MessageSendState? sendState});
 }
 
@@ -57,24 +63,24 @@ class _$MessageStateCopyWithImpl<$Res, $Val extends MessageState>
   @pragma('vm:prefer-inline')
   @override
   $Res call({
-    Object? author = null,
-    Object? timestamp = null,
-    Object? text = null,
+    Object? content = null,
+    Object? sentTimestamp = null,
+    Object? reconciledTimestamp = freezed,
     Object? sendState = freezed,
   }) {
     return _then(_value.copyWith(
-      author: null == author
-          ? _value.author
-          : author // ignore: cast_nullable_to_non_nullable
-              as Typed<FixedEncodedString43>,
-      timestamp: null == timestamp
-          ? _value.timestamp
-          : timestamp // ignore: cast_nullable_to_non_nullable
+      content: null == content
+          ? _value.content
+          : content // ignore: cast_nullable_to_non_nullable
+              as proto.Message,
+      sentTimestamp: null == sentTimestamp
+          ? _value.sentTimestamp
+          : sentTimestamp // ignore: cast_nullable_to_non_nullable
               as Timestamp,
-      text: null == text
-          ? _value.text
-          : text // ignore: cast_nullable_to_non_nullable
-              as String,
+      reconciledTimestamp: freezed == reconciledTimestamp
+          ? _value.reconciledTimestamp
+          : reconciledTimestamp // ignore: cast_nullable_to_non_nullable
+              as Timestamp?,
       sendState: freezed == sendState
           ? _value.sendState
           : sendState // ignore: cast_nullable_to_non_nullable
@@ -92,9 +98,10 @@ abstract class _$$MessageStateImplCopyWith<$Res>
   @override
   @useResult
   $Res call(
-      {Typed<FixedEncodedString43> author,
-      Timestamp timestamp,
-      String text,
+      {@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+      proto.Message content,
+      Timestamp sentTimestamp,
+      Timestamp? reconciledTimestamp,
       MessageSendState? sendState});
 }
 
@@ -109,24 +116,24 @@ class __$$MessageStateImplCopyWithImpl<$Res>
   @pragma('vm:prefer-inline')
   @override
   $Res call({
-    Object? author = null,
-    Object? timestamp = null,
-    Object? text = null,
+    Object? content = null,
+    Object? sentTimestamp = null,
+    Object? reconciledTimestamp = freezed,
     Object? sendState = freezed,
   }) {
     return _then(_$MessageStateImpl(
-      author: null == author
-          ? _value.author
-          : author // ignore: cast_nullable_to_non_nullable
-              as Typed<FixedEncodedString43>,
-      timestamp: null == timestamp
-          ? _value.timestamp
-          : timestamp // ignore: cast_nullable_to_non_nullable
+      content: null == content
+          ? _value.content
+          : content // ignore: cast_nullable_to_non_nullable
+              as proto.Message,
+      sentTimestamp: null == sentTimestamp
+          ? _value.sentTimestamp
+          : sentTimestamp // ignore: cast_nullable_to_non_nullable
               as Timestamp,
-      text: null == text
-          ? _value.text
-          : text // ignore: cast_nullable_to_non_nullable
-              as String,
+      reconciledTimestamp: freezed == reconciledTimestamp
+          ? _value.reconciledTimestamp
+          : reconciledTimestamp // ignore: cast_nullable_to_non_nullable
+              as Timestamp?,
       sendState: freezed == sendState
           ? _value.sendState
           : sendState // ignore: cast_nullable_to_non_nullable
@@ -139,26 +146,32 @@ class __$$MessageStateImplCopyWithImpl<$Res>
 @JsonSerializable()
 class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
   const _$MessageStateImpl(
-      {required this.author,
-      required this.timestamp,
-      required this.text,
+      {@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+      required this.content,
+      required this.sentTimestamp,
+      required this.reconciledTimestamp,
       required this.sendState});
 
   factory _$MessageStateImpl.fromJson(Map<String, dynamic> json) =>
       _$$MessageStateImplFromJson(json);
 
+// Content of the message
   @override
-  final Typed<FixedEncodedString43> author;
+  @JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+  final proto.Message content;
+// Sent timestamp
   @override
-  final Timestamp timestamp;
+  final Timestamp sentTimestamp;
+// Reconciled timestamp
   @override
-  final String text;
+  final Timestamp? reconciledTimestamp;
+// The state of the message
   @override
   final MessageSendState? sendState;
 
   @override
   String toString({DiagnosticLevel minLevel = DiagnosticLevel.info}) {
-    return 'MessageState(author: $author, timestamp: $timestamp, text: $text, sendState: $sendState)';
+    return 'MessageState(content: $content, sentTimestamp: $sentTimestamp, reconciledTimestamp: $reconciledTimestamp, sendState: $sendState)';
   }
 
   @override
@@ -166,9 +179,9 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
     super.debugFillProperties(properties);
     properties
       ..add(DiagnosticsProperty('type', 'MessageState'))
-      ..add(DiagnosticsProperty('author', author))
-      ..add(DiagnosticsProperty('timestamp', timestamp))
-      ..add(DiagnosticsProperty('text', text))
+      ..add(DiagnosticsProperty('content', content))
+      ..add(DiagnosticsProperty('sentTimestamp', sentTimestamp))
+      ..add(DiagnosticsProperty('reconciledTimestamp', reconciledTimestamp))
       ..add(DiagnosticsProperty('sendState', sendState));
   }
 
@@ -177,18 +190,19 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
     return identical(this, other) ||
         (other.runtimeType == runtimeType &&
             other is _$MessageStateImpl &&
-            (identical(other.author, author) || other.author == author) &&
-            (identical(other.timestamp, timestamp) ||
-                other.timestamp == timestamp) &&
-            (identical(other.text, text) || other.text == text) &&
+            (identical(other.content, content) || other.content == content) &&
+            (identical(other.sentTimestamp, sentTimestamp) ||
+                other.sentTimestamp == sentTimestamp) &&
+            (identical(other.reconciledTimestamp, reconciledTimestamp) ||
+                other.reconciledTimestamp == reconciledTimestamp) &&
             (identical(other.sendState, sendState) ||
                 other.sendState == sendState));
   }
 
   @JsonKey(ignore: true)
   @override
-  int get hashCode =>
-      Object.hash(runtimeType, author, timestamp, text, sendState);
+  int get hashCode => Object.hash(
+      runtimeType, content, sentTimestamp, reconciledTimestamp, sendState);
 
   @JsonKey(ignore: true)
   @override
@@ -206,21 +220,23 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
 
 abstract class _MessageState implements MessageState {
   const factory _MessageState(
-      {required final Typed<FixedEncodedString43> author,
-      required final Timestamp timestamp,
-      required final String text,
+      {@JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+      required final proto.Message content,
+      required final Timestamp sentTimestamp,
+      required final Timestamp? reconciledTimestamp,
       required final MessageSendState? sendState}) = _$MessageStateImpl;
 
   factory _MessageState.fromJson(Map<String, dynamic> json) =
       _$MessageStateImpl.fromJson;
 
-  @override
-  Typed<FixedEncodedString43> get author;
-  @override
-  Timestamp get timestamp;
-  @override
-  String get text;
-  @override
+  @override // Content of the message
+  @JsonKey(fromJson: messageFromJson, toJson: messageToJson)
+  proto.Message get content;
+  @override // Sent timestamp
+  Timestamp get sentTimestamp;
+  @override // Reconciled timestamp
+  Timestamp? get reconciledTimestamp;
+  @override // The state of the message
   MessageSendState? get sendState;
   @override
   @JsonKey(ignore: true)

lib/chat/models/message_state.g.dart

@@ -8,9 +8,11 @@ part of 'message_state.dart';
 
 _$MessageStateImpl _$$MessageStateImplFromJson(Map<String, dynamic> json) =>
     _$MessageStateImpl(
-      author: Typed<FixedEncodedString43>.fromJson(json['author']),
-      timestamp: Timestamp.fromJson(json['timestamp']),
-      text: json['text'] as String,
+      content: messageFromJson(json['content'] as Map<String, dynamic>),
+      sentTimestamp: Timestamp.fromJson(json['sent_timestamp']),
+      reconciledTimestamp: json['reconciled_timestamp'] == null
+          ? null
+          : Timestamp.fromJson(json['reconciled_timestamp']),
       sendState: json['send_state'] == null
           ? null
           : MessageSendState.fromJson(json['send_state']),
@@ -18,8 +20,8 @@ _$MessageStateImpl _$$MessageStateImplFromJson(Map<String, dynamic> json) =>
 
 Map<String, dynamic> _$$MessageStateImplToJson(_$MessageStateImpl instance) =>
     <String, dynamic>{
-      'author': instance.author.toJson(),
-      'timestamp': instance.timestamp.toJson(),
-      'text': instance.text,
+      'content': messageToJson(instance.content),
+      'sent_timestamp': instance.sentTimestamp.toJson(),
+      'reconciled_timestamp': instance.reconciledTimestamp?.toJson(),
       'send_state': instance.sendState?.toJson(),
     };

lib/chat/models/models.dart

@@ -1 +1,3 @@
+export 'chat_component_state.dart';
 export 'message_state.dart';
+export 'window_state.dart';

lib/chat/models/window_state.dart

@@ -0,0 +1,27 @@
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:flutter/foundation.dart';
+import 'package:freezed_annotation/freezed_annotation.dart';
+
+part 'window_state.freezed.dart';
+
+@freezed
+class WindowState<T> with _$WindowState<T> {
+  const factory WindowState({
+    // List of objects in the window
+    required IList<T> window,
+    // Total number of objects (windowTail max)
+    required int length,
+    // One past the end of the last element
+    required int windowTail,
+    // The total number of elements to try to keep in the window
+    required int windowCount,
+    // If we should have the tail following the array
+    required bool follow,
+  }) = _WindowState;
+}
+
+extension WindowStateExt<T> on WindowState<T> {
+  int get windowEnd => (length == 0) ? -1 : (windowTail - 1) % length;
+  int get windowStart =>
+      (length == 0) ? 0 : (windowTail - window.length) % length;
+}

lib/chat/models/window_state.freezed.dart

@@ -0,0 +1,249 @@
+// coverage:ignore-file
+// GENERATED CODE - DO NOT MODIFY BY HAND
+// ignore_for_file: type=lint
+// ignore_for_file: unused_element, deprecated_member_use, deprecated_member_use_from_same_package, use_function_type_syntax_for_parameters, unnecessary_const, avoid_init_to_null, invalid_override_different_default_values_named, prefer_expression_function_bodies, annotate_overrides, invalid_annotation_target, unnecessary_question_mark
+
+part of 'window_state.dart';
+
+// **************************************************************************
+// FreezedGenerator
+// **************************************************************************
+
+T _$identity<T>(T value) => value;
+
+final _privateConstructorUsedError = UnsupportedError(
+    'It seems like you constructed your class using `MyClass._()`. This constructor is only meant to be used by freezed and you are not supposed to need it nor use it.\nPlease check the documentation here for more information: https://github.com/rrousselGit/freezed#adding-getters-and-methods-to-our-models');
+
+/// @nodoc
+mixin _$WindowState<T> {
+// List of objects in the window
+  IList<T> get window =>
+      throw _privateConstructorUsedError; // Total number of objects (windowTail max)
+  int get length =>
+      throw _privateConstructorUsedError; // One past the end of the last element
+  int get windowTail =>
+      throw _privateConstructorUsedError; // The total number of elements to try to keep in the window
+  int get windowCount =>
+      throw _privateConstructorUsedError; // If we should have the tail following the array
+  bool get follow => throw _privateConstructorUsedError;
+
+  @JsonKey(ignore: true)
+  $WindowStateCopyWith<T, WindowState<T>> get copyWith =>
+      throw _privateConstructorUsedError;
+}
+
+/// @nodoc
+abstract class $WindowStateCopyWith<T, $Res> {
+  factory $WindowStateCopyWith(
+          WindowState<T> value, $Res Function(WindowState<T>) then) =
+      _$WindowStateCopyWithImpl<T, $Res, WindowState<T>>;
+  @useResult
+  $Res call(
+      {IList<T> window,
+      int length,
+      int windowTail,
+      int windowCount,
+      bool follow});
+}
+
+/// @nodoc
+class _$WindowStateCopyWithImpl<T, $Res, $Val extends WindowState<T>>
+    implements $WindowStateCopyWith<T, $Res> {
+  _$WindowStateCopyWithImpl(this._value, this._then);
+
+  // ignore: unused_field
+  final $Val _value;
+  // ignore: unused_field
+  final $Res Function($Val) _then;
+
+  @pragma('vm:prefer-inline')
+  @override
+  $Res call({
+    Object? window = null,
+    Object? length = null,
+    Object? windowTail = null,
+    Object? windowCount = null,
+    Object? follow = null,
+  }) {
+    return _then(_value.copyWith(
+      window: null == window
+          ? _value.window
+          : window // ignore: cast_nullable_to_non_nullable
+              as IList<T>,
+      length: null == length
+          ? _value.length
+          : length // ignore: cast_nullable_to_non_nullable
+              as int,
+      windowTail: null == windowTail
+          ? _value.windowTail
+          : windowTail // ignore: cast_nullable_to_non_nullable
+              as int,
+      windowCount: null == windowCount
+          ? _value.windowCount
+          : windowCount // ignore: cast_nullable_to_non_nullable
+              as int,
+      follow: null == follow
+          ? _value.follow
+          : follow // ignore: cast_nullable_to_non_nullable
+              as bool,
+    ) as $Val);
+  }
+}
+
+/// @nodoc
+abstract class _$$WindowStateImplCopyWith<T, $Res>
+    implements $WindowStateCopyWith<T, $Res> {
+  factory _$$WindowStateImplCopyWith(_$WindowStateImpl<T> value,
+          $Res Function(_$WindowStateImpl<T>) then) =
+      __$$WindowStateImplCopyWithImpl<T, $Res>;
+  @override
+  @useResult
+  $Res call(
+      {IList<T> window,
+      int length,
+      int windowTail,
+      int windowCount,
+      bool follow});
+}
+
+/// @nodoc
+class __$$WindowStateImplCopyWithImpl<T, $Res>
+    extends _$WindowStateCopyWithImpl<T, $Res, _$WindowStateImpl<T>>
+    implements _$$WindowStateImplCopyWith<T, $Res> {
+  __$$WindowStateImplCopyWithImpl(
+      _$WindowStateImpl<T> _value, $Res Function(_$WindowStateImpl<T>) _then)
+      : super(_value, _then);
+
+  @pragma('vm:prefer-inline')
+  @override
+  $Res call({
+    Object? window = null,
+    Object? length = null,
+    Object? windowTail = null,
+    Object? windowCount = null,
+    Object? follow = null,
+  }) {
+    return _then(_$WindowStateImpl<T>(
+      window: null == window
+          ? _value.window
+          : window // ignore: cast_nullable_to_non_nullable
+              as IList<T>,
+      length: null == length
+          ? _value.length
+          : length // ignore: cast_nullable_to_non_nullable
+              as int,
+      windowTail: null == windowTail
+          ? _value.windowTail
+          : windowTail // ignore: cast_nullable_to_non_nullable
+              as int,
+      windowCount: null == windowCount
+          ? _value.windowCount
+          : windowCount // ignore: cast_nullable_to_non_nullable
+              as int,
+      follow: null == follow
+          ? _value.follow
+          : follow // ignore: cast_nullable_to_non_nullable
+              as bool,
+    ));
+  }
+}
+
+/// @nodoc
+
+class _$WindowStateImpl<T>
+    with DiagnosticableTreeMixin
+    implements _WindowState<T> {
+  const _$WindowStateImpl(
+      {required this.window,
+      required this.length,
+      required this.windowTail,
+      required this.windowCount,
+      required this.follow});
+
+// List of objects in the window
+  @override
+  final IList<T> window;
+// Total number of objects (windowTail max)
+  @override
+  final int length;
+// One past the end of the last element
+  @override
+  final int windowTail;
+// The total number of elements to try to keep in the window
+  @override
+  final int windowCount;
+// If we should have the tail following the array
+  @override
+  final bool follow;
+
+  @override
+  String toString({DiagnosticLevel minLevel = DiagnosticLevel.info}) {
+    return 'WindowState<$T>(window: $window, length: $length, windowTail: $windowTail, windowCount: $windowCount, follow: $follow)';
+  }
+
+  @override
+  void debugFillProperties(DiagnosticPropertiesBuilder properties) {
+    super.debugFillProperties(properties);
+    properties
+      ..add(DiagnosticsProperty('type', 'WindowState<$T>'))
+      ..add(DiagnosticsProperty('window', window))
+      ..add(DiagnosticsProperty('length', length))
+      ..add(DiagnosticsProperty('windowTail', windowTail))
+      ..add(DiagnosticsProperty('windowCount', windowCount))
+      ..add(DiagnosticsProperty('follow', follow));
+  }
+
+  @override
+  bool operator ==(Object other) {
+    return identical(this, other) ||
+        (other.runtimeType == runtimeType &&
+            other is _$WindowStateImpl<T> &&
+            const DeepCollectionEquality().equals(other.window, window) &&
+            (identical(other.length, length) || other.length == length) &&
+            (identical(other.windowTail, windowTail) ||
+                other.windowTail == windowTail) &&
+            (identical(other.windowCount, windowCount) ||
+                other.windowCount == windowCount) &&
+            (identical(other.follow, follow) || other.follow == follow));
+  }
+
+  @override
+  int get hashCode => Object.hash(
+      runtimeType,
+      const DeepCollectionEquality().hash(window),
+      length,
+      windowTail,
+      windowCount,
+      follow);
+
+  @JsonKey(ignore: true)
+  @override
+  @pragma('vm:prefer-inline')
+  _$$WindowStateImplCopyWith<T, _$WindowStateImpl<T>> get copyWith =>
+      __$$WindowStateImplCopyWithImpl<T, _$WindowStateImpl<T>>(
+          this, _$identity);
+}
+
+abstract class _WindowState<T> implements WindowState<T> {
+  const factory _WindowState(
+      {required final IList<T> window,
+      required final int length,
+      required final int windowTail,
+      required final int windowCount,
+      required final bool follow}) = _$WindowStateImpl<T>;
+
+  @override // List of objects in the window
+  IList<T> get window;
+  @override // Total number of objects (windowTail max)
+  int get length;
+  @override // One past the end of the last element
+  int get windowTail;
+  @override // The total number of elements to try to keep in the window
+  int get windowCount;
+  @override // If we should have the tail following the array
+  bool get follow;
+  @override
+  @JsonKey(ignore: true)
+  _$$WindowStateImplCopyWith<T, _$WindowStateImpl<T>> get copyWith =>
+      throw _privateConstructorUsedError;
+}

lib/chat/views/chat_component.dart

@@ -1,231 +0,0 @@
-import 'package:async_tools/async_tools.dart';
-import 'package:awesome_extensions/awesome_extensions.dart';
-import 'package:flutter/material.dart';
-import 'package:flutter_bloc/flutter_bloc.dart';
-import 'package:flutter_chat_types/flutter_chat_types.dart' as types;
-import 'package:flutter_chat_ui/flutter_chat_ui.dart';
-import 'package:veilid_support/veilid_support.dart';
-
-import '../../account_manager/account_manager.dart';
-import '../../chat_list/chat_list.dart';
-import '../../contacts/contacts.dart';
-import '../../proto/proto.dart' as proto;
-import '../../theme/theme.dart';
-import '../chat.dart';
-
-class ChatComponent extends StatelessWidget {
-  const ChatComponent._(
-      {required TypedKey localUserIdentityKey,
-      required SingleContactMessagesCubit messagesCubit,
-      required SingleContactMessagesState messagesState,
-      required types.User localUser,
-      required types.User remoteUser,
-      super.key})
-      : _localUserIdentityKey = localUserIdentityKey,
-        _messagesCubit = messagesCubit,
-        _messagesState = messagesState,
-        _localUser = localUser,
-        _remoteUser = remoteUser;
-
-  final TypedKey _localUserIdentityKey;
-  final SingleContactMessagesCubit _messagesCubit;
-  final SingleContactMessagesState _messagesState;
-  final types.User _localUser;
-  final types.User _remoteUser;
-
-  // Builder wrapper function that takes care of state management requirements
-  static Widget builder(
-          {required TypedKey remoteConversationRecordKey, Key? key}) =>
-      Builder(builder: (context) {
-        // Get all watched dependendies
-        final activeAccountInfo = context.watch<ActiveAccountInfo>();
-        final accountRecordInfo =
-            context.watch<AccountRecordCubit>().state.asData?.value;
-        if (accountRecordInfo == null) {
-          return debugPage('should always have an account record here');
-        }
-        final contactList =
-            context.watch<ContactListCubit>().state.state.asData?.value;
-        if (contactList == null) {
-          return debugPage('should always have a contact list here');
-        }
-        final avconversation = context.select<ActiveConversationsBlocMapCubit,
-                AsyncValue<ActiveConversationState>?>(
-            (x) => x.state[remoteConversationRecordKey]);
-        if (avconversation == null) {
-          return waitingPage();
-        }
-        final conversation = avconversation.asData?.value;
-        if (conversation == null) {
-          return avconversation.buildNotData();
-        }
-
-        // Make flutter_chat_ui 'User's
-        final localUserIdentityKey = activeAccountInfo
-            .localAccount.identityMaster
-            .identityPublicTypedKey();
-
-        final localUser = types.User(
-          id: localUserIdentityKey.toString(),
-          firstName: accountRecordInfo.profile.name,
-        );
-        final editedName = conversation.contact.editedProfile.name;
-        final remoteUser = types.User(
-            id: conversation.contact.identityPublicKey.toVeilid().toString(),
-            firstName: editedName);
-
-        // Get the messages cubit
-        final messages = context.select<ActiveSingleContactChatBlocMapCubit,
-                (SingleContactMessagesCubit, SingleContactMessagesState)?>(
-            (x) => x.tryOperate(remoteConversationRecordKey,
-                closure: (cubit) => (cubit, cubit.state)));
-
-        // Get the messages to display
-        // and ensure it is safe to operate() on the MessageCubit for this chat
-        if (messages == null) {
-          return waitingPage();
-        }
-
-        return ChatComponent._(
-            localUserIdentityKey: localUserIdentityKey,
-            messagesCubit: messages.$1,
-            messagesState: messages.$2,
-            localUser: localUser,
-            remoteUser: remoteUser,
-            key: key);
-      });
-
-  /////////////////////////////////////////////////////////////////////
-
-  types.Message messageToChatMessage(MessageState message) {
-    final isLocal = message.author == _localUserIdentityKey;
-
-    types.Status? status;
-    if (message.sendState != null) {
-      assert(isLocal, 'send state should only be on sent messages');
-      switch (message.sendState!) {
-        case MessageSendState.sending:
-          status = types.Status.sending;
-        case MessageSendState.sent:
-          status = types.Status.sent;
-        case MessageSendState.delivered:
-          status = types.Status.delivered;
-      }
-    }
-
-    final textMessage = types.TextMessage(
-        author: isLocal ? _localUser : _remoteUser,
-        createdAt: (message.timestamp.value ~/ BigInt.from(1000)).toInt(),
-        id: message.timestamp.toString(),
-        text: message.text,
-        showStatus: status != null,
-        status: status);
-    return textMessage;
-  }
-
-  void _addMessage(proto.Message message) {
-    if (message.text.isEmpty) {
-      return;
-    }
-    _messagesCubit.addMessage(message: message);
-  }
-
-  void _handleSendPressed(types.PartialText message) {
-    final protoMessage = proto.Message()
-      ..author = _localUserIdentityKey.toProto()
-      ..timestamp = Veilid.instance.now().toInt64()
-      ..text = message.text;
-    //..signature = signature;
-
-    _addMessage(protoMessage);
-  }
-
-  // void _handleAttachmentPressed() async {
-  //   //
-  // }
-
-  @override
-  Widget build(BuildContext context) {
-    final theme = Theme.of(context);
-    final scale = theme.extension<ScaleScheme>()!;
-    final textTheme = Theme.of(context).textTheme;
-    final chatTheme = makeChatTheme(scale, textTheme);
-
-    final messages = _messagesState.asData?.value;
-    if (messages == null) {
-      return _messagesState.buildNotData();
-    }
-
-    // Convert protobuf messages to chat messages
-    final chatMessages = <types.Message>[];
-    final tsSet = <String>{};
-    for (final message in messages) {
-      final chatMessage = messageToChatMessage(message);
-      chatMessages.insert(0, chatMessage);
-      if (!tsSet.add(chatMessage.id)) {
-        // ignore: avoid_print
-        print('duplicate id found: ${chatMessage.id}:\n'
-            'Messages:\n$messages\n'
-            'ChatMessages:\n$chatMessages');
-        assert(false, 'should not have duplicate id');
-      }
-    }
-
-    return DefaultTextStyle(
-        style: textTheme.bodySmall!,
-        child: Align(
-          alignment: AlignmentDirectional.centerEnd,
-          child: Stack(
-            children: [
-              Column(
-                children: [
-                  Container(
-                    height: 48,
-                    decoration: BoxDecoration(
-                      color: scale.primaryScale.subtleBorder,
-                    ),
-                    child: Row(children: [
-                      Align(
-                          alignment: AlignmentDirectional.centerStart,
-                          child: Padding(
-                            padding: const EdgeInsetsDirectional.fromSTEB(
-                                16, 0, 16, 0),
-                            child: Text(_remoteUser.firstName!,
-                                textAlign: TextAlign.start,
-                                style: textTheme.titleMedium!.copyWith(
-                                    color: scale.primaryScale.borderText)),
-                          )),
-                      const Spacer(),
-                      IconButton(
-                          icon: Icon(Icons.close,
-                              color: scale.primaryScale.borderText),
-                          onPressed: () async {
-                            context.read<ActiveChatCubit>().setActiveChat(null);
-                          }).paddingLTRB(16, 0, 16, 0)
-                    ]),
-                  ),
-                  Expanded(
-                    child: DecoratedBox(
-                      decoration: const BoxDecoration(),
-                      child: Chat(
-                        theme: chatTheme,
-                        // emojiEnlargementBehavior:
-                        //     EmojiEnlargementBehavior.multi,
-                        messages: chatMessages,
-                        //onAttachmentPressed: _handleAttachmentPressed,
-                        //onMessageTap: _handleMessageTap,
-                        //onPreviewDataFetched: _handlePreviewDataFetched,
-                        onSendPressed: _handleSendPressed,
-                        //showUserAvatars: false,
-                        //showUserNames: true,
-                        user: _localUser,
-                      ),
-                    ),
-                  ),
-                ],
-              ),
-            ],
-          ),
-        ));
-  }
-}

lib/chat/views/chat_component_widget.dart

@@ -0,0 +1,294 @@
+import 'dart:math';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:awesome_extensions/awesome_extensions.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_bloc/flutter_bloc.dart';
+import 'package:flutter_chat_types/flutter_chat_types.dart' as types;
+import 'package:flutter_chat_ui/flutter_chat_ui.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+import '../../account_manager/account_manager.dart';
+import '../../chat_list/chat_list.dart';
+import '../../theme/theme.dart';
+import '../chat.dart';
+
+const onEndReachedThreshold = 0.75;
+
+class ChatComponentWidget extends StatelessWidget {
+  const ChatComponentWidget._({required super.key});
+
+  // Builder wrapper function that takes care of state management requirements
+  static Widget builder(
+          {required TypedKey localConversationRecordKey, Key? key}) =>
+      Builder(builder: (context) {
+        // Get all watched dependendies
+        final activeAccountInfo = context.watch<ActiveAccountInfo>();
+        final accountRecordInfo =
+            context.watch<AccountRecordCubit>().state.asData?.value;
+        if (accountRecordInfo == null) {
+          return debugPage('should always have an account record here');
+        }
+
+        final avconversation = context.select<ActiveConversationsBlocMapCubit,
+                AsyncValue<ActiveConversationState>?>(
+            (x) => x.state[localConversationRecordKey]);
+        if (avconversation == null) {
+          return waitingPage();
+        }
+
+        final activeConversationState = avconversation.asData?.value;
+        if (activeConversationState == null) {
+          return avconversation.buildNotData();
+        }
+
+        // Get the messages cubit
+        final messagesCubit = context.select<
+                ActiveSingleContactChatBlocMapCubit,
+                SingleContactMessagesCubit?>(
+            (x) => x.tryOperate(localConversationRecordKey,
+                closure: (cubit) => cubit));
+        if (messagesCubit == null) {
+          return waitingPage();
+        }
+
+        // Make chat component state
+        return BlocProvider(
+            create: (context) => ChatComponentCubit.singleContact(
+                  activeAccountInfo: activeAccountInfo,
+                  accountRecordInfo: accountRecordInfo,
+                  activeConversationState: activeConversationState,
+                  messagesCubit: messagesCubit,
+                ),
+            child: ChatComponentWidget._(key: key));
+      });
+
+  /////////////////////////////////////////////////////////////////////
+
+  void _handleSendPressed(
+      ChatComponentCubit chatComponentCubit, types.PartialText message) {
+    final text = message.text;
+
+    if (text.startsWith('/')) {
+      chatComponentCubit.runCommand(text);
+      return;
+    }
+
+    chatComponentCubit.sendMessage(message);
+  }
+
+  // void _handleAttachmentPressed() async {
+  //   //
+  // }
+
+  Future<void> _handlePageForward(
+      ChatComponentCubit chatComponentCubit,
+      WindowState<types.Message> messageWindow,
+      ScrollNotification notification) async {
+    print(
+        '_handlePageForward: messagesState.length=${messageWindow.length} messagesState.windowTail=${messageWindow.windowTail} messagesState.windowCount=${messageWindow.windowCount} ScrollNotification=$notification');
+
+    // Go forward a page
+    final tail = min(messageWindow.length,
+            messageWindow.windowTail + (messageWindow.windowCount ~/ 4)) %
+        messageWindow.length;
+
+    // Set follow
+    final follow = messageWindow.length == 0 ||
+        tail == 0; // xxx incorporate scroll position
+
+    // final scrollOffset = (notification.metrics.maxScrollExtent -
+    //         notification.metrics.minScrollExtent) *
+    //     (1.0 - onEndReachedThreshold);
+
+    // chatComponentCubit.scrollOffset = scrollOffset;
+
+    await chatComponentCubit.setWindow(
+        tail: tail, count: messageWindow.windowCount, follow: follow);
+
+    // chatComponentCubit.state.scrollController.position.jumpTo(
+    //     chatComponentCubit.state.scrollController.offset + scrollOffset);
+
+    //chatComponentCubit.scrollOffset = 0;
+  }
+
+  Future<void> _handlePageBackward(
+    ChatComponentCubit chatComponentCubit,
+    WindowState<types.Message> messageWindow,
+    ScrollNotification notification,
+  ) async {
+    print(
+        '_handlePageBackward: messagesState.length=${messageWindow.length} messagesState.windowTail=${messageWindow.windowTail} messagesState.windowCount=${messageWindow.windowCount} ScrollNotification=$notification');
+
+    // Go back a page
+    final tail = max(
+        messageWindow.windowCount,
+        (messageWindow.windowTail - (messageWindow.windowCount ~/ 4)) %
+            messageWindow.length);
+
+    // Set follow
+    final follow = messageWindow.length == 0 ||
+        tail == 0; // xxx incorporate scroll position
+
+    // final scrollOffset = -(notification.metrics.maxScrollExtent -
+    //         notification.metrics.minScrollExtent) *
+    //     (1.0 - onEndReachedThreshold);
+
+    // chatComponentCubit.scrollOffset = scrollOffset;
+
+    await chatComponentCubit.setWindow(
+        tail: tail, count: messageWindow.windowCount, follow: follow);
+
+    // chatComponentCubit.scrollOffset = scrollOffset;
+
+    // chatComponentCubit.state.scrollController.position.jumpTo(
+    //     chatComponentCubit.state.scrollController.offset + scrollOffset);
+
+    //chatComponentCubit.scrollOffset = 0;
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final theme = Theme.of(context);
+    final scale = theme.extension<ScaleScheme>()!;
+    final textTheme = Theme.of(context).textTheme;
+    final chatTheme = makeChatTheme(scale, textTheme);
+
+    // Get the enclosing chat component cubit that contains our state
+    // (created by ChatComponentWidget.builder())
+    final chatComponentCubit = context.watch<ChatComponentCubit>();
+    final chatComponentState = chatComponentCubit.state;
+
+    final messageWindow = chatComponentState.messageWindow.asData?.value;
+    if (messageWindow == null) {
+      return chatComponentState.messageWindow.buildNotData();
+    }
+    final isLastPage = messageWindow.windowStart == 0;
+    final isFirstPage = messageWindow.windowEnd == messageWindow.length - 1;
+    final title = chatComponentState.title;
+
+    if (chatComponentCubit.scrollOffset != 0) {
+      chatComponentState.scrollController.position.correctPixels(
+          chatComponentState.scrollController.position.pixels +
+              chatComponentCubit.scrollOffset);
+
+      chatComponentCubit.scrollOffset = 0;
+    }
+
+    return DefaultTextStyle(
+        style: textTheme.bodySmall!,
+        child: Align(
+          alignment: AlignmentDirectional.centerEnd,
+          child: Stack(
+            children: [
+              Column(
+                children: [
+                  Container(
+                    height: 48,
+                    decoration: BoxDecoration(
+                      color: scale.primaryScale.subtleBorder,
+                    ),
+                    child: Row(children: [
+                      Align(
+                          alignment: AlignmentDirectional.centerStart,
+                          child: Padding(
+                            padding: const EdgeInsetsDirectional.fromSTEB(
+                                16, 0, 16, 0),
+                            child: Text(title,
+                                textAlign: TextAlign.start,
+                                style: textTheme.titleMedium!.copyWith(
+                                    color: scale.primaryScale.borderText)),
+                          )),
+                      const Spacer(),
+                      IconButton(
+                          icon: Icon(Icons.close,
+                              color: scale.primaryScale.borderText),
+                          onPressed: () async {
+                            context.read<ActiveChatCubit>().setActiveChat(null);
+                          }).paddingLTRB(16, 0, 16, 0)
+                    ]),
+                  ),
+                  Expanded(
+                    child: DecoratedBox(
+                      decoration: const BoxDecoration(),
+                      child: NotificationListener<ScrollNotification>(
+                          onNotification: (notification) {
+                            if (chatComponentCubit.scrollOffset != 0) {
+                              return false;
+                            }
+
+                            if (!isFirstPage &&
+                                notification.metrics.pixels <=
+                                    ((notification.metrics.maxScrollExtent -
+                                                notification
+                                                    .metrics.minScrollExtent) *
+                                            (1.0 - onEndReachedThreshold) +
+                                        notification.metrics.minScrollExtent)) {
+                              //
+                              final scrollOffset = (notification
+                                          .metrics.maxScrollExtent -
+                                      notification.metrics.minScrollExtent) *
+                                  (1.0 - onEndReachedThreshold);
+
+                              chatComponentCubit.scrollOffset = scrollOffset;
+
+                              //
+                              singleFuture(chatComponentState.chatKey,
+                                  () async {
+                                await _handlePageForward(chatComponentCubit,
+                                    messageWindow, notification);
+                              });
+                            } else if (!isLastPage &&
+                                notification.metrics.pixels >=
+                                    ((notification.metrics.maxScrollExtent -
+                                                notification
+                                                    .metrics.minScrollExtent) *
+                                            onEndReachedThreshold +
+                                        notification.metrics.minScrollExtent)) {
+                              //
+                              final scrollOffset = -(notification
+                                          .metrics.maxScrollExtent -
+                                      notification.metrics.minScrollExtent) *
+                                  (1.0 - onEndReachedThreshold);
+
+                              chatComponentCubit.scrollOffset = scrollOffset;
+                              //
+                              singleFuture(chatComponentState.chatKey,
+                                  () async {
+                                await _handlePageBackward(chatComponentCubit,
+                                    messageWindow, notification);
+                              });
+                            }
+                            return false;
+                          },
+                          child: Chat(
+                              key: chatComponentState.chatKey,
+                              theme: chatTheme,
+                              messages: messageWindow.window.toList(),
+                              scrollToBottomOnSend: isFirstPage,
+                              scrollController:
+                                  chatComponentState.scrollController,
+                              // isLastPage: isLastPage,
+                              // onEndReached: () async {
+                              //   await _handlePageBackward(
+                              //       chatComponentCubit, messageWindow);
+                              // },
+                              //onEndReachedThreshold: onEndReachedThreshold,
+                              //onAttachmentPressed: _handleAttachmentPressed,
+                              //onMessageTap: _handleMessageTap,
+                              //onPreviewDataFetched: _handlePreviewDataFetched,
+                              onSendPressed: (pt) =>
+                                  _handleSendPressed(chatComponentCubit, pt),
+                              //showUserAvatars: false,
+                              //showUserNames: true,
+                              user: chatComponentState.localUser,
+                              emptyState: const EmptyChatWidget())),
+                    ),
+                  ),
+                ],
+              ),
+            ],
+          ),
+        ));
+  }
+}

lib/chat/views/no_conversation_widget.dart

@@ -1,4 +1,7 @@
 import 'package:flutter/material.dart';
+import 'package:flutter_translate/flutter_translate.dart';
+
+import '../../theme/models/scale_scheme.dart';
 
 class NoConversationWidget extends StatelessWidget {
   const NoConversationWidget({super.key});
@@ -7,28 +10,32 @@ class NoConversationWidget extends StatelessWidget {
   // ignore: prefer_expression_function_bodies
   Widget build(
     BuildContext context,
-  ) =>
-      Container(
+  ) {
+    final theme = Theme.of(context);
+    final scale = theme.extension<ScaleScheme>()!;
+
+    return Container(
       width: double.infinity,
       height: double.infinity,
       decoration: BoxDecoration(
-          color: Theme.of(context).primaryColor,
+        color: Theme.of(context).scaffoldBackgroundColor,
       ),
       child: Column(
         mainAxisAlignment: MainAxisAlignment.center,
         children: [
           Icon(
-              Icons.emoji_people_outlined,
-              color: Theme.of(context).disabledColor,
+            Icons.diversity_3,
+            color: scale.primaryScale.subtleBorder,
             size: 48,
           ),
           Text(
-              'Choose A Conversation To Chat',
+            translate('chat.start_a_conversation'),
             style: Theme.of(context).textTheme.bodyMedium?.copyWith(
-                    color: Theme.of(context).disabledColor,
+                  color: scale.primaryScale.subtleBorder,
                 ),
           ),
         ],
       ),
     );
   }
+}

lib/chat/views/views.dart

@@ -1,4 +1,4 @@
-export 'chat_component.dart';
+export 'chat_component_widget.dart';
 export 'empty_chat_widget.dart';
 export 'new_chat_bottom_sheet.dart';
 export 'no_conversation_widget.dart';

lib/chat_list/cubits/active_conversations_bloc_map_cubit.dart

@@ -31,7 +31,7 @@ typedef ActiveConversationCubit = TransformerCubit<
 typedef ActiveConversationsBlocMapState
     = BlocMapState<TypedKey, AsyncValue<ActiveConversationState>>;
 
-// Map of remoteConversationRecordKey to ActiveConversationCubit
+// Map of localConversationRecordKey to ActiveConversationCubit
 // Wraps a conversation cubit to only expose completely built conversations
 // Automatically follows the state of a ChatListCubit.
 // Even though 'conversations' are per-contact and not per-chat
@@ -49,7 +49,7 @@ class ActiveConversationsBlocMapCubit extends BlocMapCubit<TypedKey,
   // Add an active conversation to be tracked for changes
   Future<void> _addConversation({required proto.Contact contact}) async =>
       add(() => MapEntry(
-          contact.remoteConversationRecordKey.toVeilid(),
+          contact.localConversationRecordKey.toVeilid(),
           TransformerCubit(
               ConversationCubit(
                 activeAccountInfo: _activeAccountInfo,
@@ -86,7 +86,7 @@ class ActiveConversationsBlocMapCubit extends BlocMapCubit<TypedKey,
       return;
     }
     final contactIndex = contactList.indexWhere(
-        (c) => c.value.remoteConversationRecordKey.toVeilid() == key);
+        (c) => c.value.localConversationRecordKey.toVeilid() == key);
     if (contactIndex == -1) {
       await addState(key, AsyncValue.error('Contact not found'));
       return;

lib/chat_list/cubits/active_single_contact_chat_bloc_map_cubit.dart

@@ -11,7 +11,7 @@ import '../../proto/proto.dart' as proto;
 import 'active_conversations_bloc_map_cubit.dart';
 import 'chat_list_cubit.dart';
 
-// Map of remoteConversationRecordKey to MessagesCubit
+// Map of localConversationRecordKey to MessagesCubit
 // Wraps a MessagesCubit to stream the latest messages to the state
 // Automatically follows the state of a ActiveConversationsBlocMapCubit.
 class ActiveSingleContactChatBlocMapCubit extends BlocMapCubit<TypedKey,
@@ -33,7 +33,7 @@ class ActiveSingleContactChatBlocMapCubit extends BlocMapCubit<TypedKey,
           required proto.Conversation localConversation,
           required proto.Conversation remoteConversation}) async =>
       add(() => MapEntry(
-          contact.remoteConversationRecordKey.toVeilid(),
+          contact.localConversationRecordKey.toVeilid(),
           SingleContactMessagesCubit(
             activeAccountInfo: _activeAccountInfo,
             remoteIdentityPublicKey: contact.identityPublicKey.toVeilid(),
@@ -43,7 +43,6 @@ class ActiveSingleContactChatBlocMapCubit extends BlocMapCubit<TypedKey,
                 contact.remoteConversationRecordKey.toVeilid(),
             localMessagesRecordKey: localConversation.messages.toVeilid(),
             remoteMessagesRecordKey: remoteConversation.messages.toVeilid(),
-            reconciledChatRecord: chat.reconciledChatRecord.toVeilid(),
           )));
 
   /// StateFollower /////////////////////////
@@ -61,7 +60,7 @@ class ActiveSingleContactChatBlocMapCubit extends BlocMapCubit<TypedKey,
       return;
     }
     final contactIndex = contactList.indexWhere(
-        (c) => c.value.remoteConversationRecordKey.toVeilid() == key);
+        (c) => c.value.localConversationRecordKey.toVeilid() == key);
     if (contactIndex == -1) {
       await addState(
           key, AsyncValue.error('Contact not found for conversation'));
@@ -76,7 +75,7 @@ class ActiveSingleContactChatBlocMapCubit extends BlocMapCubit<TypedKey,
       return;
     }
     final chatIndex = chatList.indexWhere(
-        (c) => c.value.remoteConversationRecordKey.toVeilid() == key);
+        (c) => c.value.localConversationRecordKey.toVeilid() == key);
     if (contactIndex == -1) {
       await addState(key, AsyncValue.error('Chat not found for conversation'));
       return;

lib/chat_list/cubits/chat_list_cubit.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 
 import 'package:bloc_advanced_tools/bloc_advanced_tools.dart';
 import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:fixnum/fixnum.dart';
 import 'package:veilid_support/veilid_support.dart';
 
 import '../../account_manager/account_manager.dart';
@@ -21,8 +22,7 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
     required ActiveAccountInfo activeAccountInfo,
     required proto.Account account,
     required this.activeChatCubit,
-  })  : _activeAccountInfo = activeAccountInfo,
-        super(
+  }) : super(
             open: () => _open(activeAccountInfo, account),
             decodeElement: proto.Chat.fromBuffer);
 
@@ -39,46 +39,52 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
     return dhtRecord;
   }
 
+  Future<proto.ChatSettings> getDefaultChatSettings(
+      proto.Contact contact) async {
+    final pronouns = contact.editedProfile.pronouns.isEmpty
+        ? ''
+        : ' (${contact.editedProfile.pronouns})';
+    return proto.ChatSettings()
+      ..title = '${contact.editedProfile.name}$pronouns'
+      ..description = ''
+      ..defaultExpiration = Int64.ZERO;
+  }
+
   /// Create a new chat (singleton for single contact chats)
   Future<void> getOrCreateChatSingleContact({
-    required TypedKey remoteConversationRecordKey,
+    required proto.Contact contact,
   }) async {
+    // Make local copy so we don't share the buffer
+    final localConversationRecordKey =
+        contact.localConversationRecordKey.toVeilid();
+    final remoteConversationRecordKey =
+        contact.remoteConversationRecordKey.toVeilid();
+
     // Add Chat to account's list
     // if this fails, don't keep retrying, user can try again later
     await operateWrite((writer) async {
-      final remoteConversationRecordKeyProto =
-          remoteConversationRecordKey.toProto();
-
       // See if we have added this chat already
       for (var i = 0; i < writer.length; i++) {
-        final cbuf = await writer.getItem(i);
+        final cbuf = await writer.get(i);
         if (cbuf == null) {
           throw Exception('Failed to get chat');
         }
         final c = proto.Chat.fromBuffer(cbuf);
-        if (c.remoteConversationRecordKey == remoteConversationRecordKeyProto) {
+        if (c.localConversationRecordKey ==
+            contact.localConversationRecordKey) {
           // Nothing to do here
           return;
         }
       }
-      final accountRecordKey = _activeAccountInfo
-          .userLogin.accountRecordInfo.accountRecord.recordKey;
 
-      // Make a record that can store the reconciled version of the chat
-      final reconciledChatRecord = await (await DHTShortArray.create(
-              debugName:
-                  'ChatListCubit::getOrCreateChatSingleContact::ReconciledChat',
-              parent: accountRecordKey))
-          .scope((r) async => r.recordPointer);
-
-      // Create conversation type Chat
+      // Create 1:1 conversation type Chat
       final chat = proto.Chat()
-        ..type = proto.ChatType.SINGLE_CONTACT
-        ..remoteConversationRecordKey = remoteConversationRecordKeyProto
-        ..reconciledChatRecord = reconciledChatRecord.toProto();
+        ..settings = await getDefaultChatSettings(contact)
+        ..localConversationRecordKey = localConversationRecordKey.toProto()
+        ..remoteConversationRecordKey = remoteConversationRecordKey.toProto();
 
       // Add chat
-      final added = await writer.tryAddItem(chat.writeToBuffer());
+      final added = await writer.tryAdd(chat.writeToBuffer());
       if (!added) {
         throw Exception('Failed to add chat');
       }
@@ -87,26 +93,27 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
 
   /// Delete a chat
   Future<void> deleteChat(
-      {required TypedKey remoteConversationRecordKey}) async {
-    final remoteConversationKey = remoteConversationRecordKey.toProto();
+      {required TypedKey localConversationRecordKey}) async {
+    final localConversationRecordKeyProto =
+        localConversationRecordKey.toProto();
 
     // Remove Chat from account's list
     // if this fails, don't keep retrying, user can try again later
     final deletedItem =
         // Ensure followers get their changes before we return
         await syncFollowers(() => operateWrite((writer) async {
-              if (activeChatCubit.state == remoteConversationRecordKey) {
+              if (activeChatCubit.state == localConversationRecordKey) {
                 activeChatCubit.setActiveChat(null);
               }
               for (var i = 0; i < writer.length; i++) {
-                final c =
-                    await writer.getItemProtobuf(proto.Chat.fromBuffer, i);
+                final c = await writer.getProtobuf(proto.Chat.fromBuffer, i);
                 if (c == null) {
                   throw Exception('Failed to get chat');
                 }
-                if (c.remoteConversationRecordKey == remoteConversationKey) {
+                if (c.localConversationRecordKey ==
+                    localConversationRecordKeyProto) {
                   // Found the right chat
-                  await writer.removeItem(i);
+                  await writer.remove(i);
                   return c;
                 }
               }
@@ -116,10 +123,10 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
     // chat record now
     if (deletedItem != null) {
       try {
-        await DHTRecordPool.instance.deleteRecord(
-            deletedItem.reconciledChatRecord.toVeilid().recordKey);
+        await SingleContactMessagesCubit.cleanupAndDeleteMessages(
+            localConversationRecordKey: localConversationRecordKey);
       } on Exception catch (e) {
-        log.debug('error removing reconciled chat record: $e', e);
+        log.debug('error removing reconciled chat table: $e', e);
       }
     }
   }
@@ -132,10 +139,9 @@ class ChatListCubit extends DHTShortArrayCubit<proto.Chat>
       return IMap();
     }
     return IMap.fromIterable(stateValue,
-        keyMapper: (e) => e.value.remoteConversationRecordKey.toVeilid(),
+        keyMapper: (e) => e.value.localConversationRecordKey.toVeilid(),
         valueMapper: (e) => e.value);
   }
 
   final ActiveChatCubit activeChatCubit;
-  final ActiveAccountInfo _activeAccountInfo;
 }

lib/chat_list/views/chat_single_contact_item_widget.dart

@@ -24,9 +24,9 @@ class ChatSingleContactItemWidget extends StatelessWidget {
     BuildContext context,
   ) {
     final activeChatCubit = context.watch<ActiveChatCubit>();
-    final remoteConversationRecordKey =
-        _contact.remoteConversationRecordKey.toVeilid();
-    final selected = activeChatCubit.state == remoteConversationRecordKey;
+    final localConversationRecordKey =
+        _contact.localConversationRecordKey.toVeilid();
+    final selected = activeChatCubit.state == localConversationRecordKey;
 
     return SliderTile(
       key: ObjectKey(_contact),
@@ -38,7 +38,7 @@ class ChatSingleContactItemWidget extends StatelessWidget {
       icon: Icons.chat,
       onTap: () {
         singleFuture(activeChatCubit, () async {
-          activeChatCubit.setActiveChat(remoteConversationRecordKey);
+          activeChatCubit.setActiveChat(localConversationRecordKey);
         });
       },
       endActions: [
@@ -49,7 +49,7 @@ class ChatSingleContactItemWidget extends StatelessWidget {
             onPressed: (context) async {
               final chatListCubit = context.read<ChatListCubit>();
               await chatListCubit.deleteChat(
-                  remoteConversationRecordKey: remoteConversationRecordKey);
+                  localConversationRecordKey: localConversationRecordKey);
             })
       ],
     );

lib/chat_list/views/chat_single_contact_list_widget.dart

@@ -20,7 +20,7 @@ class ChatSingleContactListWidget extends StatelessWidget {
 
     return contactListV.builder((context, contactList) {
       final contactMap = IMap.fromIterable(contactList,
-          keyMapper: (c) => c.value.remoteConversationRecordKey,
+          keyMapper: (c) => c.value.localConversationRecordKey,
           valueMapper: (c) => c.value);
 
       final chatListV = context.watch<ChatListCubit>().state;
@@ -36,7 +36,7 @@ class ChatSingleContactListWidget extends StatelessWidget {
                             initialList: chatList.map((x) => x.value).toList(),
                             itemBuilder: (c) {
                               final contact =
-                                  contactMap[c.remoteConversationRecordKey];
+                                  contactMap[c.localConversationRecordKey];
                               if (contact == null) {
                                 return const Text('...');
                               }
@@ -49,7 +49,7 @@ class ChatSingleContactListWidget extends StatelessWidget {
                               final lowerValue = value.toLowerCase();
                               return chatList.map((x) => x.value).where((c) {
                                 final contact =
-                                    contactMap[c.remoteConversationRecordKey];
+                                    contactMap[c.localConversationRecordKey];
                                 if (contact == null) {
                                   return false;
                                 }

lib/contact_invitation/cubits/contact_invitation_list_cubit.dart

@@ -121,7 +121,7 @@ class ContactInvitationListCubit
               schema: DHTSchema.smpl(oCnt: 1, members: [
                 DHTSchemaMember(mCnt: 1, mKey: contactRequestWriter.key)
               ]),
-              crypto: const DHTRecordCryptoPublic()))
+              crypto: const VeilidCryptoPublic()))
           .deleteScope((contactRequestInbox) async {
         // Store ContactRequest in owner subkey
         await contactRequestInbox.eventualWriteProtobuf(creq);
@@ -129,9 +129,9 @@ class ContactInvitationListCubit
         await contactRequestInbox.eventualWriteBytes(Uint8List(0),
             subkey: 1,
             writer: contactRequestWriter,
-            crypto: await DHTRecordCryptoPrivate.fromTypedKeyPair(
-                TypedKeyPair.fromKeyPair(
-                    contactRequestInbox.key.kind, contactRequestWriter)));
+            crypto: await DHTRecordPool.privateCryptoFromTypedSecret(TypedKey(
+                kind: contactRequestInbox.key.kind,
+                value: contactRequestWriter.secret)));
 
         // Create ContactInvitation and SignedContactInvitation
         final cinv = proto.ContactInvitation()
@@ -159,7 +159,7 @@ class ContactInvitationListCubit
         // Add ContactInvitationRecord to account's list
         // if this fails, don't keep retrying, user can try again later
         await operateWrite((writer) async {
-          if (await writer.tryAddItem(cinvrec.writeToBuffer()) == false) {
+          if (await writer.tryAdd(cinvrec.writeToBuffer()) == false) {
             throw Exception('Failed to add contact invitation record');
           }
         });
@@ -179,14 +179,14 @@ class ContactInvitationListCubit
     // Remove ContactInvitationRecord from account's list
     final deletedItem = await operateWrite((writer) async {
       for (var i = 0; i < writer.length; i++) {
-        final item = await writer.getItemProtobuf(
+        final item = await writer.getProtobuf(
             proto.ContactInvitationRecord.fromBuffer, i);
         if (item == null) {
           throw Exception('Failed to get contact invitation record');
         }
         if (item.contactRequestInbox.recordKey.toVeilid() ==
             contactRequestInboxRecordKey) {
-          await writer.removeItem(i);
+          await writer.remove(i);
           return item;
         }
       }

lib/contact_invitation/cubits/contact_request_inbox_cubit.dart

@@ -28,16 +28,16 @@ class ContactRequestInboxCubit
     final pool = DHTRecordPool.instance;
     final accountRecordKey =
         activeAccountInfo.userLogin.accountRecordInfo.accountRecord.recordKey;
-    final writerKey = contactInvitationRecord.writerKey.toVeilid();
     final writerSecret = contactInvitationRecord.writerSecret.toVeilid();
     final recordKey =
         contactInvitationRecord.contactRequestInbox.recordKey.toVeilid();
-    final writer = TypedKeyPair(
-        kind: recordKey.kind, key: writerKey, secret: writerSecret);
+    final writerTypedSecret =
+        TypedKey(kind: recordKey.kind, value: writerSecret);
     return pool.openRecordRead(recordKey,
         debugName: 'ContactRequestInboxCubit::_open::'
             'ContactRequestInbox',
-        crypto: await DHTRecordCryptoPrivate.fromTypedKeyPair(writer),
+        crypto:
+            await DHTRecordPool.privateCryptoFromTypedSecret(writerTypedSecret),
         parent: accountRecordKey,
         defaultSubkey: 1);
   }

lib/contact_invitation/views/contact_invitation_item_widget.dart

@@ -27,12 +27,12 @@ class ContactInvitationItemWidget extends StatelessWidget {
   @override
   // ignore: prefer_expression_function_bodies
   Widget build(BuildContext context) {
-    // final remoteConversationKey =
-    //     contact.remoteConversationRecordKey.toVeilid();
+    // final localConversationKey =
+    //     contact.localConversationRecordKey.toVeilid();
 
     const selected =
         false; // xxx: eventually when we have selectable invitations:
-    // activeContactCubit.state == remoteConversationRecordKey;
+    // activeContactCubit.state == localConversationRecordKey;
 
     final tileDisabled =
         disabled || context.watch<ContactInvitationListCubit>().isBusy;

lib/contacts/cubits/contact_list_cubit.dart

@@ -56,7 +56,7 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
     // Add Contact to account's list
     // if this fails, don't keep retrying, user can try again later
     await operateWrite((writer) async {
-      if (!await writer.tryAddItem(contact.writeToBuffer())) {
+      if (!await writer.tryAdd(contact.writeToBuffer())) {
         throw Exception('Failed to add contact record');
       }
     });
@@ -72,13 +72,13 @@ class ContactListCubit extends DHTShortArrayCubit<proto.Contact> {
     // Remove Contact from account's list
     final deletedItem = await operateWrite((writer) async {
       for (var i = 0; i < writer.length; i++) {
-        final item = await writer.getItemProtobuf(proto.Contact.fromBuffer, i);
+        final item = await writer.getProtobuf(proto.Contact.fromBuffer, i);
         if (item == null) {
           throw Exception('Failed to get contact');
         }
-        if (item.remoteConversationRecordKey ==
-            contact.remoteConversationRecordKey) {
-          await writer.removeItem(i);
+        if (item.localConversationRecordKey ==
+            contact.localConversationRecordKey) {
+          await writer.remove(i);
           return item;
         }
       }

lib/contacts/cubits/conversation_cubit.dart

@@ -285,13 +285,13 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
     required ActiveAccountInfo activeAccountInfo,
     required TypedKey remoteIdentityPublicKey,
     required TypedKey localConversationKey,
-    required FutureOr<T> Function(DHTShortArray) callback,
+    required FutureOr<T> Function(DHTLog) callback,
   }) async {
     final crypto =
         await activeAccountInfo.makeConversationCrypto(remoteIdentityPublicKey);
     final writer = activeAccountInfo.conversationWriter;
 
-    return (await DHTShortArray.create(
+    return (await DHTLog.create(
             debugName: 'ConversationCubit::initLocalMessages::LocalMessages',
             parent: localConversationKey,
             crypto: crypto,
@@ -327,7 +327,7 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
     return update;
   }
 
-  Future<DHTRecordCrypto> _cachedConversationCrypto() async {
+  Future<VeilidCrypto> _cachedConversationCrypto() async {
     var conversationCrypto = _conversationCrypto;
     if (conversationCrypto != null) {
       return conversationCrypto;
@@ -350,6 +350,6 @@ class ConversationCubit extends Cubit<AsyncValue<ConversationState>> {
   ConversationState _incrementalState = const ConversationState(
       localConversation: null, remoteConversation: null);
   //
-  DHTRecordCrypto? _conversationCrypto;
+  VeilidCrypto? _conversationCrypto;
   final WaitSet<void> _initWait = WaitSet();
 }

lib/contacts/views/contact_item_widget.dart

@@ -29,11 +29,11 @@ class ContactItemWidget extends StatelessWidget {
   Widget build(
     BuildContext context,
   ) {
-    final remoteConversationKey =
-        contact.remoteConversationRecordKey.toVeilid();
+    final localConversationRecordKey =
+        contact.localConversationRecordKey.toVeilid();
 
     const selected = false; // xxx: eventually when we have selectable contacts:
-    // activeContactCubit.state == remoteConversationRecordKey;
+    // activeContactCubit.state == localConversationRecordKey;
 
     final tileDisabled = disabled || context.watch<ContactListCubit>().isBusy;
 
@@ -49,8 +49,7 @@ class ContactItemWidget extends StatelessWidget {
         // Start a chat
         final chatListCubit = context.read<ChatListCubit>();
 
-        await chatListCubit.getOrCreateChatSingleContact(
-            remoteConversationRecordKey: remoteConversationKey);
+        await chatListCubit.getOrCreateChatSingleContact(contact: contact);
         // Click over to chats
         if (context.mounted) {
           await MainPager.of(context)
@@ -69,7 +68,7 @@ class ContactItemWidget extends StatelessWidget {
 
               // Remove any chats for this contact
               await chatListCubit.deleteChat(
-                  remoteConversationRecordKey: remoteConversationKey);
+                  localConversationRecordKey: localConversationRecordKey);
 
               // Delete the contact itself
               await contactListCubit.deleteContact(contact: contact);

lib/layout/home/home_account_ready/home_account_ready_chat.dart

@@ -28,13 +28,14 @@ class HomeAccountReadyChatState extends State<HomeAccountReadyChat> {
   }
 
   Widget buildChatComponent(BuildContext context) {
-    final activeChatRemoteConversationKey =
+    final activeChatLocalConversationKey =
         context.watch<ActiveChatCubit>().state;
-    if (activeChatRemoteConversationKey == null) {
-      return const EmptyChatWidget();
+    if (activeChatLocalConversationKey == null) {
+      return const NoConversationWidget();
     }
-    return ChatComponent.builder(
-        remoteConversationRecordKey: activeChatRemoteConversationKey);
+    return ChatComponentWidget.builder(
+        localConversationRecordKey: activeChatLocalConversationKey,
+        key: ValueKey(activeChatLocalConversationKey));
   }
 
   @override

lib/layout/home/home_account_ready/home_account_ready_main.dart

@@ -40,12 +40,10 @@ class _HomeAccountReadyMainState extends State<HomeAccountReadyMain> {
                 color: scale.secondaryScale.borderText,
                 constraints: const BoxConstraints.expand(height: 64, width: 64),
                 style: ButtonStyle(
-                    backgroundColor: MaterialStateProperty.all(
-                        scale.primaryScale.hoverBorder),
-                    shape: MaterialStateProperty.all(
-                        const RoundedRectangleBorder(
-                            borderRadius:
-                                BorderRadius.all(Radius.circular(16))))),
+                    backgroundColor:
+                        WidgetStateProperty.all(scale.primaryScale.hoverBorder),
+                    shape: WidgetStateProperty.all(const RoundedRectangleBorder(
+                        borderRadius: BorderRadius.all(Radius.circular(16))))),
                 tooltip: translate('app_bar.settings_tooltip'),
                 onPressed: () async {
                   await GoRouterHelper(context).push('/settings');
@@ -66,13 +64,14 @@ class _HomeAccountReadyMainState extends State<HomeAccountReadyMain> {
           Material(color: Colors.transparent, child: buildUserPanel()));
 
   Widget buildTabletRightPane(BuildContext context) {
-    final activeChatRemoteConversationKey =
+    final activeChatLocalConversationKey =
         context.watch<ActiveChatCubit>().state;
-    if (activeChatRemoteConversationKey == null) {
-      return const EmptyChatWidget();
+    if (activeChatLocalConversationKey == null) {
+      return const NoConversationWidget();
     }
-    return ChatComponent.builder(
-        remoteConversationRecordKey: activeChatRemoteConversationKey);
+    return ChatComponentWidget.builder(
+      localConversationRecordKey: activeChatLocalConversationKey,
+    );
   }
 
   // ignore: prefer_expression_function_bodies

lib/main.dart

@@ -6,6 +6,7 @@ import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_translate/flutter_translate.dart';
 import 'package:intl/date_symbol_data_local.dart';
+import 'package:stack_trace/stack_trace.dart';
 
 import 'app.dart';
 import 'settings/preferences_repository.dart';
@@ -52,7 +53,8 @@ void main() async {
 
   if (kDebugMode) {
     // In debug mode, run the app without catching exceptions for debugging
-    await mainFunc();
+    // but do a much deeper async stack trace capture
+    await Chain.capture(mainFunc);
   } else {
     // Catch errors in production without killing the app
     await runZonedGuarded(mainFunc, (error, stackTrace) {

lib/proto/extensions.dart

@@ -0,0 +1,31 @@
+import 'dart:typed_data';
+
+import 'package:veilid_support/veilid_support.dart';
+
+import 'proto.dart' as proto;
+
+proto.Message messageFromJson(Map<String, dynamic> j) =>
+    proto.Message.create()..mergeFromJsonMap(j);
+
+Map<String, dynamic> messageToJson(proto.Message m) => m.writeToJsonMap();
+
+proto.ReconciledMessage reconciledMessageFromJson(Map<String, dynamic> j) =>
+    proto.ReconciledMessage.create()..mergeFromJsonMap(j);
+
+Map<String, dynamic> reconciledMessageToJson(proto.ReconciledMessage m) =>
+    m.writeToJsonMap();
+
+extension MessageExt on proto.Message {
+  Uint8List get idBytes => Uint8List.fromList(id);
+
+  Uint8List get authorUniqueIdBytes {
+    final author = this.author.toVeilid().decode();
+    final id = this.id;
+    return Uint8List.fromList([...author, ...id]);
+  }
+
+  String get authorUniqueIdString => base64UrlNoPadEncode(authorUniqueIdBytes);
+
+  static int compareTimestamp(proto.Message a, proto.Message b) =>
+      a.timestamp.compareTo(b.timestamp);
+}

lib/proto/proto.dart

@@ -1,5 +1,7 @@
 export 'package:veilid_support/dht_support/proto/proto.dart';
 export 'package:veilid_support/proto/proto.dart';
+
+export 'extensions.dart';
 export 'veilidchat.pb.dart';
 export 'veilidchat.pbenum.dart';
 export 'veilidchat.pbjson.dart';

lib/proto/veilidchat.pbenum.dart

@@ -13,23 +13,6 @@ import 'dart:core' as $core;
 
 import 'package:protobuf/protobuf.dart' as $pb;
 
-class AttachmentKind extends $pb.ProtobufEnum {
-  static const AttachmentKind ATTACHMENT_KIND_UNSPECIFIED = AttachmentKind._(0, _omitEnumNames ? '' : 'ATTACHMENT_KIND_UNSPECIFIED');
-  static const AttachmentKind ATTACHMENT_KIND_FILE = AttachmentKind._(1, _omitEnumNames ? '' : 'ATTACHMENT_KIND_FILE');
-  static const AttachmentKind ATTACHMENT_KIND_IMAGE = AttachmentKind._(2, _omitEnumNames ? '' : 'ATTACHMENT_KIND_IMAGE');
-
-  static const $core.List<AttachmentKind> values = <AttachmentKind> [
-    ATTACHMENT_KIND_UNSPECIFIED,
-    ATTACHMENT_KIND_FILE,
-    ATTACHMENT_KIND_IMAGE,
-  ];
-
-  static final $core.Map<$core.int, AttachmentKind> _byValue = $pb.ProtobufEnum.initByValue(values);
-  static AttachmentKind? valueOf($core.int value) => _byValue[value];
-
-  const AttachmentKind._($core.int v, $core.String n) : super(v, n);
-}
-
 class Availability extends $pb.ProtobufEnum {
   static const Availability AVAILABILITY_UNSPECIFIED = Availability._(0, _omitEnumNames ? '' : 'AVAILABILITY_UNSPECIFIED');
   static const Availability AVAILABILITY_OFFLINE = Availability._(1, _omitEnumNames ? '' : 'AVAILABILITY_OFFLINE');
@@ -51,23 +34,6 @@ class Availability extends $pb.ProtobufEnum {
   const Availability._($core.int v, $core.String n) : super(v, n);
 }
 
-class ChatType extends $pb.ProtobufEnum {
-  static const ChatType CHAT_TYPE_UNSPECIFIED = ChatType._(0, _omitEnumNames ? '' : 'CHAT_TYPE_UNSPECIFIED');
-  static const ChatType SINGLE_CONTACT = ChatType._(1, _omitEnumNames ? '' : 'SINGLE_CONTACT');
-  static const ChatType GROUP = ChatType._(2, _omitEnumNames ? '' : 'GROUP');
-
-  static const $core.List<ChatType> values = <ChatType> [
-    CHAT_TYPE_UNSPECIFIED,
-    SINGLE_CONTACT,
-    GROUP,
-  ];
-
-  static final $core.Map<$core.int, ChatType> _byValue = $pb.ProtobufEnum.initByValue(values);
-  static ChatType? valueOf($core.int value) => _byValue[value];
-
-  const ChatType._($core.int v, $core.String n) : super(v, n);
-}
-
 class EncryptionKeyType extends $pb.ProtobufEnum {
   static const EncryptionKeyType ENCRYPTION_KEY_TYPE_UNSPECIFIED = EncryptionKeyType._(0, _omitEnumNames ? '' : 'ENCRYPTION_KEY_TYPE_UNSPECIFIED');
   static const EncryptionKeyType ENCRYPTION_KEY_TYPE_NONE = EncryptionKeyType._(1, _omitEnumNames ? '' : 'ENCRYPTION_KEY_TYPE_NONE');
@@ -87,5 +53,26 @@ class EncryptionKeyType extends $pb.ProtobufEnum {
   const EncryptionKeyType._($core.int v, $core.String n) : super(v, n);
 }
 
+class Scope extends $pb.ProtobufEnum {
+  static const Scope WATCHERS = Scope._(0, _omitEnumNames ? '' : 'WATCHERS');
+  static const Scope MODERATED = Scope._(1, _omitEnumNames ? '' : 'MODERATED');
+  static const Scope TALKERS = Scope._(2, _omitEnumNames ? '' : 'TALKERS');
+  static const Scope MODERATORS = Scope._(3, _omitEnumNames ? '' : 'MODERATORS');
+  static const Scope ADMINS = Scope._(4, _omitEnumNames ? '' : 'ADMINS');
+
+  static const $core.List<Scope> values = <Scope> [
+    WATCHERS,
+    MODERATED,
+    TALKERS,
+    MODERATORS,
+    ADMINS,
+  ];
+
+  static final $core.Map<$core.int, Scope> _byValue = $pb.ProtobufEnum.initByValue(values);
+  static Scope? valueOf($core.int value) => _byValue[value];
+
+  const Scope._($core.int v, $core.String n) : super(v, n);
+}
+
 
 const _omitEnumNames = $core.bool.fromEnvironment('protobuf.omit_enum_names');

lib/proto/veilidchat.pbjson.dart

@@ -13,21 +13,6 @@ import 'dart:convert' as $convert;
 import 'dart:core' as $core;
 import 'dart:typed_data' as $typed_data;
 
-@$core.Deprecated('Use attachmentKindDescriptor instead')
-const AttachmentKind$json = {
-  '1': 'AttachmentKind',
-  '2': [
-    {'1': 'ATTACHMENT_KIND_UNSPECIFIED', '2': 0},
-    {'1': 'ATTACHMENT_KIND_FILE', '2': 1},
-    {'1': 'ATTACHMENT_KIND_IMAGE', '2': 2},
-  ],
-};
-
-/// Descriptor for `AttachmentKind`. Decode as a `google.protobuf.EnumDescriptorProto`.
-final $typed_data.Uint8List attachmentKindDescriptor = $convert.base64Decode(
-    'Cg5BdHRhY2htZW50S2luZBIfChtBVFRBQ0hNRU5UX0tJTkRfVU5TUEVDSUZJRUQQABIYChRBVF'
-    'RBQ0hNRU5UX0tJTkRfRklMRRABEhkKFUFUVEFDSE1FTlRfS0lORF9JTUFHRRAC');
-
 @$core.Deprecated('Use availabilityDescriptor instead')
 const Availability$json = {
   '1': 'Availability',
@@ -46,21 +31,6 @@ final $typed_data.Uint8List availabilityDescriptor = $convert.base64Decode(
     'lMSVRZX09GRkxJTkUQARIVChFBVkFJTEFCSUxJVFlfRlJFRRACEhUKEUFWQUlMQUJJTElUWV9C'
     'VVNZEAMSFQoRQVZBSUxBQklMSVRZX0FXQVkQBA==');
 
-@$core.Deprecated('Use chatTypeDescriptor instead')
-const ChatType$json = {
-  '1': 'ChatType',
-  '2': [
-    {'1': 'CHAT_TYPE_UNSPECIFIED', '2': 0},
-    {'1': 'SINGLE_CONTACT', '2': 1},
-    {'1': 'GROUP', '2': 2},
-  ],
-};
-
-/// Descriptor for `ChatType`. Decode as a `google.protobuf.EnumDescriptorProto`.
-final $typed_data.Uint8List chatTypeDescriptor = $convert.base64Decode(
-    'CghDaGF0VHlwZRIZChVDSEFUX1RZUEVfVU5TUEVDSUZJRUQQABISCg5TSU5HTEVfQ09OVEFDVB'
-    'ABEgkKBUdST1VQEAI=');
-
 @$core.Deprecated('Use encryptionKeyTypeDescriptor instead')
 const EncryptionKeyType$json = {
   '1': 'EncryptionKeyType',
@@ -78,43 +48,261 @@ final $typed_data.Uint8List encryptionKeyTypeDescriptor = $convert.base64Decode(
     'ASHAoYRU5DUllQVElPTl9LRVlfVFlQRV9OT05FEAESGwoXRU5DUllQVElPTl9LRVlfVFlQRV9Q'
     'SU4QAhIgChxFTkNSWVBUSU9OX0tFWV9UWVBFX1BBU1NXT1JEEAM=');
 
+@$core.Deprecated('Use scopeDescriptor instead')
+const Scope$json = {
+  '1': 'Scope',
+  '2': [
+    {'1': 'WATCHERS', '2': 0},
+    {'1': 'MODERATED', '2': 1},
+    {'1': 'TALKERS', '2': 2},
+    {'1': 'MODERATORS', '2': 3},
+    {'1': 'ADMINS', '2': 4},
+  ],
+};
+
+/// Descriptor for `Scope`. Decode as a `google.protobuf.EnumDescriptorProto`.
+final $typed_data.Uint8List scopeDescriptor = $convert.base64Decode(
+    'CgVTY29wZRIMCghXQVRDSEVSUxAAEg0KCU1PREVSQVRFRBABEgsKB1RBTEtFUlMQAhIOCgpNT0'
+    'RFUkFUT1JTEAMSCgoGQURNSU5TEAQ=');
+
 @$core.Deprecated('Use attachmentDescriptor instead')
 const Attachment$json = {
   '1': 'Attachment',
   '2': [
-    {'1': 'kind', '3': 1, '4': 1, '5': 14, '6': '.veilidchat.AttachmentKind', '10': 'kind'},
-    {'1': 'mime', '3': 2, '4': 1, '5': 9, '10': 'mime'},
-    {'1': 'name', '3': 3, '4': 1, '5': 9, '10': 'name'},
-    {'1': 'content', '3': 4, '4': 1, '5': 11, '6': '.dht.DataReference', '10': 'content'},
-    {'1': 'signature', '3': 5, '4': 1, '5': 11, '6': '.veilid.Signature', '10': 'signature'},
+    {'1': 'media', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.AttachmentMedia', '9': 0, '10': 'media'},
+    {'1': 'signature', '3': 2, '4': 1, '5': 11, '6': '.veilid.Signature', '10': 'signature'},
+  ],
+  '8': [
+    {'1': 'kind'},
   ],
 };
 
 /// Descriptor for `Attachment`. Decode as a `google.protobuf.DescriptorProto`.
 final $typed_data.Uint8List attachmentDescriptor = $convert.base64Decode(
-    'CgpBdHRhY2htZW50Ei4KBGtpbmQYASABKA4yGi52ZWlsaWRjaGF0LkF0dGFjaG1lbnRLaW5kUg'
-    'RraW5kEhIKBG1pbWUYAiABKAlSBG1pbWUSEgoEbmFtZRgDIAEoCVIEbmFtZRIsCgdjb250ZW50'
-    'GAQgASgLMhIuZGh0LkRhdGFSZWZlcmVuY2VSB2NvbnRlbnQSLwoJc2lnbmF0dXJlGAUgASgLMh'
-    'EudmVpbGlkLlNpZ25hdHVyZVIJc2lnbmF0dXJl');
+    'CgpBdHRhY2htZW50EjMKBW1lZGlhGAEgASgLMhsudmVpbGlkY2hhdC5BdHRhY2htZW50TWVkaW'
+    'FIAFIFbWVkaWESLwoJc2lnbmF0dXJlGAIgASgLMhEudmVpbGlkLlNpZ25hdHVyZVIJc2lnbmF0'
+    'dXJlQgYKBGtpbmQ=');
+
+@$core.Deprecated('Use attachmentMediaDescriptor instead')
+const AttachmentMedia$json = {
+  '1': 'AttachmentMedia',
+  '2': [
+    {'1': 'mime', '3': 1, '4': 1, '5': 9, '10': 'mime'},
+    {'1': 'name', '3': 2, '4': 1, '5': 9, '10': 'name'},
+    {'1': 'content', '3': 3, '4': 1, '5': 11, '6': '.dht.DataReference', '10': 'content'},
+  ],
+};
+
+/// Descriptor for `AttachmentMedia`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List attachmentMediaDescriptor = $convert.base64Decode(
+    'Cg9BdHRhY2htZW50TWVkaWESEgoEbWltZRgBIAEoCVIEbWltZRISCgRuYW1lGAIgASgJUgRuYW'
+    '1lEiwKB2NvbnRlbnQYAyABKAsyEi5kaHQuRGF0YVJlZmVyZW5jZVIHY29udGVudA==');
+
+@$core.Deprecated('Use permissionsDescriptor instead')
+const Permissions$json = {
+  '1': 'Permissions',
+  '2': [
+    {'1': 'can_add_members', '3': 1, '4': 1, '5': 14, '6': '.veilidchat.Scope', '10': 'canAddMembers'},
+    {'1': 'can_edit_info', '3': 2, '4': 1, '5': 14, '6': '.veilidchat.Scope', '10': 'canEditInfo'},
+    {'1': 'moderated', '3': 3, '4': 1, '5': 8, '10': 'moderated'},
+  ],
+};
+
+/// Descriptor for `Permissions`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List permissionsDescriptor = $convert.base64Decode(
+    'CgtQZXJtaXNzaW9ucxI5Cg9jYW5fYWRkX21lbWJlcnMYASABKA4yES52ZWlsaWRjaGF0LlNjb3'
+    'BlUg1jYW5BZGRNZW1iZXJzEjUKDWNhbl9lZGl0X2luZm8YAiABKA4yES52ZWlsaWRjaGF0LlNj'
+    'b3BlUgtjYW5FZGl0SW5mbxIcCgltb2RlcmF0ZWQYAyABKAhSCW1vZGVyYXRlZA==');
+
+@$core.Deprecated('Use membershipDescriptor instead')
+const Membership$json = {
+  '1': 'Membership',
+  '2': [
+    {'1': 'watchers', '3': 1, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'watchers'},
+    {'1': 'moderated', '3': 2, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'moderated'},
+    {'1': 'talkers', '3': 3, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'talkers'},
+    {'1': 'moderators', '3': 4, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'moderators'},
+    {'1': 'admins', '3': 5, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'admins'},
+  ],
+};
+
+/// Descriptor for `Membership`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List membershipDescriptor = $convert.base64Decode(
+    'CgpNZW1iZXJzaGlwEiwKCHdhdGNoZXJzGAEgAygLMhAudmVpbGlkLlR5cGVkS2V5Ugh3YXRjaG'
+    'VycxIuCgltb2RlcmF0ZWQYAiADKAsyEC52ZWlsaWQuVHlwZWRLZXlSCW1vZGVyYXRlZBIqCgd0'
+    'YWxrZXJzGAMgAygLMhAudmVpbGlkLlR5cGVkS2V5Ugd0YWxrZXJzEjAKCm1vZGVyYXRvcnMYBC'
+    'ADKAsyEC52ZWlsaWQuVHlwZWRLZXlSCm1vZGVyYXRvcnMSKAoGYWRtaW5zGAUgAygLMhAudmVp'
+    'bGlkLlR5cGVkS2V5UgZhZG1pbnM=');
+
+@$core.Deprecated('Use chatSettingsDescriptor instead')
+const ChatSettings$json = {
+  '1': 'ChatSettings',
+  '2': [
+    {'1': 'title', '3': 1, '4': 1, '5': 9, '10': 'title'},
+    {'1': 'description', '3': 2, '4': 1, '5': 9, '10': 'description'},
+    {'1': 'icon', '3': 3, '4': 1, '5': 11, '6': '.dht.DataReference', '9': 0, '10': 'icon', '17': true},
+    {'1': 'default_expiration', '3': 4, '4': 1, '5': 4, '10': 'defaultExpiration'},
+  ],
+  '8': [
+    {'1': '_icon'},
+  ],
+};
+
+/// Descriptor for `ChatSettings`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List chatSettingsDescriptor = $convert.base64Decode(
+    'CgxDaGF0U2V0dGluZ3MSFAoFdGl0bGUYASABKAlSBXRpdGxlEiAKC2Rlc2NyaXB0aW9uGAIgAS'
+    'gJUgtkZXNjcmlwdGlvbhIrCgRpY29uGAMgASgLMhIuZGh0LkRhdGFSZWZlcmVuY2VIAFIEaWNv'
+    'bogBARItChJkZWZhdWx0X2V4cGlyYXRpb24YBCABKARSEWRlZmF1bHRFeHBpcmF0aW9uQgcKBV'
+    '9pY29u');
 
 @$core.Deprecated('Use messageDescriptor instead')
 const Message$json = {
   '1': 'Message',
   '2': [
-    {'1': 'author', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'author'},
-    {'1': 'timestamp', '3': 2, '4': 1, '5': 4, '10': 'timestamp'},
-    {'1': 'text', '3': 3, '4': 1, '5': 9, '10': 'text'},
-    {'1': 'signature', '3': 4, '4': 1, '5': 11, '6': '.veilid.Signature', '10': 'signature'},
-    {'1': 'attachments', '3': 5, '4': 3, '5': 11, '6': '.veilidchat.Attachment', '10': 'attachments'},
+    {'1': 'id', '3': 1, '4': 1, '5': 12, '10': 'id'},
+    {'1': 'author', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'author'},
+    {'1': 'timestamp', '3': 3, '4': 1, '5': 4, '10': 'timestamp'},
+    {'1': 'text', '3': 4, '4': 1, '5': 11, '6': '.veilidchat.Message.Text', '9': 0, '10': 'text'},
+    {'1': 'secret', '3': 5, '4': 1, '5': 11, '6': '.veilidchat.Message.Secret', '9': 0, '10': 'secret'},
+    {'1': 'delete', '3': 6, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlDelete', '9': 0, '10': 'delete'},
+    {'1': 'erase', '3': 7, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlErase', '9': 0, '10': 'erase'},
+    {'1': 'settings', '3': 8, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlSettings', '9': 0, '10': 'settings'},
+    {'1': 'permissions', '3': 9, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlPermissions', '9': 0, '10': 'permissions'},
+    {'1': 'membership', '3': 10, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlMembership', '9': 0, '10': 'membership'},
+    {'1': 'moderation', '3': 11, '4': 1, '5': 11, '6': '.veilidchat.Message.ControlModeration', '9': 0, '10': 'moderation'},
+    {'1': 'signature', '3': 12, '4': 1, '5': 11, '6': '.veilid.Signature', '10': 'signature'},
+  ],
+  '3': [Message_Text$json, Message_Secret$json, Message_ControlDelete$json, Message_ControlErase$json, Message_ControlSettings$json, Message_ControlPermissions$json, Message_ControlMembership$json, Message_ControlModeration$json, Message_ControlReadReceipt$json],
+  '8': [
+    {'1': 'kind'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_Text$json = {
+  '1': 'Text',
+  '2': [
+    {'1': 'text', '3': 1, '4': 1, '5': 9, '10': 'text'},
+    {'1': 'topic', '3': 2, '4': 1, '5': 9, '9': 0, '10': 'topic', '17': true},
+    {'1': 'reply_id', '3': 3, '4': 1, '5': 12, '9': 1, '10': 'replyId', '17': true},
+    {'1': 'expiration', '3': 4, '4': 1, '5': 4, '10': 'expiration'},
+    {'1': 'view_limit', '3': 5, '4': 1, '5': 13, '10': 'viewLimit'},
+    {'1': 'attachments', '3': 6, '4': 3, '5': 11, '6': '.veilidchat.Attachment', '10': 'attachments'},
+  ],
+  '8': [
+    {'1': '_topic'},
+    {'1': '_reply_id'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_Secret$json = {
+  '1': 'Secret',
+  '2': [
+    {'1': 'ciphertext', '3': 1, '4': 1, '5': 12, '10': 'ciphertext'},
+    {'1': 'expiration', '3': 2, '4': 1, '5': 4, '10': 'expiration'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlDelete$json = {
+  '1': 'ControlDelete',
+  '2': [
+    {'1': 'ids', '3': 1, '4': 3, '5': 12, '10': 'ids'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlErase$json = {
+  '1': 'ControlErase',
+  '2': [
+    {'1': 'timestamp', '3': 1, '4': 1, '5': 4, '10': 'timestamp'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlSettings$json = {
+  '1': 'ControlSettings',
+  '2': [
+    {'1': 'settings', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.ChatSettings', '10': 'settings'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlPermissions$json = {
+  '1': 'ControlPermissions',
+  '2': [
+    {'1': 'permissions', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.Permissions', '10': 'permissions'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlMembership$json = {
+  '1': 'ControlMembership',
+  '2': [
+    {'1': 'membership', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.Membership', '10': 'membership'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlModeration$json = {
+  '1': 'ControlModeration',
+  '2': [
+    {'1': 'accepted_ids', '3': 1, '4': 3, '5': 12, '10': 'acceptedIds'},
+    {'1': 'rejected_ids', '3': 2, '4': 3, '5': 12, '10': 'rejectedIds'},
+  ],
+};
+
+@$core.Deprecated('Use messageDescriptor instead')
+const Message_ControlReadReceipt$json = {
+  '1': 'ControlReadReceipt',
+  '2': [
+    {'1': 'read_ids', '3': 1, '4': 3, '5': 12, '10': 'readIds'},
   ],
 };
 
 /// Descriptor for `Message`. Decode as a `google.protobuf.DescriptorProto`.
 final $typed_data.Uint8List messageDescriptor = $convert.base64Decode(
-    'CgdNZXNzYWdlEigKBmF1dGhvchgBIAEoCzIQLnZlaWxpZC5UeXBlZEtleVIGYXV0aG9yEhwKCX'
-    'RpbWVzdGFtcBgCIAEoBFIJdGltZXN0YW1wEhIKBHRleHQYAyABKAlSBHRleHQSLwoJc2lnbmF0'
-    'dXJlGAQgASgLMhEudmVpbGlkLlNpZ25hdHVyZVIJc2lnbmF0dXJlEjgKC2F0dGFjaG1lbnRzGA'
-    'UgAygLMhYudmVpbGlkY2hhdC5BdHRhY2htZW50UgthdHRhY2htZW50cw==');
+    'CgdNZXNzYWdlEg4KAmlkGAEgASgMUgJpZBIoCgZhdXRob3IYAiABKAsyEC52ZWlsaWQuVHlwZW'
+    'RLZXlSBmF1dGhvchIcCgl0aW1lc3RhbXAYAyABKARSCXRpbWVzdGFtcBIuCgR0ZXh0GAQgASgL'
+    'MhgudmVpbGlkY2hhdC5NZXNzYWdlLlRleHRIAFIEdGV4dBI0CgZzZWNyZXQYBSABKAsyGi52ZW'
+    'lsaWRjaGF0Lk1lc3NhZ2UuU2VjcmV0SABSBnNlY3JldBI7CgZkZWxldGUYBiABKAsyIS52ZWls'
+    'aWRjaGF0Lk1lc3NhZ2UuQ29udHJvbERlbGV0ZUgAUgZkZWxldGUSOAoFZXJhc2UYByABKAsyIC'
+    '52ZWlsaWRjaGF0Lk1lc3NhZ2UuQ29udHJvbEVyYXNlSABSBWVyYXNlEkEKCHNldHRpbmdzGAgg'
+    'ASgLMiMudmVpbGlkY2hhdC5NZXNzYWdlLkNvbnRyb2xTZXR0aW5nc0gAUghzZXR0aW5ncxJKCg'
+    'twZXJtaXNzaW9ucxgJIAEoCzImLnZlaWxpZGNoYXQuTWVzc2FnZS5Db250cm9sUGVybWlzc2lv'
+    'bnNIAFILcGVybWlzc2lvbnMSRwoKbWVtYmVyc2hpcBgKIAEoCzIlLnZlaWxpZGNoYXQuTWVzc2'
+    'FnZS5Db250cm9sTWVtYmVyc2hpcEgAUgptZW1iZXJzaGlwEkcKCm1vZGVyYXRpb24YCyABKAsy'
+    'JS52ZWlsaWRjaGF0Lk1lc3NhZ2UuQ29udHJvbE1vZGVyYXRpb25IAFIKbW9kZXJhdGlvbhIvCg'
+    'lzaWduYXR1cmUYDCABKAsyES52ZWlsaWQuU2lnbmF0dXJlUglzaWduYXR1cmUa5QEKBFRleHQS'
+    'EgoEdGV4dBgBIAEoCVIEdGV4dBIZCgV0b3BpYxgCIAEoCUgAUgV0b3BpY4gBARIeCghyZXBseV'
+    '9pZBgDIAEoDEgBUgdyZXBseUlkiAEBEh4KCmV4cGlyYXRpb24YBCABKARSCmV4cGlyYXRpb24S'
+    'HQoKdmlld19saW1pdBgFIAEoDVIJdmlld0xpbWl0EjgKC2F0dGFjaG1lbnRzGAYgAygLMhYudm'
+    'VpbGlkY2hhdC5BdHRhY2htZW50UgthdHRhY2htZW50c0IICgZfdG9waWNCCwoJX3JlcGx5X2lk'
+    'GkgKBlNlY3JldBIeCgpjaXBoZXJ0ZXh0GAEgASgMUgpjaXBoZXJ0ZXh0Eh4KCmV4cGlyYXRpb2'
+    '4YAiABKARSCmV4cGlyYXRpb24aIQoNQ29udHJvbERlbGV0ZRIQCgNpZHMYASADKAxSA2lkcxos'
+    'CgxDb250cm9sRXJhc2USHAoJdGltZXN0YW1wGAEgASgEUgl0aW1lc3RhbXAaRwoPQ29udHJvbF'
+    'NldHRpbmdzEjQKCHNldHRpbmdzGAEgASgLMhgudmVpbGlkY2hhdC5DaGF0U2V0dGluZ3NSCHNl'
+    'dHRpbmdzGk8KEkNvbnRyb2xQZXJtaXNzaW9ucxI5CgtwZXJtaXNzaW9ucxgBIAEoCzIXLnZlaW'
+    'xpZGNoYXQuUGVybWlzc2lvbnNSC3Blcm1pc3Npb25zGksKEUNvbnRyb2xNZW1iZXJzaGlwEjYK'
+    'Cm1lbWJlcnNoaXAYASABKAsyFi52ZWlsaWRjaGF0Lk1lbWJlcnNoaXBSCm1lbWJlcnNoaXAaWQ'
+    'oRQ29udHJvbE1vZGVyYXRpb24SIQoMYWNjZXB0ZWRfaWRzGAEgAygMUgthY2NlcHRlZElkcxIh'
+    'CgxyZWplY3RlZF9pZHMYAiADKAxSC3JlamVjdGVkSWRzGi8KEkNvbnRyb2xSZWFkUmVjZWlwdB'
+    'IZCghyZWFkX2lkcxgBIAMoDFIHcmVhZElkc0IGCgRraW5k');
+
+@$core.Deprecated('Use reconciledMessageDescriptor instead')
+const ReconciledMessage$json = {
+  '1': 'ReconciledMessage',
+  '2': [
+    {'1': 'content', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.Message', '10': 'content'},
+    {'1': 'reconciled_time', '3': 2, '4': 1, '5': 4, '10': 'reconciledTime'},
+  ],
+};
+
+/// Descriptor for `ReconciledMessage`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List reconciledMessageDescriptor = $convert.base64Decode(
+    'ChFSZWNvbmNpbGVkTWVzc2FnZRItCgdjb250ZW50GAEgASgLMhMudmVpbGlkY2hhdC5NZXNzYW'
+    'dlUgdjb250ZW50EicKD3JlY29uY2lsZWRfdGltZRgCIAEoBFIOcmVjb25jaWxlZFRpbWU=');
 
 @$core.Deprecated('Use conversationDescriptor instead')
 const Conversation$json = {
@@ -132,6 +320,91 @@ final $typed_data.Uint8List conversationDescriptor = $convert.base64Decode(
     'JvZmlsZRIwChRpZGVudGl0eV9tYXN0ZXJfanNvbhgCIAEoCVISaWRlbnRpdHlNYXN0ZXJKc29u'
     'EiwKCG1lc3NhZ2VzGAMgASgLMhAudmVpbGlkLlR5cGVkS2V5UghtZXNzYWdlcw==');
 
+@$core.Deprecated('Use chatDescriptor instead')
+const Chat$json = {
+  '1': 'Chat',
+  '2': [
+    {'1': 'settings', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.ChatSettings', '10': 'settings'},
+    {'1': 'local_conversation_record_key', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'localConversationRecordKey'},
+    {'1': 'remote_conversation_record_key', '3': 3, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'remoteConversationRecordKey'},
+  ],
+};
+
+/// Descriptor for `Chat`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List chatDescriptor = $convert.base64Decode(
+    'CgRDaGF0EjQKCHNldHRpbmdzGAEgASgLMhgudmVpbGlkY2hhdC5DaGF0U2V0dGluZ3NSCHNldH'
+    'RpbmdzElMKHWxvY2FsX2NvbnZlcnNhdGlvbl9yZWNvcmRfa2V5GAIgASgLMhAudmVpbGlkLlR5'
+    'cGVkS2V5Uhpsb2NhbENvbnZlcnNhdGlvblJlY29yZEtleRJVCh5yZW1vdGVfY29udmVyc2F0aW'
+    '9uX3JlY29yZF9rZXkYAyABKAsyEC52ZWlsaWQuVHlwZWRLZXlSG3JlbW90ZUNvbnZlcnNhdGlv'
+    'blJlY29yZEtleQ==');
+
+@$core.Deprecated('Use groupChatDescriptor instead')
+const GroupChat$json = {
+  '1': 'GroupChat',
+  '2': [
+    {'1': 'settings', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.ChatSettings', '10': 'settings'},
+    {'1': 'local_conversation_record_key', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'localConversationRecordKey'},
+    {'1': 'remote_conversation_record_keys', '3': 3, '4': 3, '5': 11, '6': '.veilid.TypedKey', '10': 'remoteConversationRecordKeys'},
+  ],
+};
+
+/// Descriptor for `GroupChat`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List groupChatDescriptor = $convert.base64Decode(
+    'CglHcm91cENoYXQSNAoIc2V0dGluZ3MYASABKAsyGC52ZWlsaWRjaGF0LkNoYXRTZXR0aW5nc1'
+    'IIc2V0dGluZ3MSUwodbG9jYWxfY29udmVyc2F0aW9uX3JlY29yZF9rZXkYAiABKAsyEC52ZWls'
+    'aWQuVHlwZWRLZXlSGmxvY2FsQ29udmVyc2F0aW9uUmVjb3JkS2V5ElcKH3JlbW90ZV9jb252ZX'
+    'JzYXRpb25fcmVjb3JkX2tleXMYAyADKAsyEC52ZWlsaWQuVHlwZWRLZXlSHHJlbW90ZUNvbnZl'
+    'cnNhdGlvblJlY29yZEtleXM=');
+
+@$core.Deprecated('Use profileDescriptor instead')
+const Profile$json = {
+  '1': 'Profile',
+  '2': [
+    {'1': 'name', '3': 1, '4': 1, '5': 9, '10': 'name'},
+    {'1': 'pronouns', '3': 2, '4': 1, '5': 9, '10': 'pronouns'},
+    {'1': 'about', '3': 3, '4': 1, '5': 9, '10': 'about'},
+    {'1': 'status', '3': 4, '4': 1, '5': 9, '10': 'status'},
+    {'1': 'availability', '3': 5, '4': 1, '5': 14, '6': '.veilidchat.Availability', '10': 'availability'},
+    {'1': 'avatar', '3': 6, '4': 1, '5': 11, '6': '.veilid.TypedKey', '9': 0, '10': 'avatar', '17': true},
+  ],
+  '8': [
+    {'1': '_avatar'},
+  ],
+};
+
+/// Descriptor for `Profile`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List profileDescriptor = $convert.base64Decode(
+    'CgdQcm9maWxlEhIKBG5hbWUYASABKAlSBG5hbWUSGgoIcHJvbm91bnMYAiABKAlSCHByb25vdW'
+    '5zEhQKBWFib3V0GAMgASgJUgVhYm91dBIWCgZzdGF0dXMYBCABKAlSBnN0YXR1cxI8CgxhdmFp'
+    'bGFiaWxpdHkYBSABKA4yGC52ZWlsaWRjaGF0LkF2YWlsYWJpbGl0eVIMYXZhaWxhYmlsaXR5Ei'
+    '0KBmF2YXRhchgGIAEoCzIQLnZlaWxpZC5UeXBlZEtleUgAUgZhdmF0YXKIAQFCCQoHX2F2YXRh'
+    'cg==');
+
+@$core.Deprecated('Use accountDescriptor instead')
+const Account$json = {
+  '1': 'Account',
+  '2': [
+    {'1': 'profile', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.Profile', '10': 'profile'},
+    {'1': 'invisible', '3': 2, '4': 1, '5': 8, '10': 'invisible'},
+    {'1': 'auto_away_timeout_sec', '3': 3, '4': 1, '5': 13, '10': 'autoAwayTimeoutSec'},
+    {'1': 'contact_list', '3': 4, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'contactList'},
+    {'1': 'contact_invitation_records', '3': 5, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'contactInvitationRecords'},
+    {'1': 'chat_list', '3': 6, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'chatList'},
+    {'1': 'group_chat_list', '3': 7, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'groupChatList'},
+  ],
+};
+
+/// Descriptor for `Account`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List accountDescriptor = $convert.base64Decode(
+    'CgdBY2NvdW50Ei0KB3Byb2ZpbGUYASABKAsyEy52ZWlsaWRjaGF0LlByb2ZpbGVSB3Byb2ZpbG'
+    'USHAoJaW52aXNpYmxlGAIgASgIUglpbnZpc2libGUSMQoVYXV0b19hd2F5X3RpbWVvdXRfc2Vj'
+    'GAMgASgNUhJhdXRvQXdheVRpbWVvdXRTZWMSPQoMY29udGFjdF9saXN0GAQgASgLMhouZGh0Lk'
+    '93bmVkREhUUmVjb3JkUG9pbnRlclILY29udGFjdExpc3QSWAoaY29udGFjdF9pbnZpdGF0aW9u'
+    'X3JlY29yZHMYBSABKAsyGi5kaHQuT3duZWRESFRSZWNvcmRQb2ludGVyUhhjb250YWN0SW52aX'
+    'RhdGlvblJlY29yZHMSNwoJY2hhdF9saXN0GAYgASgLMhouZGh0Lk93bmVkREhUUmVjb3JkUG9p'
+    'bnRlclIIY2hhdExpc3QSQgoPZ3JvdXBfY2hhdF9saXN0GAcgASgLMhouZGh0Lk93bmVkREhUUm'
+    'Vjb3JkUG9pbnRlclINZ3JvdXBDaGF0TGlzdA==');
+
 @$core.Deprecated('Use contactDescriptor instead')
 const Contact$json = {
   '1': 'Contact',
@@ -158,68 +431,6 @@ final $typed_data.Uint8List contactDescriptor = $convert.base64Decode(
     'lSGmxvY2FsQ29udmVyc2F0aW9uUmVjb3JkS2V5EisKEXNob3dfYXZhaWxhYmlsaXR5GAcgASgI'
     'UhBzaG93QXZhaWxhYmlsaXR5');
 
-@$core.Deprecated('Use profileDescriptor instead')
-const Profile$json = {
-  '1': 'Profile',
-  '2': [
-    {'1': 'name', '3': 1, '4': 1, '5': 9, '10': 'name'},
-    {'1': 'pronouns', '3': 2, '4': 1, '5': 9, '10': 'pronouns'},
-    {'1': 'status', '3': 3, '4': 1, '5': 9, '10': 'status'},
-    {'1': 'availability', '3': 4, '4': 1, '5': 14, '6': '.veilidchat.Availability', '10': 'availability'},
-    {'1': 'avatar', '3': 5, '4': 1, '5': 11, '6': '.veilid.TypedKey', '9': 0, '10': 'avatar', '17': true},
-  ],
-  '8': [
-    {'1': '_avatar'},
-  ],
-};
-
-/// Descriptor for `Profile`. Decode as a `google.protobuf.DescriptorProto`.
-final $typed_data.Uint8List profileDescriptor = $convert.base64Decode(
-    'CgdQcm9maWxlEhIKBG5hbWUYASABKAlSBG5hbWUSGgoIcHJvbm91bnMYAiABKAlSCHByb25vdW'
-    '5zEhYKBnN0YXR1cxgDIAEoCVIGc3RhdHVzEjwKDGF2YWlsYWJpbGl0eRgEIAEoDjIYLnZlaWxp'
-    'ZGNoYXQuQXZhaWxhYmlsaXR5UgxhdmFpbGFiaWxpdHkSLQoGYXZhdGFyGAUgASgLMhAudmVpbG'
-    'lkLlR5cGVkS2V5SABSBmF2YXRhcogBAUIJCgdfYXZhdGFy');
-
-@$core.Deprecated('Use chatDescriptor instead')
-const Chat$json = {
-  '1': 'Chat',
-  '2': [
-    {'1': 'type', '3': 1, '4': 1, '5': 14, '6': '.veilidchat.ChatType', '10': 'type'},
-    {'1': 'remote_conversation_record_key', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'remoteConversationRecordKey'},
-    {'1': 'reconciled_chat_record', '3': 3, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'reconciledChatRecord'},
-  ],
-};
-
-/// Descriptor for `Chat`. Decode as a `google.protobuf.DescriptorProto`.
-final $typed_data.Uint8List chatDescriptor = $convert.base64Decode(
-    'CgRDaGF0EigKBHR5cGUYASABKA4yFC52ZWlsaWRjaGF0LkNoYXRUeXBlUgR0eXBlElUKHnJlbW'
-    '90ZV9jb252ZXJzYXRpb25fcmVjb3JkX2tleRgCIAEoCzIQLnZlaWxpZC5UeXBlZEtleVIbcmVt'
-    'b3RlQ29udmVyc2F0aW9uUmVjb3JkS2V5ElAKFnJlY29uY2lsZWRfY2hhdF9yZWNvcmQYAyABKA'
-    'syGi5kaHQuT3duZWRESFRSZWNvcmRQb2ludGVyUhRyZWNvbmNpbGVkQ2hhdFJlY29yZA==');
-
-@$core.Deprecated('Use accountDescriptor instead')
-const Account$json = {
-  '1': 'Account',
-  '2': [
-    {'1': 'profile', '3': 1, '4': 1, '5': 11, '6': '.veilidchat.Profile', '10': 'profile'},
-    {'1': 'invisible', '3': 2, '4': 1, '5': 8, '10': 'invisible'},
-    {'1': 'auto_away_timeout_sec', '3': 3, '4': 1, '5': 13, '10': 'autoAwayTimeoutSec'},
-    {'1': 'contact_list', '3': 4, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'contactList'},
-    {'1': 'contact_invitation_records', '3': 5, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'contactInvitationRecords'},
-    {'1': 'chat_list', '3': 6, '4': 1, '5': 11, '6': '.dht.OwnedDHTRecordPointer', '10': 'chatList'},
-  ],
-};
-
-/// Descriptor for `Account`. Decode as a `google.protobuf.DescriptorProto`.
-final $typed_data.Uint8List accountDescriptor = $convert.base64Decode(
-    'CgdBY2NvdW50Ei0KB3Byb2ZpbGUYASABKAsyEy52ZWlsaWRjaGF0LlByb2ZpbGVSB3Byb2ZpbG'
-    'USHAoJaW52aXNpYmxlGAIgASgIUglpbnZpc2libGUSMQoVYXV0b19hd2F5X3RpbWVvdXRfc2Vj'
-    'GAMgASgNUhJhdXRvQXdheVRpbWVvdXRTZWMSPQoMY29udGFjdF9saXN0GAQgASgLMhouZGh0Lk'
-    '93bmVkREhUUmVjb3JkUG9pbnRlclILY29udGFjdExpc3QSWAoaY29udGFjdF9pbnZpdGF0aW9u'
-    'X3JlY29yZHMYBSABKAsyGi5kaHQuT3duZWRESFRSZWNvcmRQb2ludGVyUhhjb250YWN0SW52aX'
-    'RhdGlvblJlY29yZHMSNwoJY2hhdF9saXN0GAYgASgLMhouZGh0Lk93bmVkREhUUmVjb3JkUG9p'
-    'bnRlclIIY2hhdExpc3Q=');
-
 @$core.Deprecated('Use contactInvitationDescriptor instead')
 const ContactInvitation$json = {
   '1': 'ContactInvitation',

lib/proto/veilidchat.proto

@@ -1,51 +1,234 @@
+////////////////////////////////////////////////////////////////////////////////////
+// VeilidChat Protocol Buffer Definitions
+//
+// * Timestamps are in microseconds (us) since epoch
+// * Durations are in microseconds (us)
+////////////////////////////////////////////////////////////////////////////////////
+
 syntax = "proto3";
 package veilidchat;
 
 import "veilid.proto";
 import "dht.proto";
 
-// AttachmentKind
-// Enumeration of well-known attachment types
-enum AttachmentKind {
-    ATTACHMENT_KIND_UNSPECIFIED = 0;
-    ATTACHMENT_KIND_FILE = 1;
-    ATTACHMENT_KIND_IMAGE = 2;
+////////////////////////////////////////////////////////////////////////////////////
+// Enumerations
+////////////////////////////////////////////////////////////////////////////////////
+
+// Contact availability
+enum Availability {
+    AVAILABILITY_UNSPECIFIED = 0;
+    AVAILABILITY_OFFLINE = 1;
+    AVAILABILITY_FREE = 2;
+    AVAILABILITY_BUSY = 3;
+    AVAILABILITY_AWAY = 4;
 }
 
+// Encryption used on secret keys
+enum EncryptionKeyType {
+    ENCRYPTION_KEY_TYPE_UNSPECIFIED = 0;
+    ENCRYPTION_KEY_TYPE_NONE = 1;
+    ENCRYPTION_KEY_TYPE_PIN = 2;
+    ENCRYPTION_KEY_TYPE_PASSWORD = 3;
+}
+
+// Scope of a chat
+enum Scope {
+    // Can read chats but not send messages
+    WATCHERS = 0;
+    // Can send messages subject to moderation
+    // If moderation is disabled, this is equivalent to WATCHERS
+    MODERATED = 1;
+    // Can send messages without moderation
+    TALKERS = 2;
+    // Can moderate messages sent my members if moderation is enabled
+    MODERATORS = 3;
+    // Can perform all actions
+    ADMINS = 4;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// Attachments
+////////////////////////////////////////////////////////////////////////////////////
+
 // A single attachment
 message Attachment {
-    // Type of the data
-    AttachmentKind kind = 1;
-    // MIME type of the data
-    string mime = 2;
-    // Title or filename
-    string name = 3;
-    // Pointer to the data content
-    dht.DataReference content = 4;
-    // Author signature over all attachment fields and content fields and bytes
-    veilid.Signature signature = 5;
+    oneof kind {
+        AttachmentMedia media = 1;
     }
+    // Author signature over all attachment fields and content fields and bytes
+    veilid.Signature signature = 2;
+}
+
+// A file, audio, image, or video attachment
+message AttachmentMedia {
+    // MIME type of the data
+    string mime = 1;
+    // Title or filename
+    string name = 2;
+    // Pointer to the data content
+    dht.DataReference content = 3;
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////
+// Chat room controls
+////////////////////////////////////////////////////////////////////////////////////
+
+// Permissions of a chat
+message Permissions {
+    // Parties in this scope or higher can add members to their own group or lower
+    Scope can_add_members = 1;
+    // Parties in this scope or higher can change the 'info' of a group
+    Scope can_edit_info = 2;
+    // If moderation is enabled or not.
+    bool moderated = 3;
+}
+
+// The membership of a chat
+message Membership {
+    // Conversation keys for parties in the 'watchers' group
+    repeated veilid.TypedKey watchers = 1;
+    // Conversation keys for parties in the 'moderated' group
+    repeated veilid.TypedKey moderated = 2;
+    // Conversation keys for parties in the 'talkers' group
+    repeated veilid.TypedKey talkers = 3;
+    // Conversation keys for parties in the 'moderators' group
+    repeated veilid.TypedKey moderators = 4;
+    // Conversation keys for parties in the 'admins' group
+    repeated veilid.TypedKey admins = 5;
+}
+
+// The chat settings
+message ChatSettings {
+    // Title for the chat 
+    string title = 1;
+    // Description for the chat 
+    string description = 2;
+    // Icon for the chat 
+    optional dht.DataReference icon = 3;
+    // Default message expiration duration (in us)
+    uint64 default_expiration = 4;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// Messages
+////////////////////////////////////////////////////////////////////////////////////
 
 // A single message as part of a series of messages
 message Message {
-    // Author of the message
-    veilid.TypedKey author = 1;
-    // Time the message was sent (us since epoch)
-    uint64 timestamp = 2;
+
+    // A text message
+    message Text {
         // Text of the message
-    string text = 3;
-    // Author signature over all of the fields and attachment signatures
-    veilid.Signature signature = 4;
+        string text = 1;
+        // Topic of the message / Content warning
+        optional string topic = 2;
+        // Message id replied to (author id + message id)
+        optional bytes reply_id = 3;
+        // Message expiration timestamp
+        uint64 expiration = 4;
+        // Message view limit before deletion
+        uint32 view_limit = 5;
         // Attachments on the message
-    repeated Attachment attachments = 5;
+        repeated Attachment attachments = 6;
     }
     
+    // A secret message
+    message Secret {
+        // Text message protobuf encrypted by a key
+        bytes ciphertext = 1;
+        // Secret expiration timestamp
+        // This is the time after which an un-revealed secret will get deleted
+        uint64 expiration = 2;
+    }
+
+    // A 'delete' control message
+    // Deletes a set of messages by their ids
+    message ControlDelete {
+        repeated bytes ids = 1;
+    } 
+    // An 'erase' control message
+    // Deletes a set of messages from before some timestamp
+    message ControlErase {
+        // The latest timestamp to delete messages before
+        // If this is zero then all messages are cleared
+        uint64 timestamp = 1;
+    }
+    // A 'change settings' control message
+    message ControlSettings {
+        ChatSettings settings = 1;
+    }
+
+    // A 'change permissions' control message
+    // Changes the permissions of a chat
+    message ControlPermissions {
+        Permissions permissions = 1;
+    }
+
+    // A 'change membership' control message
+    // Changes the 
+    message ControlMembership {
+        Membership membership = 1;
+    }
+
+    // A 'moderation' control message
+    // Accepts or rejects a set of messages
+    message ControlModeration {
+        repeated bytes accepted_ids = 1;
+        repeated bytes rejected_ids = 2;
+    }
+
+    // A 'read receipt' control message
+    message ControlReadReceipt {
+        repeated bytes read_ids = 1;
+    }
+
+    //////////////////////////////////////////////////////////////////////////
+
+    // Unique id for this author stream
+    // Calculated from the hash of the previous message from this author
+    bytes id = 1;
+    // Author of the message (identity public key)
+    veilid.TypedKey author = 2;
+    // Time the message was sent according to sender
+    uint64 timestamp = 3;
+
+    // Message kind
+    oneof kind {
+        Text text = 4;
+        Secret secret = 5;
+        ControlDelete delete = 6;
+        ControlErase erase = 7;
+        ControlSettings settings = 8;
+        ControlPermissions permissions = 9;
+        ControlMembership membership = 10;
+        ControlModeration moderation = 11;
+    }
+    
+    // Author signature over all of the fields and attachment signatures
+    veilid.Signature signature = 12;
+}
+
+// Locally stored messages for chats
+message ReconciledMessage {
+    // The message as sent
+    Message content = 1;
+    // The timestamp the message was reconciled
+    uint64 reconciled_time = 2;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// Chats
+////////////////////////////////////////////////////////////////////////////////////
+
 // The means of direct communications that is synchronized between
 // two users. Visible and encrypted for the other party.
 // Includes communications for:
 //  * Profile changes
 //  * Identity changes
 //  * 1-1 chat messages
+//  * Group chat messages
 //
 // DHT Schema: SMPL(0,1,[identityPublicKey])
 // DHT Key (UnicastOutbox): localConversation
@@ -54,12 +237,84 @@ message Message {
 message Conversation {
     // Profile to publish to friend
     Profile profile = 1;
-    // Identity master (JSON) to publish to friend
+    // Identity master (JSON) to publish to friend or chat room
     string identity_master_json = 2;
-    // Messages DHTLog (xxx for now DHTShortArray)
+    // Messages DHTLog
     veilid.TypedKey messages = 3;
 }
 
+// Either a 1-1 conversation or a group chat
+// Privately encrypted, this is the local user's copy of the chat
+message Chat {
+    // Settings
+    ChatSettings settings = 1;
+    // Conversation key for this user
+    veilid.TypedKey local_conversation_record_key = 2;
+    // Conversation key for the other party
+    veilid.TypedKey remote_conversation_record_key = 3;
+}
+
+// A group chat
+// Privately encrypted, this is the local user's copy of the chat
+message GroupChat {
+    // Settings
+    ChatSettings settings = 1;
+    // Conversation key for this user
+    veilid.TypedKey local_conversation_record_key = 2;
+    // Conversation keys for the other parties
+    repeated veilid.TypedKey remote_conversation_record_keys = 3;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// Accounts
+////////////////////////////////////////////////////////////////////////////////////
+
+// Publicly shared profile information for both contacts and accounts
+// Contains:
+// Name - Friendly name
+// Pronouns - Pronouns of user
+// Icon - Little picture to represent user in contact list
+message Profile  {
+    // Friendy name
+    string name = 1;
+    // Pronouns of user
+    string pronouns = 2;
+    // Description of the user
+    string about = 3;
+    // Status/away message
+    string status = 4;
+    // Availability
+    Availability availability = 5;
+    // Avatar DHTData
+    optional veilid.TypedKey avatar = 6;
+}
+
+// A record of an individual account
+// Pointed to by the identity account map in the identity key
+//
+// DHT Schema: DFLT(1)
+// DHT Private: accountSecretKey
+message Account {
+    // The user's profile that gets shared with contacts
+    Profile profile = 1;
+    // Invisibility makes you always look 'Offline'
+    bool invisible = 2;
+    // Auto-away sets 'away' mode after an inactivity time
+    uint32 auto_away_timeout_sec = 3;
+    // The contacts DHTList for this account
+    // DHT Private
+    dht.OwnedDHTRecordPointer contact_list = 4;
+    // The ContactInvitationRecord DHTShortArray for this account
+    // DHT Private
+    dht.OwnedDHTRecordPointer contact_invitation_records = 5;
+    // The Chats DHTList for this account
+    // DHT Private
+    dht.OwnedDHTRecordPointer chat_list = 6;
+    // The GroupChats DHTList for this account
+    // DHT Private
+    dht.OwnedDHTRecordPointer group_chat_list = 7;
+}
+
 // A record of a contact that has accepted a contact invitation
 // Contains a copy of the most recent remote profile as well as
 // a locally edited profile.
@@ -80,87 +335,13 @@ message Contact {
     veilid.TypedKey remote_conversation_record_key = 5;
     // Our conversation key for friend to sync
     veilid.TypedKey local_conversation_record_key = 6;
-    // Show availability
+    // Show availability to this contact
     bool show_availability = 7;
 }
 
-// Contact availability
-enum Availability {
-    AVAILABILITY_UNSPECIFIED = 0;
-    AVAILABILITY_OFFLINE = 1;
-    AVAILABILITY_FREE = 2;
-    AVAILABILITY_BUSY = 3;
-    AVAILABILITY_AWAY = 4;
-}
-
-// Publicly shared profile information for both contacts and accounts
-// Contains:
-// Name - Friendly name
-// Pronouns - Pronouns of user
-// Icon - Little picture to represent user in contact list
-message Profile  {
-    // Friendy name
-    string name = 1;
-    // Pronouns of user
-    string pronouns = 2;
-    // Status/away message
-    string status = 3;
-    // Availability
-    Availability availability = 4;
-    // Avatar DHTData
-    optional veilid.TypedKey avatar = 5;
-}
-
-
-enum ChatType {
-    CHAT_TYPE_UNSPECIFIED = 0;
-    SINGLE_CONTACT = 1;
-    GROUP = 2;
-}
-
-// Either a 1-1 conversation or a group chat (eventually)
-// Privately encrypted, this is the local user's copy of the chat
-message Chat {
-    // What kind of chat is this
-    ChatType type = 1;
-    // Conversation key for the other party
-    veilid.TypedKey remote_conversation_record_key = 2;
-    // Reconciled chat record DHTLog (xxx for now DHTShortArray)
-    dht.OwnedDHTRecordPointer reconciled_chat_record = 3;    
-}
-
-// A record of an individual account
-// Pointed to by the identity account map in the identity key
-//
-// DHT Schema: DFLT(1)
-// DHT Private: accountSecretKey
-message Account {
-    // The user's profile that gets shared with contacts
-    Profile profile = 1;
-    // Invisibility makes you always look 'Offline'
-    bool invisible = 2;
-    // Auto-away sets 'away' mode after an inactivity time
-    uint32 auto_away_timeout_sec = 3;
-    // The contacts DHTList for this account
-    // DHT Private
-    dht.OwnedDHTRecordPointer contact_list = 4;
-    // The ContactInvitationRecord DHTShortArray for this account
-    // DHT Private
-    dht.OwnedDHTRecordPointer contact_invitation_records = 5;
-    // The chats DHTList for this account
-    // DHT Private
-    dht.OwnedDHTRecordPointer chat_list = 6;
-    
-}
-
-// EncryptionKeyType
-// Encryption of secret
-enum EncryptionKeyType {
-    ENCRYPTION_KEY_TYPE_UNSPECIFIED = 0;
-    ENCRYPTION_KEY_TYPE_NONE = 1;
-    ENCRYPTION_KEY_TYPE_PIN = 2;
-    ENCRYPTION_KEY_TYPE_PASSWORD = 3;
-}
+////////////////////////////////////////////////////////////////////////////////////
+// Invitations
+////////////////////////////////////////////////////////////////////////////////////
 
 // Invitation that is shared for VeilidChat contact connections
 // serialized to QR code or data blob, not send over DHT, out of band.

lib/theme/models/radix_generator.dart

@@ -609,6 +609,29 @@ ThemeData radixGenerator(Brightness brightness, RadixThemeColor themeColor) {
   final themeData = ThemeData.from(
       colorScheme: colorScheme, textTheme: textTheme, useMaterial3: true);
   return themeData.copyWith(
+      scrollbarTheme: themeData.scrollbarTheme.copyWith(
+          thumbColor: WidgetStateProperty.resolveWith((states) {
+        if (states.contains(WidgetState.pressed)) {
+          return scaleScheme.primaryScale.border;
+        } else if (states.contains(WidgetState.hovered)) {
+          return scaleScheme.primaryScale.hoverBorder;
+        }
+        return scaleScheme.primaryScale.subtleBorder;
+      }), trackColor: WidgetStateProperty.resolveWith((states) {
+        if (states.contains(WidgetState.pressed)) {
+          return scaleScheme.primaryScale.activeElementBackground;
+        } else if (states.contains(WidgetState.hovered)) {
+          return scaleScheme.primaryScale.hoverElementBackground;
+        }
+        return scaleScheme.primaryScale.elementBackground;
+      }), trackBorderColor: WidgetStateProperty.resolveWith((states) {
+        if (states.contains(WidgetState.pressed)) {
+          return scaleScheme.primaryScale.subtleBorder;
+        } else if (states.contains(WidgetState.hovered)) {
+          return scaleScheme.primaryScale.subtleBorder;
+        }
+        return scaleScheme.primaryScale.subtleBorder;
+      })),
       bottomSheetTheme: themeData.bottomSheetTheme.copyWith(
           elevation: 0,
           modalElevation: 0,

lib/theme/views/widget_helpers.dart

@@ -44,7 +44,7 @@ Widget waitingPage({String? text}) => Builder(builder: (context) {
       final theme = Theme.of(context);
       final scale = theme.extension<ScaleScheme>()!;
       return ColoredBox(
-          color: scale.tertiaryScale.primaryText,
+          color: scale.tertiaryScale.appBackground,
           child: Center(
               child: Column(children: [
             buildProgressIndicator().expanded(),

lib/tools/misc.dart

@@ -0,0 +1,18 @@
+extension StringExt on String {
+  (String, String?) splitOnce(Pattern p) {
+    final pos = indexOf(p);
+    if (pos == -1) {
+      return (this, null);
+    }
+    final rest = substring(pos);
+    var offset = 0;
+    while (true) {
+      final match = p.matchAsPrefix(rest, offset);
+      if (match == null) {
+        break;
+      }
+      offset = match.end;
+    }
+    return (substring(0, pos), rest.substring(offset));
+  }
+}

lib/tools/state_logger.dart

@@ -6,10 +6,13 @@ const Map<String, LogLevel> _blocChangeLogLevels = {
   'ConnectionStateCubit': LogLevel.off,
   'ActiveSingleContactChatBlocMapCubit': LogLevel.off,
   'ActiveConversationsBlocMapCubit': LogLevel.off,
-  'DHTShortArrayCubit<Message>': LogLevel.off,
   'PersistentQueueCubit<Message>': LogLevel.off,
+  'TableDBArrayProtobufCubit<ReconciledMessage>': LogLevel.off,
+  'DHTLogCubit<Message>': LogLevel.off,
   'SingleContactMessagesCubit': LogLevel.off,
+  'ChatComponentCubit': LogLevel.off,
 };
+
 const Map<String, LogLevel> _blocCreateCloseLogLevels = {};
 const Map<String, LogLevel> _blocErrorLogLevels = {};
 

lib/tools/tools.dart

@@ -2,6 +2,7 @@ export 'animations.dart';
 export 'enter_password.dart';
 export 'enter_pin.dart';
 export 'loggy.dart';
+export 'misc.dart';
 export 'phono_byte.dart';
 export 'pop_control.dart';
 export 'responsive.dart';

packages/veilid_support/example/integration_test/app_test.dart

@@ -1,12 +1,14 @@
 import 'package:flutter/foundation.dart';
 import 'package:integration_test/integration_test.dart';
 import 'package:test/test.dart';
+import 'package:veilid_support/veilid_support.dart';
 import 'package:veilid_test/veilid_test.dart';
 
 import 'fixtures/fixtures.dart';
 import 'test_dht_log.dart';
 import 'test_dht_record_pool.dart';
 import 'test_dht_short_array.dart';
+import 'test_table_db_array.dart';
 
 void main() {
   final startTime = DateTime.now();
@@ -34,6 +36,117 @@ void main() {
       setUpAll(veilidFixture.attach);
       tearDownAll(veilidFixture.detach);
 
+      group('TableDB Tests', () {
+        group('TableDBArray Tests', () {
+          // test('create/delete TableDBArray', testTableDBArrayCreateDelete);
+
+          group('TableDBArray Add/Get Tests', () {
+            for (final params in [
+              //
+              (99, 3, 15),
+              (100, 4, 16),
+              (101, 5, 17),
+              //
+              (511, 3, 127),
+              (512, 4, 128),
+              (513, 5, 129),
+              //
+              (4095, 3, 1023),
+              (4096, 4, 1024),
+              (4097, 5, 1025),
+              //
+              (65535, 3, 16383),
+              (65536, 4, 16384),
+              (65537, 5, 16385),
+            ]) {
+              final count = params.$1;
+              final singles = params.$2;
+              final batchSize = params.$3;
+
+              test(
+                timeout: const Timeout(Duration(seconds: 480)),
+                'add/remove TableDBArray count = $count batchSize=$batchSize',
+                makeTestTableDBArrayAddGetClear(
+                    count: count,
+                    singles: singles,
+                    batchSize: batchSize,
+                    crypto: const VeilidCryptoPublic()),
+              );
+            }
+          });
+
+          group('TableDBArray Insert Tests', () {
+            for (final params in [
+              //
+              (99, 3, 15),
+              (100, 4, 16),
+              (101, 5, 17),
+              //
+              (511, 3, 127),
+              (512, 4, 128),
+              (513, 5, 129),
+              //
+              (4095, 3, 1023),
+              (4096, 4, 1024),
+              (4097, 5, 1025),
+              //
+              (65535, 3, 16383),
+              (65536, 4, 16384),
+              (65537, 5, 16385),
+            ]) {
+              final count = params.$1;
+              final singles = params.$2;
+              final batchSize = params.$3;
+
+              test(
+                timeout: const Timeout(Duration(seconds: 480)),
+                'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
+                makeTestTableDBArrayInsert(
+                    count: count,
+                    singles: singles,
+                    batchSize: batchSize,
+                    crypto: const VeilidCryptoPublic()),
+              );
+            }
+          });
+
+          group('TableDBArray Remove Tests', () {
+            for (final params in [
+              //
+              (99, 3, 15),
+              (100, 4, 16),
+              (101, 5, 17),
+              //
+              (511, 3, 127),
+              (512, 4, 128),
+              (513, 5, 129),
+              //
+              (4095, 3, 1023),
+              (4096, 4, 1024),
+              (4097, 5, 1025),
+              //
+              (16383, 3, 4095),
+              (16384, 4, 4096),
+              (16385, 5, 4097),
+            ]) {
+              final count = params.$1;
+              final singles = params.$2;
+              final batchSize = params.$3;
+
+              test(
+                timeout: const Timeout(Duration(seconds: 480)),
+                'remove TableDBArray count=$count singles=$singles batchSize=$batchSize',
+                makeTestTableDBArrayRemove(
+                    count: count,
+                    singles: singles,
+                    batchSize: batchSize,
+                    crypto: const VeilidCryptoPublic()),
+              );
+            }
+          });
+        });
+      });
+
       group('DHT Support Tests', () {
         setUpAll(updateProcessorFixture.setUp);
         setUpAll(tickerFixture.setUp);

packages/veilid_support/example/integration_test/test_dht_log.dart

@@ -64,8 +64,7 @@ Future<void> Function() makeTestDHTLogAddTruncate({required int stride}) =>
           const chunk = 25;
           for (var n = 0; n < dataset.length; n += chunk) {
             print('$n-${n + chunk - 1} ');
-            final success =
-                await w.tryAppendItems(dataset.sublist(n, n + chunk));
+            final success = await w.tryAddAll(dataset.sublist(n, n + chunk));
             expect(success, isTrue);
           }
         });
@@ -74,40 +73,40 @@ Future<void> Function() makeTestDHTLogAddTruncate({required int stride}) =>
 
       print('get all\n');
       {
-        final dataset2 = await dlog.operate((r) async => r.getItemRange(0));
+        final dataset2 = await dlog.operate((r) async => r.getRange(0));
         expect(dataset2, equals(dataset));
       }
       {
         final dataset3 =
-            await dlog.operate((r) async => r.getItemRange(64, length: 128));
+            await dlog.operate((r) async => r.getRange(64, length: 128));
         expect(dataset3, equals(dataset.sublist(64, 64 + 128)));
       }
       {
         final dataset4 =
-            await dlog.operate((r) async => r.getItemRange(0, length: 1000));
+            await dlog.operate((r) async => r.getRange(0, length: 1000));
         expect(dataset4, equals(dataset.sublist(0, 1000)));
       }
       {
         final dataset5 =
-            await dlog.operate((r) async => r.getItemRange(500, length: 499));
+            await dlog.operate((r) async => r.getRange(500, length: 499));
         expect(dataset5, equals(dataset.sublist(500, 999)));
       }
       print('truncate\n');
       {
-        await dlog.operateAppend((w) async => w.truncate(5));
+        await dlog.operateAppend((w) async => w.truncate(w.length - 5));
       }
       {
-        final dataset6 = await dlog
-            .operate((r) async => r.getItemRange(500 - 5, length: 499));
+        final dataset6 =
+            await dlog.operate((r) async => r.getRange(500 - 5, length: 499));
         expect(dataset6, equals(dataset.sublist(500, 999)));
       }
       print('truncate 2\n');
       {
-        await dlog.operateAppend((w) async => w.truncate(251));
+        await dlog.operateAppend((w) async => w.truncate(w.length - 251));
       }
       {
-        final dataset7 = await dlog
-            .operate((r) async => r.getItemRange(500 - 256, length: 499));
+        final dataset7 =
+            await dlog.operate((r) async => r.getRange(500 - 256, length: 499));
         expect(dataset7, equals(dataset.sublist(500, 999)));
       }
       print('clear\n');
@@ -116,7 +115,7 @@ Future<void> Function() makeTestDHTLogAddTruncate({required int stride}) =>
       }
       print('get all\n');
       {
-        final dataset8 = await dlog.operate((r) async => r.getItemRange(0));
+        final dataset8 = await dlog.operate((r) async => r.getRange(0));
         expect(dataset8, isEmpty);
       }
       print('delete and close\n');

packages/veilid_support/example/integration_test/test_dht_short_array.dart

@@ -64,7 +64,7 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
         final res = await arr.operateWrite((w) async {
           for (var n = 4; n < 8; n++) {
             print('$n ');
-            final success = await w.tryAddItem(dataset[n]);
+            final success = await w.tryAdd(dataset[n]);
             expect(success, isTrue);
           }
         });
@@ -75,8 +75,8 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
       {
         final res = await arr.operateWrite((w) async {
           print('${dataset.length ~/ 2}-${dataset.length}');
-          final success = await w.tryAddItems(
-              dataset.sublist(dataset.length ~/ 2, dataset.length));
+          final success = await w
+              .tryAddAll(dataset.sublist(dataset.length ~/ 2, dataset.length));
           expect(success, isTrue);
         });
         expect(res, isNull);
@@ -87,7 +87,7 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
         final res = await arr.operateWrite((w) async {
           for (var n = 0; n < 4; n++) {
             print('$n ');
-            final success = await w.tryInsertItem(n, dataset[n]);
+            final success = await w.tryInsert(n, dataset[n]);
             expect(success, isTrue);
           }
         });
@@ -98,8 +98,8 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
       {
         final res = await arr.operateWrite((w) async {
           print('8-${dataset.length ~/ 2}');
-          final success = await w.tryInsertItems(
-              8, dataset.sublist(8, dataset.length ~/ 2));
+          final success =
+              await w.tryInsertAll(8, dataset.sublist(8, dataset.length ~/ 2));
           expect(success, isTrue);
         });
         expect(res, isNull);
@@ -107,12 +107,12 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
 
       //print('get all\n');
       {
-        final dataset2 = await arr.operate((r) async => r.getItemRange(0));
+        final dataset2 = await arr.operate((r) async => r.getRange(0));
         expect(dataset2, equals(dataset));
       }
       {
         final dataset3 =
-            await arr.operate((r) async => r.getItemRange(64, length: 128));
+            await arr.operate((r) async => r.getRange(64, length: 128));
         expect(dataset3, equals(dataset.sublist(64, 64 + 128)));
       }
 
@@ -126,7 +126,7 @@ Future<void> Function() makeTestDHTShortArrayAdd({required int stride}) =>
 
       //print('get all\n');
       {
-        final dataset4 = await arr.operate((r) async => r.getItemRange(0));
+        final dataset4 = await arr.operate((r) async => r.getRange(0));
         expect(dataset4, isEmpty);
       }
 

packages/veilid_support/example/integration_test/test_table_db_array.dart

@@ -0,0 +1,250 @@
+import 'dart:convert';
+import 'dart:math';
+import 'dart:typed_data';
+
+import 'package:test/test.dart';
+import 'package:veilid_support/veilid_support.dart';
+
+Future<void> testTableDBArrayCreateDelete() async {
+  // Close before delete
+  {
+    final arr =
+        TableDBArray(table: 'testArray', crypto: const VeilidCryptoPublic());
+    expect(() => arr.length, throwsA(isA<StateError>()));
+    expect(arr.isOpen, isTrue);
+    await arr.initWait();
+    expect(arr.isOpen, isTrue);
+    expect(arr.length, isZero);
+    await arr.close();
+    expect(arr.isOpen, isFalse);
+    await arr.delete();
+    expect(arr.isOpen, isFalse);
+  }
+
+  // Async create with close after delete and then reopen
+  {
+    final arr = await TableDBArray.make(
+        table: 'testArray', crypto: const VeilidCryptoPublic());
+    expect(arr.length, isZero);
+    expect(arr.isOpen, isTrue);
+    await expectLater(() async {
+      await arr.delete();
+    }, throwsA(isA<StateError>()));
+    expect(arr.isOpen, isTrue);
+    await arr.close();
+    expect(arr.isOpen, isFalse);
+
+    final arr2 = await TableDBArray.make(
+        table: 'testArray', crypto: const VeilidCryptoPublic());
+    expect(arr2.isOpen, isTrue);
+    expect(arr.isOpen, isFalse);
+    await arr2.close();
+    expect(arr2.isOpen, isFalse);
+    await arr2.delete();
+  }
+}
+
+Uint8List makeData(int n) => utf8.encode('elem $n');
+List<Uint8List> makeDataBatch(int n, int batchSize) =>
+    List.generate(batchSize, (x) => makeData(n + x));
+
+Future<void> Function() makeTestTableDBArrayAddGetClear(
+        {required int count,
+        required int singles,
+        required int batchSize,
+        required VeilidCrypto crypto}) =>
+    () async {
+      final arr = await TableDBArray.make(table: 'testArray', crypto: crypto);
+
+      print('adding');
+      {
+        for (var n = 0; n < count;) {
+          var toAdd = min(batchSize, count - n);
+          for (var s = 0; s < min(singles, toAdd); s++) {
+            await arr.add(makeData(n));
+            toAdd--;
+            n++;
+          }
+
+          await arr.addAll(makeDataBatch(n, toAdd));
+          n += toAdd;
+
+          print('  $n/$count');
+        }
+      }
+
+      print('get singles');
+      {
+        for (var n = 0; n < batchSize; n++) {
+          expect(await arr.get(n), equals(makeData(n)));
+        }
+      }
+
+      print('get batch');
+      {
+        for (var n = batchSize; n < count; n += batchSize) {
+          final toGet = min(batchSize, count - n);
+          expect(await arr.getRange(n, n + toGet),
+              equals(makeDataBatch(n, toGet)));
+        }
+      }
+
+      print('clear');
+      {
+        await arr.clear();
+        expect(arr.length, isZero);
+      }
+
+      await arr.close(delete: true);
+    };
+
+Future<void> Function() makeTestTableDBArrayInsert(
+        {required int count,
+        required int singles,
+        required int batchSize,
+        required VeilidCrypto crypto}) =>
+    () async {
+      final arr = await TableDBArray.make(table: 'testArray', crypto: crypto);
+
+      final match = <Uint8List>[];
+
+      print('inserting');
+      {
+        for (var n = 0; n < count;) {
+          final start = n;
+          var toAdd = min(batchSize, count - n);
+          for (var s = 0; s < min(singles, toAdd); s++) {
+            final data = makeData(n);
+            await arr.insert(start, data);
+            match.insert(start, data);
+            toAdd--;
+            n++;
+          }
+
+          final data = makeDataBatch(n, toAdd);
+          await arr.insertAll(start, data);
+          match.insertAll(start, data);
+          n += toAdd;
+
+          print('  $n/$count');
+        }
+      }
+
+      print('get singles');
+      {
+        for (var n = 0; n < batchSize; n++) {
+          expect(await arr.get(n), equals(match[n]));
+        }
+      }
+
+      print('get batch');
+      {
+        for (var n = batchSize; n < count; n += batchSize) {
+          final toGet = min(batchSize, count - n);
+          expect(await arr.getRange(n, n + toGet),
+              equals(match.sublist(n, n + toGet)));
+        }
+      }
+
+      print('clear');
+      {
+        await arr.clear();
+        expect(arr.length, isZero);
+      }
+
+      await arr.close(delete: true);
+    };
+
+Future<void> Function() makeTestTableDBArrayRemove(
+        {required int count,
+        required int singles,
+        required int batchSize,
+        required VeilidCrypto crypto}) =>
+    () async {
+      final arr = await TableDBArray.make(table: 'testArray', crypto: crypto);
+
+      final match = <Uint8List>[];
+
+      {
+        final rems = [
+          (0, 0),
+          (0, 1),
+          (0, batchSize),
+          (1, batchSize - 1),
+          (batchSize, 1),
+          (batchSize + 1, batchSize),
+          (batchSize - 1, batchSize + 1)
+        ];
+        for (final rem in rems) {
+          print('adding ');
+          {
+            for (var n = match.length; n < count;) {
+              final toAdd = min(batchSize, count - n);
+              final data = makeDataBatch(n, toAdd);
+              await arr.addAll(data);
+              match.addAll(data);
+              n += toAdd;
+              print('  $n/$count');
+            }
+            expect(arr.length, equals(match.length));
+          }
+
+          {
+            final start = rem.$1;
+            final length = rem.$2;
+            print('removing start=$start length=$length');
+
+            final out = Output<List<Uint8List>>();
+            await arr.removeRange(start, start + length, out: out);
+            expect(out.value, equals(match.sublist(start, start + length)));
+            match.removeRange(start, start + length);
+            expect(arr.length, equals(match.length));
+
+            print('get batch');
+            {
+              final checkCount = match.length;
+              for (var n = 0; n < checkCount;) {
+                final toGet = min(batchSize, checkCount - n);
+                expect(await arr.getRange(n, n + toGet),
+                    equals(match.sublist(n, n + toGet)));
+                n += toGet;
+                print('  $n/$checkCount');
+              }
+            }
+          }
+
+          {
+            final start = match.length - rem.$1 - rem.$2;
+            final length = rem.$2;
+            print('removing from end start=$start length=$length');
+
+            final out = Output<List<Uint8List>>();
+            await arr.removeRange(start, start + length, out: out);
+            expect(out.value, equals(match.sublist(start, start + length)));
+            match.removeRange(start, start + length);
+            expect(arr.length, equals(match.length));
+
+            print('get batch');
+            {
+              final checkCount = match.length;
+              for (var n = 0; n < checkCount;) {
+                final toGet = min(batchSize, checkCount - n);
+                expect(await arr.getRange(n, n + toGet),
+                    equals(match.sublist(n, n + toGet)));
+                n += toGet;
+                print('  $n/$checkCount');
+              }
+              expect(arr.length, equals(match.length));
+            }
+          }
+        }
+      }
+
+      print('clear');
+      {
+        await arr.clear();
+        expect(arr.length, isZero);
+      }
+
+      await arr.close(delete: true);
+    };

packages/veilid_support/lib/dht_support/proto/dht.proto

@@ -62,13 +62,24 @@ message DHTShortArray {
     // calculated through iteration
 }
 
+// Reference to data on the DHT
+message DHTDataReference {
+    veilid.TypedKey dht_data = 1;
+    veilid.TypedKey hash = 2;
+}
+
+// Reference to data on the BlockStore
+message BlockStoreDataReference {
+    veilid.TypedKey block = 1;
+}
+
 // DataReference
 // Pointer to data somewhere in Veilid
 // Abstraction over DHTData and BlockStore
 message DataReference {
     oneof kind {
-        veilid.TypedKey dht_data = 1;
-        // TypedKey block = 2;
+        DHTDataReference dht_data = 1;
+        BlockStoreDataReference block_store_data = 2;
     }
 }
 

packages/veilid_support/lib/dht_support/src/dht_log/dht_log.dart

@@ -9,11 +9,11 @@ import 'package:meta/meta.dart';
 
 import '../../../veilid_support.dart';
 import '../../proto/proto.dart' as proto;
-import '../interfaces/dht_append_truncate.dart';
+import '../interfaces/dht_add.dart';
 
 part 'dht_log_spine.dart';
 part 'dht_log_read.dart';
-part 'dht_log_append.dart';
+part 'dht_log_write.dart';
 
 ///////////////////////////////////////////////////////////////////////
 
@@ -60,7 +60,7 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
       int stride = DHTShortArray.maxElements,
       VeilidRoutingContext? routingContext,
       TypedKey? parent,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       KeyPair? writer}) async {
     assert(stride <= DHTShortArray.maxElements, 'stride too long');
     final pool = DHTRecordPool.instance;
@@ -102,7 +102,7 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
       {required String debugName,
       VeilidRoutingContext? routingContext,
       TypedKey? parent,
-      DHTRecordCrypto? crypto}) async {
+      VeilidCrypto? crypto}) async {
     final spineRecord = await DHTRecordPool.instance.openRecordRead(
         logRecordKey,
         debugName: debugName,
@@ -125,7 +125,7 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
     required String debugName,
     VeilidRoutingContext? routingContext,
     TypedKey? parent,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) async {
     final spineRecord = await DHTRecordPool.instance.openRecordWrite(
         logRecordKey, writer,
@@ -148,7 +148,7 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
     required String debugName,
     required TypedKey parent,
     VeilidRoutingContext? routingContext,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) =>
       openWrite(
         ownedLogRecordPointer.recordKey,
@@ -209,7 +209,7 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
   OwnedDHTRecordPointer get recordPointer => _spine.recordPointer;
 
   /// Runs a closure allowing read-only access to the log
-  Future<T?> operate<T>(Future<T?> Function(DHTRandomRead) closure) async {
+  Future<T> operate<T>(Future<T> Function(DHTLogReadOperations) closure) async {
     if (!isOpen) {
       throw StateError('log is not open"');
     }
@@ -226,13 +226,13 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
   /// Throws DHTOperateException if the write could not be performed
   /// at this time
   Future<T> operateAppend<T>(
-      Future<T> Function(DHTAppendTruncateRandomRead) closure) async {
+      Future<T> Function(DHTLogWriteOperations) closure) async {
     if (!isOpen) {
       throw StateError('log is not open"');
     }
 
     return _spine.operateAppend((spine) async {
-      final writer = _DHTLogAppend._(spine);
+      final writer = _DHTLogWrite._(spine);
       return closure(writer);
     });
   }
@@ -244,14 +244,14 @@ class DHTLog implements DHTDeleteable<DHTLog, DHTLog> {
   /// succeeded, returning false will trigger another eventual consistency
   /// attempt.
   Future<void> operateAppendEventual(
-      Future<bool> Function(DHTAppendTruncateRandomRead) closure,
+      Future<bool> Function(DHTLogWriteOperations) closure,
       {Duration? timeout}) async {
     if (!isOpen) {
       throw StateError('log is not open"');
     }
 
     return _spine.operateAppendEventual((spine) async {
-      final writer = _DHTLogAppend._(spine);
+      final writer = _DHTLogWrite._(spine);
       return closure(writer);
     }, timeout: timeout);
   }

packages/veilid_support/lib/dht_support/src/dht_log/dht_log_append.dart

@@ -1,94 +0,0 @@
-part of 'dht_log.dart';
-
-////////////////////////////////////////////////////////////////////////////
-// Append/truncate implementation
-
-class _DHTLogAppend extends _DHTLogRead implements DHTAppendTruncateRandomRead {
-  _DHTLogAppend._(super.spine) : super._();
-
-  @override
-  Future<bool> tryAppendItem(Uint8List value) async {
-    // Allocate empty index at the end of the list
-    final insertPos = _spine.length;
-    _spine.allocateTail(1);
-    final lookup = await _spine.lookupPosition(insertPos);
-    if (lookup == null) {
-      throw StateError("can't write to dht log");
-    }
-
-    // Write item to the segment
-    return lookup.scope((sa) => sa.operateWrite((write) async {
-          // If this a new segment, then clear it in case we have wrapped around
-          if (lookup.pos == 0) {
-            await write.clear();
-          } else if (lookup.pos != write.length) {
-            // We should always be appending at the length
-            throw StateError('appending should be at the end');
-          }
-          return write.tryAddItem(value);
-        }));
-  }
-
-  @override
-  Future<bool> tryAppendItems(List<Uint8List> values) async {
-    // Allocate empty index at the end of the list
-    final insertPos = _spine.length;
-    _spine.allocateTail(values.length);
-
-    // Look up the first position and shortarray
-    final dws = DelayedWaitSet<void>();
-
-    var success = true;
-    for (var valueIdx = 0; valueIdx < values.length;) {
-      final remaining = values.length - valueIdx;
-
-      final lookup = await _spine.lookupPosition(insertPos + valueIdx);
-      if (lookup == null) {
-        throw StateError("can't write to dht log");
-      }
-
-      final sacount = min(remaining, DHTShortArray.maxElements - lookup.pos);
-      final sublistValues = values.sublist(valueIdx, valueIdx + sacount);
-
-      dws.add(() async {
-        final ok = await lookup.scope((sa) => sa.operateWrite((write) async {
-              // If this a new segment, then clear it in
-              // case we have wrapped around
-              if (lookup.pos == 0) {
-                await write.clear();
-              } else if (lookup.pos != write.length) {
-                // We should always be appending at the length
-                throw StateError('appending should be at the end');
-              }
-              return write.tryAddItems(sublistValues);
-            }));
-        if (!ok) {
-          success = false;
-        }
-      });
-
-      valueIdx += sacount;
-    }
-
-    await dws();
-
-    return success;
-  }
-
-  @override
-  Future<void> truncate(int count) async {
-    count = min(count, _spine.length);
-    if (count == 0) {
-      return;
-    }
-    if (count < 0) {
-      throw StateError('can not remove negative items');
-    }
-    await _spine.releaseHead(count);
-  }
-
-  @override
-  Future<void> clear() async {
-    await _spine.releaseHead(_spine.length);
-  }
-}

packages/veilid_support/lib/dht_support/src/dht_log/dht_log_cubit.dart

@@ -8,37 +8,29 @@ import 'package:fast_immutable_collections/fast_immutable_collections.dart';
 import 'package:meta/meta.dart';
 
 import '../../../veilid_support.dart';
-import '../interfaces/dht_append_truncate.dart';
-
-@immutable
-class DHTLogElementState<T> extends Equatable {
-  const DHTLogElementState({required this.value, required this.isOffline});
-  final T value;
-  final bool isOffline;
-
-  @override
-  List<Object?> get props => [value, isOffline];
-}
 
 @immutable
 class DHTLogStateData<T> extends Equatable {
   const DHTLogStateData(
-      {required this.elements,
-      required this.tail,
-      required this.count,
+      {required this.length,
+      required this.window,
+      required this.windowTail,
+      required this.windowSize,
       required this.follow});
-  // The view of the elements in the dhtlog
-  // Span is from [tail-length, tail)
-  final IList<DHTLogElementState<T>> elements;
-  // One past the end of the last element
-  final int tail;
-  // The total number of elements to try to keep in 'elements'
-  final int count;
-  // If we should have the tail following the log
+  // The total number of elements in the whole log
+  final int length;
+  // The view window of the elements in the dhtlog
+  // Span is from [tail - window.length, tail)
+  final IList<OnlineElementState<T>> window;
+  // The position of the view window, one past the last element
+  final int windowTail;
+  // The total number of elements to try to keep in the window
+  final int windowSize;
+  // If we have the window following the log
   final bool follow;
 
   @override
-  List<Object?> get props => [elements, tail, count, follow];
+  List<Object?> get props => [length, window, windowTail, windowSize, follow];
 }
 
 typedef DHTLogState<T> = AsyncValue<DHTLogStateData<T>>;
@@ -69,13 +61,16 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
   // If tail is positive, the position is absolute from the head of the log
   // If follow is enabled, the tail offset will update when the log changes
   Future<void> setWindow(
-      {int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
+      {int? windowTail,
+      int? windowSize,
+      bool? follow,
+      bool forceRefresh = false}) async {
     await _initWait();
-    if (tail != null) {
-      _tail = tail;
+    if (windowTail != null) {
+      _windowTail = windowTail;
     }
-    if (count != null) {
-      _count = count;
+    if (windowSize != null) {
+      _windowSize = windowSize;
     }
     if (follow != null) {
       _follow = follow;
@@ -93,7 +88,13 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
 
   Future<void> _refreshInner(void Function(AsyncValue<DHTLogStateData<T>>) emit,
       {bool forceRefresh = false}) async {
-    final avElements = await _loadElements(_tail, _count);
+    late final AsyncValue<IList<OnlineElementState<T>>> avElements;
+    late final int length;
+    await _log.operate((reader) async {
+      length = reader.length;
+      avElements =
+          await loadElementsFromReader(reader, _windowTail, _windowSize);
+    });
     final err = avElements.asError;
     if (err != null) {
       emit(AsyncValue.error(err.error, err.stackTrace));
@@ -104,30 +105,35 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
       emit(const AsyncValue.loading());
       return;
     }
-    final elements = avElements.asData!.value;
+    final window = avElements.asData!.value;
     emit(AsyncValue.data(DHTLogStateData(
-        elements: elements, tail: _tail, count: _count, follow: _follow)));
+        length: length,
+        window: window,
+        windowTail: _windowTail,
+        windowSize: _windowSize,
+        follow: _follow)));
   }
 
-  Future<AsyncValue<IList<DHTLogElementState<T>>>> _loadElements(
-      int tail, int count,
+  // Tail is one past the last element to load
+  Future<AsyncValue<IList<OnlineElementState<T>>>> loadElementsFromReader(
+      DHTLogReadOperations reader, int tail, int count,
       {bool forceRefresh = false}) async {
     try {
-      final allItems = await _log.operate((reader) async {
       final length = reader.length;
+      if (length == 0) {
+        return const AsyncValue.data(IList.empty());
+      }
       final end = ((tail - 1) % length) + 1;
       final start = (count < end) ? end - count : 0;
 
       final offlinePositions = await reader.getOfflinePositions();
-        final allItems = (await reader.getItemRange(start,
+      final allItems = (await reader.getRange(start,
               length: end - start, forceRefresh: forceRefresh))
           ?.indexed
-            .map((x) => DHTLogElementState(
+          .map((x) => OnlineElementState(
               value: _decodeElement(x.$2),
               isOffline: offlinePositions.contains(x.$1)))
           .toIList();
-        return allItems;
-      });
       if (allItems == null) {
         return const AsyncValue.loading();
       }
@@ -150,18 +156,18 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
     _sspUpdate.busyUpdate<T, DHTLogState<T>>(busy, (emit) async {
       // apply follow
       if (_follow) {
-        if (_tail <= 0) {
+        if (_windowTail <= 0) {
           // Negative tail is already following tail changes
         } else {
           // Positive tail is measured from the head, so apply deltas
-          _tail = (_tail + _tailDelta - _headDelta) % upd.length;
+          _windowTail = (_windowTail + _tailDelta - _headDelta) % upd.length;
         }
       } else {
-        if (_tail <= 0) {
+        if (_windowTail <= 0) {
           // Negative tail is following tail changes so apply deltas
-          var posTail = _tail + upd.length;
+          var posTail = _windowTail + upd.length;
           posTail = (posTail + _tailDelta - _headDelta) % upd.length;
-          _tail = posTail - upd.length;
+          _windowTail = posTail - upd.length;
         } else {
           // Positive tail is measured from head so not following tail
         }
@@ -184,19 +190,19 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
     await super.close();
   }
 
-  Future<R?> operate<R>(Future<R?> Function(DHTRandomRead) closure) async {
+  Future<R> operate<R>(Future<R> Function(DHTLogReadOperations) closure) async {
     await _initWait();
     return _log.operate(closure);
   }
 
   Future<R> operateAppend<R>(
-      Future<R> Function(DHTAppendTruncateRandomRead) closure) async {
+      Future<R> Function(DHTLogWriteOperations) closure) async {
     await _initWait();
     return _log.operateAppend(closure);
   }
 
   Future<void> operateAppendEventual(
-      Future<bool> Function(DHTAppendTruncateRandomRead) closure,
+      Future<bool> Function(DHTLogWriteOperations) closure,
       {Duration? timeout}) async {
     await _initWait();
     return _log.operateAppendEventual(closure, timeout: timeout);
@@ -214,7 +220,7 @@ class DHTLogCubit<T> extends Cubit<DHTLogBusyState<T>>
   var _tailDelta = 0;
 
   // Cubit window into the DHTLog
-  var _tail = 0;
-  var _count = DHTShortArray.maxElements;
+  var _windowTail = 0;
+  var _windowSize = DHTShortArray.maxElements;
   var _follow = true;
 }

packages/veilid_support/lib/dht_support/src/dht_log/dht_log_read.dart

@@ -3,14 +3,16 @@ part of 'dht_log.dart';
 ////////////////////////////////////////////////////////////////////////////
 // Reader-only implementation
 
-class _DHTLogRead implements DHTRandomRead {
+abstract class DHTLogReadOperations implements DHTRandomRead {}
+
+class _DHTLogRead implements DHTLogReadOperations {
   _DHTLogRead._(_DHTLogSpine spine) : _spine = spine;
 
   @override
   int get length => _spine.length;
 
   @override
-  Future<Uint8List?> getItem(int pos, {bool forceRefresh = false}) async {
+  Future<Uint8List?> get(int pos, {bool forceRefresh = false}) async {
     if (pos < 0 || pos >= length) {
       throw IndexError.withLength(pos, length);
     }
@@ -19,8 +21,8 @@ class _DHTLogRead implements DHTRandomRead {
       return null;
     }
 
-    return lookup.scope((sa) => sa.operate(
-        (read) => read.getItem(lookup.pos, forceRefresh: forceRefresh)));
+    return lookup.scope((sa) =>
+        sa.operate((read) => read.get(lookup.pos, forceRefresh: forceRefresh)));
   }
 
   (int, int) _clampStartLen(int start, int? len) {
@@ -38,14 +40,14 @@ class _DHTLogRead implements DHTRandomRead {
   }
 
   @override
-  Future<List<Uint8List>?> getItemRange(int start,
+  Future<List<Uint8List>?> getRange(int start,
       {int? length, bool forceRefresh = false}) async {
     final out = <Uint8List>[];
     (start, length) = _clampStartLen(start, length);
 
     final chunks = Iterable<int>.generate(length).slices(maxDHTConcurrency).map(
-        (chunk) => chunk
-            .map((pos) => getItem(pos + start, forceRefresh: forceRefresh)));
+        (chunk) =>
+            chunk.map((pos) => get(pos + start, forceRefresh: forceRefresh)));
 
     for (final chunk in chunks) {
       final elems = await chunk.wait;

packages/veilid_support/lib/dht_support/src/dht_log/dht_log_spine.dart

@@ -3,16 +3,15 @@ part of 'dht_log.dart';
 class _DHTLogPosition extends DHTCloseable<_DHTLogPosition, DHTShortArray> {
   _DHTLogPosition._({
     required _DHTLogSpine dhtLogSpine,
-    required DHTShortArray shortArray,
+    required this.shortArray,
     required this.pos,
     required int segmentNumber,
-  })  : _segmentShortArray = shortArray,
-        _dhtLogSpine = dhtLogSpine,
+  })  : _dhtLogSpine = dhtLogSpine,
         _segmentNumber = segmentNumber;
   final int pos;
 
   final _DHTLogSpine _dhtLogSpine;
-  final DHTShortArray _segmentShortArray;
+  final DHTShortArray shortArray;
   var _openCount = 1;
   final int _segmentNumber;
   final Mutex _mutex = Mutex();
@@ -23,7 +22,7 @@ class _DHTLogPosition extends DHTCloseable<_DHTLogPosition, DHTShortArray> {
 
   /// The type of the openable scope
   @override
-  FutureOr<DHTShortArray> scoped() => _segmentShortArray;
+  FutureOr<DHTShortArray> scoped() => shortArray;
 
   /// Add a reference to this log
   @override
@@ -201,9 +200,13 @@ class _DHTLogSpine {
                 throw TimeoutException('timeout reached');
               }
             }
+            try {
               if (await closure(this)) {
                 break;
               }
+            } on DHTExceptionTryAgain {
+              //
+            }
             // Failed to write in closure resets state
             _head = oldHead;
             _tail = oldTail;
@@ -452,22 +455,9 @@ class _DHTLogSpine {
   ///////////////////////////////////////////
   // API for public interfaces
 
-  Future<_DHTLogPosition?> lookupPosition(int pos) async {
-    assert(_spineMutex.isLocked, 'should be locked');
-    return _spineCacheMutex.protect(() async {
-      // Check if our position is in bounds
-      final endPos = length;
-      if (pos < 0 || pos >= endPos) {
-        throw IndexError.withLength(pos, endPos);
-      }
-
-      // Calculate absolute position, ring-buffer style
-      final absolutePosition = (_head + pos) % _positionLimit;
-
-      // Determine the segment number and position within the segment
-      final segmentNumber = absolutePosition ~/ DHTShortArray.maxElements;
-      final segmentPos = absolutePosition % DHTShortArray.maxElements;
-
+  Future<_DHTLogPosition?> lookupPositionBySegmentNumber(
+          int segmentNumber, int segmentPos) async =>
+      _spineCacheMutex.protect(() async {
         // Get the segment shortArray
         final openedSegment = _openedSegments[segmentNumber];
         late final DHTShortArray shortArray;
@@ -494,6 +484,24 @@ class _DHTLogSpine {
             pos: segmentPos,
             segmentNumber: segmentNumber);
       });
+
+  Future<_DHTLogPosition?> lookupPosition(int pos) async {
+    assert(_spineMutex.isLocked, 'should be locked');
+
+    // Check if our position is in bounds
+    final endPos = length;
+    if (pos < 0 || pos >= endPos) {
+      throw IndexError.withLength(pos, endPos);
+    }
+
+    // Calculate absolute position, ring-buffer style
+    final absolutePosition = (_head + pos) % _positionLimit;
+
+    // Determine the segment number and position within the segment
+    final segmentNumber = absolutePosition ~/ DHTShortArray.maxElements;
+    final segmentPos = absolutePosition % DHTShortArray.maxElements;
+
+    return lookupPositionBySegmentNumber(segmentNumber, segmentPos);
   }
 
   Future<void> _segmentClosed(int segmentNumber) async {
@@ -661,6 +669,34 @@ class _DHTLogSpine {
       final oldHead = _head;
       final oldTail = _tail;
       await _updateHead(headData);
+
+      // Lookup tail position segments that have changed
+      // and force their short arrays to refresh their heads
+      final segmentsToRefresh = <_DHTLogPosition>[];
+      int? lastSegmentNumber;
+      for (var curTail = oldTail;
+          curTail != _tail;
+          curTail = (curTail + 1) % _positionLimit) {
+        final segmentNumber = curTail ~/ DHTShortArray.maxElements;
+        final segmentPos = curTail % DHTShortArray.maxElements;
+        if (segmentNumber == lastSegmentNumber) {
+          continue;
+        }
+        lastSegmentNumber = segmentNumber;
+        final dhtLogPosition =
+            await lookupPositionBySegmentNumber(segmentNumber, segmentPos);
+        if (dhtLogPosition == null) {
+          throw Exception('missing segment in dht log');
+        }
+        segmentsToRefresh.add(dhtLogPosition);
+      }
+
+      // Refresh the segments that have probably changed
+      await segmentsToRefresh.map((p) async {
+        await p.shortArray.refresh();
+        await p.close();
+      }).wait;
+
       sendUpdate(oldHead, oldTail);
     });
   }

packages/veilid_support/lib/dht_support/src/dht_log/dht_log_write.dart

@@ -0,0 +1,166 @@
+part of 'dht_log.dart';
+
+////////////////////////////////////////////////////////////////////////////
+// Writer implementation
+
+abstract class DHTLogWriteOperations
+    implements DHTRandomRead, DHTRandomWrite, DHTAdd, DHTTruncate, DHTClear {}
+
+class _DHTLogWrite extends _DHTLogRead implements DHTLogWriteOperations {
+  _DHTLogWrite._(super.spine) : super._();
+
+  @override
+  Future<bool> tryWriteItem(int pos, Uint8List newValue,
+      {Output<Uint8List>? output}) async {
+    if (pos < 0 || pos >= _spine.length) {
+      throw IndexError.withLength(pos, _spine.length);
+    }
+    final lookup = await _spine.lookupPosition(pos);
+    if (lookup == null) {
+      throw StateError("can't lookup position in write to dht log");
+    }
+
+    // Write item to the segment
+    return lookup.scope((sa) => sa.operateWrite((write) async =>
+        write.tryWriteItem(lookup.pos, newValue, output: output)));
+  }
+
+  @override
+  Future<void> swap(int aPos, int bPos) async {
+    if (aPos < 0 || aPos >= _spine.length) {
+      throw IndexError.withLength(aPos, _spine.length);
+    }
+    if (bPos < 0 || bPos >= _spine.length) {
+      throw IndexError.withLength(bPos, _spine.length);
+    }
+    final aLookup = await _spine.lookupPosition(aPos);
+    if (aLookup == null) {
+      throw StateError("can't lookup position a in swap of dht log");
+    }
+    final bLookup = await _spine.lookupPosition(bPos);
+    if (bLookup == null) {
+      await aLookup.close();
+      throw StateError("can't lookup position b in swap of dht log");
+    }
+
+    // Swap items in the segments
+    if (aLookup.shortArray == bLookup.shortArray) {
+      await bLookup.close();
+      await aLookup.scope((sa) => sa.operateWriteEventual((aWrite) async {
+            await aWrite.swap(aLookup.pos, bLookup.pos);
+            return true;
+          }));
+    } else {
+      final bItem = Output<Uint8List>();
+      await aLookup.scope(
+          (sa) => bLookup.scope((sb) => sa.operateWriteEventual((aWrite) async {
+                if (bItem.value == null) {
+                  final aItem = await aWrite.get(aLookup.pos);
+                  if (aItem == null) {
+                    throw StateError("can't get item for position a in swap");
+                  }
+                  await sb.operateWriteEventual((bWrite) async =>
+                      bWrite.tryWriteItem(bLookup.pos, aItem, output: bItem));
+                }
+                return aWrite.tryWriteItem(aLookup.pos, bItem.value!);
+              })));
+    }
+  }
+
+  @override
+  Future<bool> tryAdd(Uint8List value) async {
+    // Allocate empty index at the end of the list
+    final insertPos = _spine.length;
+    _spine.allocateTail(1);
+    final lookup = await _spine.lookupPosition(insertPos);
+    if (lookup == null) {
+      throw StateError("can't write to dht log");
+    }
+
+    // Write item to the segment
+    return lookup.scope((sa) async {
+      try {
+        return sa.operateWrite((write) async {
+          // If this a new segment, then clear it in case we have wrapped around
+          if (lookup.pos == 0) {
+            await write.clear();
+          } else if (lookup.pos != write.length) {
+            // We should always be appending at the length
+            throw StateError('appending should be at the end');
+          }
+          return write.tryAdd(value);
+        });
+      } on DHTExceptionTryAgain {
+        return false;
+      }
+    });
+  }
+
+  @override
+  Future<bool> tryAddAll(List<Uint8List> values) async {
+    // Allocate empty index at the end of the list
+    final insertPos = _spine.length;
+    _spine.allocateTail(values.length);
+
+    // Look up the first position and shortarray
+    final dws = DelayedWaitSet<void>();
+
+    var success = true;
+    for (var valueIdx = 0; valueIdx < values.length;) {
+      final remaining = values.length - valueIdx;
+
+      final lookup = await _spine.lookupPosition(insertPos + valueIdx);
+      if (lookup == null) {
+        throw StateError("can't write to dht log");
+      }
+
+      final sacount = min(remaining, DHTShortArray.maxElements - lookup.pos);
+      final sublistValues = values.sublist(valueIdx, valueIdx + sacount);
+
+      dws.add(() async {
+        final ok = await lookup.scope((sa) async {
+          try {
+            return sa.operateWrite((write) async {
+              // If this a new segment, then clear it in
+              // case we have wrapped around
+              if (lookup.pos == 0) {
+                await write.clear();
+              } else if (lookup.pos != write.length) {
+                // We should always be appending at the length
+                throw StateError('appending should be at the end');
+              }
+              return write.tryAddAll(sublistValues);
+            });
+          } on DHTExceptionTryAgain {
+            return false;
+          }
+        });
+        if (!ok) {
+          success = false;
+        }
+      });
+
+      valueIdx += sacount;
+    }
+
+    await dws();
+
+    return success;
+  }
+
+  @override
+  Future<void> truncate(int newLength) async {
+    if (newLength < 0) {
+      throw StateError('can not truncate to negative length');
+    }
+    if (newLength >= _spine.length) {
+      return;
+    }
+    await _spine.releaseHead(_spine.length - newLength);
+  }
+
+  @override
+  Future<void> clear() async {
+    await _spine.releaseHead(_spine.length);
+  }
+}

packages/veilid_support/lib/dht_support/src/dht_record/barrel.dart

@@ -1,4 +1,3 @@
 export 'default_dht_record_cubit.dart';
-export 'dht_record_crypto.dart';
 export 'dht_record_cubit.dart';
 export 'dht_record_pool.dart';

packages/veilid_support/lib/dht_support/src/dht_record/dht_record.dart

@@ -42,7 +42,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
       required SharedDHTRecordData sharedDHTRecordData,
       required int defaultSubkey,
       required KeyPair? writer,
-      required DHTRecordCrypto crypto,
+      required VeilidCrypto crypto,
       required this.debugName})
       : _crypto = crypto,
         _routingContext = routingContext,
@@ -104,7 +104,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   int get subkeyCount =>
       _sharedDHTRecordData.recordDescriptor.schema.subkeyCount();
   KeyPair? get writer => _writer;
-  DHTRecordCrypto get crypto => _crypto;
+  VeilidCrypto get crypto => _crypto;
   OwnedDHTRecordPointer get ownedDHTRecordPointer =>
       OwnedDHTRecordPointer(recordKey: key, owner: ownerKeyPair!);
   int subkeyOrDefault(int subkey) => (subkey == -1) ? _defaultSubkey : subkey;
@@ -118,7 +118,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   ///   returned if one was returned.
   Future<Uint8List?> get(
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
       Output<int>? outSeqNum}) async {
     subkey = subkeyOrDefault(subkey);
@@ -146,7 +146,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
       return null;
     }
     // If we're returning a value, decrypt it
-    final out = (crypto ?? _crypto).decrypt(valueData.data, subkey);
+    final out = (crypto ?? _crypto).decrypt(valueData.data);
     if (outSeqNum != null) {
       outSeqNum.save(valueData.seq);
     }
@@ -163,7 +163,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   ///   returned if one was returned.
   Future<T?> getJson<T>(T Function(dynamic) fromJson,
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
       Output<int>? outSeqNum}) async {
     final data = await get(
@@ -189,7 +189,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   Future<T?> getProtobuf<T extends GeneratedMessage>(
       T Function(List<int> i) fromBuffer,
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       DHTRecordRefreshMode refreshMode = DHTRecordRefreshMode.cached,
       Output<int>? outSeqNum}) async {
     final data = await get(
@@ -208,13 +208,12 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   /// If the value was succesfully written, null is returned
   Future<Uint8List?> tryWriteBytes(Uint8List newValue,
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       KeyPair? writer,
       Output<int>? outSeqNum}) async {
     subkey = subkeyOrDefault(subkey);
     final lastSeq = await _localSubkeySeq(subkey);
-    final encryptedNewValue =
-        await (crypto ?? _crypto).encrypt(newValue, subkey);
+    final encryptedNewValue = await (crypto ?? _crypto).encrypt(newValue);
 
     // Set the new data if possible
     var newValueData = await _routingContext
@@ -246,7 +245,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
 
     // Decrypt value to return it
     final decryptedNewValue =
-        await (crypto ?? _crypto).decrypt(newValueData.data, subkey);
+        await (crypto ?? _crypto).decrypt(newValueData.data);
     if (isUpdated) {
       DHTRecordPool.instance
           .processLocalValueChange(key, decryptedNewValue, subkey);
@@ -259,13 +258,12 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   /// will be made to write the subkey until this succeeds
   Future<void> eventualWriteBytes(Uint8List newValue,
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       KeyPair? writer,
       Output<int>? outSeqNum}) async {
     subkey = subkeyOrDefault(subkey);
     final lastSeq = await _localSubkeySeq(subkey);
-    final encryptedNewValue =
-        await (crypto ?? _crypto).encrypt(newValue, subkey);
+    final encryptedNewValue = await (crypto ?? _crypto).encrypt(newValue);
 
     ValueData? newValueData;
     do {
@@ -309,7 +307,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   Future<void> eventualUpdateBytes(
       Future<Uint8List> Function(Uint8List? oldValue) update,
       {int subkey = -1,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       KeyPair? writer,
       Output<int>? outSeqNum}) async {
     subkey = subkeyOrDefault(subkey);
@@ -334,7 +332,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   /// Like 'tryWriteBytes' but with JSON marshal/unmarshal of the value
   Future<T?> tryWriteJson<T>(T Function(dynamic) fromJson, T newValue,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       tryWriteBytes(jsonEncodeBytes(newValue),
@@ -353,7 +351,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   Future<T?> tryWriteProtobuf<T extends GeneratedMessage>(
           T Function(List<int>) fromBuffer, T newValue,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       tryWriteBytes(newValue.writeToBuffer(),
@@ -371,7 +369,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   /// Like 'eventualWriteBytes' but with JSON marshal/unmarshal of the value
   Future<void> eventualWriteJson<T>(T newValue,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       eventualWriteBytes(jsonEncodeBytes(newValue),
@@ -380,7 +378,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   /// Like 'eventualWriteBytes' but with protobuf marshal/unmarshal of the value
   Future<void> eventualWriteProtobuf<T extends GeneratedMessage>(T newValue,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       eventualWriteBytes(newValue.writeToBuffer(),
@@ -390,7 +388,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   Future<void> eventualUpdateJson<T>(
           T Function(dynamic) fromJson, Future<T> Function(T?) update,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       eventualUpdateBytes(jsonUpdate(fromJson, update),
@@ -400,7 +398,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   Future<void> eventualUpdateProtobuf<T extends GeneratedMessage>(
           T Function(List<int>) fromBuffer, Future<T> Function(T?) update,
           {int subkey = -1,
-          DHTRecordCrypto? crypto,
+          VeilidCrypto? crypto,
           KeyPair? writer,
           Output<int>? outSeqNum}) =>
       eventualUpdateBytes(protobufUpdate(fromBuffer, update),
@@ -433,7 +431,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
             DHTRecord record, Uint8List? data, List<ValueSubkeyRange> subkeys)
         onUpdate, {
     bool localChanges = true,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) async {
     // Set up watch requirements
     _watchController ??=
@@ -457,8 +455,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
               final changeData = change.data;
               data = changeData == null
                   ? null
-                  : await (crypto ?? _crypto)
-                      .decrypt(changeData, change.subkeys.first.low);
+                  : await (crypto ?? _crypto).decrypt(changeData);
             }
             await onUpdate(this, data, change.subkeys);
           });
@@ -544,7 +541,7 @@ class DHTRecord implements DHTDeleteable<DHTRecord, DHTRecord> {
   final VeilidRoutingContext _routingContext;
   final int _defaultSubkey;
   final KeyPair? _writer;
-  final DHTRecordCrypto _crypto;
+  final VeilidCrypto _crypto;
   final String debugName;
   final _mutex = Mutex();
   int _openCount;

packages/veilid_support/lib/dht_support/src/dht_record/dht_record_crypto.dart

@@ -1,53 +0,0 @@
-import 'dart:async';
-import 'dart:typed_data';
-import '../../../../../veilid_support.dart';
-
-abstract class DHTRecordCrypto {
-  Future<Uint8List> encrypt(Uint8List data, int subkey);
-  Future<Uint8List> decrypt(Uint8List data, int subkey);
-}
-
-////////////////////////////////////
-/// Private DHT Record: Encrypted for a specific symmetric key
-class DHTRecordCryptoPrivate implements DHTRecordCrypto {
-  DHTRecordCryptoPrivate._(
-      VeilidCryptoSystem cryptoSystem, SharedSecret secretKey)
-      : _cryptoSystem = cryptoSystem,
-        _secretKey = secretKey;
-  final VeilidCryptoSystem _cryptoSystem;
-  final SharedSecret _secretKey;
-
-  static Future<DHTRecordCryptoPrivate> fromTypedKeyPair(
-      TypedKeyPair typedKeyPair) async {
-    final cryptoSystem =
-        await Veilid.instance.getCryptoSystem(typedKeyPair.kind);
-    final secretKey = typedKeyPair.secret;
-    return DHTRecordCryptoPrivate._(cryptoSystem, secretKey);
-  }
-
-  static Future<DHTRecordCryptoPrivate> fromSecret(
-      CryptoKind kind, SharedSecret secretKey) async {
-    final cryptoSystem = await Veilid.instance.getCryptoSystem(kind);
-    return DHTRecordCryptoPrivate._(cryptoSystem, secretKey);
-  }
-
-  @override
-  Future<Uint8List> encrypt(Uint8List data, int subkey) =>
-      _cryptoSystem.encryptNoAuthWithNonce(data, _secretKey);
-
-  @override
-  Future<Uint8List> decrypt(Uint8List data, int subkey) =>
-      _cryptoSystem.decryptNoAuthWithNonce(data, _secretKey);
-}
-
-////////////////////////////////////
-/// Public DHT Record: No encryption
-class DHTRecordCryptoPublic implements DHTRecordCrypto {
-  const DHTRecordCryptoPublic();
-
-  @override
-  Future<Uint8List> encrypt(Uint8List data, int subkey) async => data;
-
-  @override
-  Future<Uint8List> decrypt(Uint8List data, int subkey) async => data;
-}

packages/veilid_support/lib/dht_support/src/dht_record/dht_record_pool.dart

@@ -27,6 +27,9 @@ const int watchRenewalDenominator = 5;
 // Maximum number of concurrent DHT operations to perform on the network
 const int maxDHTConcurrency = 8;
 
+// DHT crypto domain
+const String cryptoDomainDHT = 'dht';
+
 typedef DHTRecordPoolLogger = void Function(String message);
 
 /// Record pool that managed DHTRecords and allows for tagged deletion
@@ -526,7 +529,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
     TypedKey? parent,
     DHTSchema schema = const DHTSchema.dflt(oCnt: 1),
     int defaultSubkey = 0,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
     KeyPair? writer,
   }) async =>
       _mutex.protect(() async {
@@ -547,9 +550,9 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
             writer: writer ??
                 openedRecordInfo.shared.recordDescriptor.ownerKeyPair(),
             crypto: crypto ??
-                await DHTRecordCryptoPrivate.fromTypedKeyPair(openedRecordInfo
+                await privateCryptoFromTypedSecret(openedRecordInfo
                     .shared.recordDescriptor
-                    .ownerTypedKeyPair()!));
+                    .ownerTypedSecret()!));
 
         openedRecordInfo.records.add(rec);
 
@@ -562,7 +565,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
           VeilidRoutingContext? routingContext,
           TypedKey? parent,
           int defaultSubkey = 0,
-          DHTRecordCrypto? crypto}) async =>
+          VeilidCrypto? crypto}) async =>
       _mutex.protect(() async {
         final dhtctx = routingContext ?? _routingContext;
 
@@ -578,7 +581,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
             defaultSubkey: defaultSubkey,
             sharedDHTRecordData: openedRecordInfo.shared,
             writer: null,
-            crypto: crypto ?? const DHTRecordCryptoPublic());
+            crypto: crypto ?? const VeilidCryptoPublic());
 
         openedRecordInfo.records.add(rec);
 
@@ -593,7 +596,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
     VeilidRoutingContext? routingContext,
     TypedKey? parent,
     int defaultSubkey = 0,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) async =>
       _mutex.protect(() async {
         final dhtctx = routingContext ?? _routingContext;
@@ -612,8 +615,8 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
             writer: writer,
             sharedDHTRecordData: openedRecordInfo.shared,
             crypto: crypto ??
-                await DHTRecordCryptoPrivate.fromTypedKeyPair(
-                    TypedKeyPair.fromKeyPair(recordKey.kind, writer)));
+                await privateCryptoFromTypedSecret(
+                    TypedKey(kind: recordKey.kind, value: writer.secret)));
 
         openedRecordInfo.records.add(rec);
 
@@ -632,7 +635,7 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
     required TypedKey parent,
     VeilidRoutingContext? routingContext,
     int defaultSubkey = 0,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) =>
       openRecordWrite(
         ownedDHTRecordPointer.recordKey,
@@ -663,6 +666,11 @@ class DHTRecordPool with TableDBBackedJson<DHTRecordPoolAllocations> {
     }
   }
 
+  /// Generate default VeilidCrypto for a writer
+  static Future<VeilidCrypto> privateCryptoFromTypedSecret(
+          TypedKey typedSecret) async =>
+      VeilidCryptoPrivate.fromTypedKey(typedSecret, cryptoDomainDHT);
+
   /// Handle the DHT record updates coming from Veilid
   void processRemoteValueChange(VeilidUpdateValueChange updateValueChange) {
     if (updateValueChange.subkeys.isNotEmpty) {

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array.dart

@@ -33,7 +33,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
       int stride = maxElements,
       VeilidRoutingContext? routingContext,
       TypedKey? parent,
-      DHTRecordCrypto? crypto,
+      VeilidCrypto? crypto,
       KeyPair? writer}) async {
     assert(stride <= maxElements, 'stride too long');
     final pool = DHTRecordPool.instance;
@@ -79,7 +79,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
       {required String debugName,
       VeilidRoutingContext? routingContext,
       TypedKey? parent,
-      DHTRecordCrypto? crypto}) async {
+      VeilidCrypto? crypto}) async {
     final dhtRecord = await DHTRecordPool.instance.openRecordRead(headRecordKey,
         debugName: debugName,
         parent: parent,
@@ -101,7 +101,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
     required String debugName,
     VeilidRoutingContext? routingContext,
     TypedKey? parent,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) async {
     final dhtRecord = await DHTRecordPool.instance.openRecordWrite(
         headRecordKey, writer,
@@ -124,7 +124,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
     required String debugName,
     required TypedKey parent,
     VeilidRoutingContext? routingContext,
-    DHTRecordCrypto? crypto,
+    VeilidCrypto? crypto,
   }) =>
       openWrite(
         ownedShortArrayRecordPointer.recordKey,
@@ -185,8 +185,20 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
   /// Get the record pointer foir this shortarray
   OwnedDHTRecordPointer get recordPointer => _head.recordPointer;
 
+  /// Refresh this DHTShortArray
+  /// Useful if you aren't 'watching' the array and want to poll for an update
+  Future<void> refresh() async {
+    if (!isOpen) {
+      throw StateError('short array is not open"');
+    }
+    await _head.operate((head) async {
+      await head._loadHead();
+    });
+  }
+
   /// Runs a closure allowing read-only access to the shortarray
-  Future<T> operate<T>(Future<T> Function(DHTRandomRead) closure) async {
+  Future<T> operate<T>(
+      Future<T> Function(DHTShortArrayReadOperations) closure) async {
     if (!isOpen) {
       throw StateError('short array is not open"');
     }
@@ -203,7 +215,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
   /// Throws DHTOperateException if the write could not be performed
   /// at this time
   Future<T> operateWrite<T>(
-      Future<T> Function(DHTRandomReadWrite) closure) async {
+      Future<T> Function(DHTShortArrayWriteOperations) closure) async {
     if (!isOpen) {
       throw StateError('short array is not open"');
     }
@@ -221,7 +233,7 @@ class DHTShortArray implements DHTDeleteable<DHTShortArray, DHTShortArray> {
   /// succeeded, returning false will trigger another eventual consistency
   /// attempt.
   Future<void> operateWriteEventual(
-      Future<bool> Function(DHTRandomReadWrite) closure,
+      Future<bool> Function(DHTShortArrayWriteOperations) closure,
       {Duration? timeout}) async {
     if (!isOpen) {
       throw StateError('short array is not open"');

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_cubit.dart

@@ -54,8 +54,7 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
     try {
       final newState = await _shortArray.operate((reader) async {
         final offlinePositions = await reader.getOfflinePositions();
-        final allItems =
-            (await reader.getItemRange(0, forceRefresh: forceRefresh))
+        final allItems = (await reader.getRange(0, forceRefresh: forceRefresh))
             ?.indexed
             .map((x) => DHTShortArrayElementState(
                 value: _decodeElement(x.$2),
@@ -91,19 +90,20 @@ class DHTShortArrayCubit<T> extends Cubit<DHTShortArrayBusyState<T>>
     await super.close();
   }
 
-  Future<R> operate<R>(Future<R> Function(DHTRandomRead) closure) async {
+  Future<R> operate<R>(
+      Future<R> Function(DHTShortArrayReadOperations) closure) async {
     await _initWait();
     return _shortArray.operate(closure);
   }
 
   Future<R> operateWrite<R>(
-      Future<R> Function(DHTRandomReadWrite) closure) async {
+      Future<R> Function(DHTShortArrayWriteOperations) closure) async {
     await _initWait();
     return _shortArray.operateWrite(closure);
   }
 
   Future<void> operateWriteEventual(
-      Future<bool> Function(DHTRandomReadWrite) closure,
+      Future<bool> Function(DHTShortArrayWriteOperations) closure,
       {Duration? timeout}) async {
     await _initWait();
     return _shortArray.operateWriteEventual(closure, timeout: timeout);

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_head.dart

@@ -139,9 +139,14 @@ class _DHTShortArrayHead {
                 throw TimeoutException('timeout reached');
               }
             }
+            try {
               if (await closure(this)) {
                 break;
               }
+            } on DHTExceptionTryAgain {
+              //
+            }
+
             // Failed to write in closure resets state
             _linkedRecords = List.of(oldLinkedRecords);
             _index = List.of(oldIndex);

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_read.dart

@@ -3,14 +3,16 @@ part of 'dht_short_array.dart';
 ////////////////////////////////////////////////////////////////////////////
 // Reader-only implementation
 
-class _DHTShortArrayRead implements DHTRandomRead {
+abstract class DHTShortArrayReadOperations implements DHTRandomRead {}
+
+class _DHTShortArrayRead implements DHTShortArrayReadOperations {
   _DHTShortArrayRead._(_DHTShortArrayHead head) : _head = head;
 
   @override
   int get length => _head.length;
 
   @override
-  Future<Uint8List?> getItem(int pos, {bool forceRefresh = false}) async {
+  Future<Uint8List?> get(int pos, {bool forceRefresh = false}) async {
     if (pos < 0 || pos >= length) {
       throw IndexError.withLength(pos, length);
     }
@@ -47,14 +49,14 @@ class _DHTShortArrayRead implements DHTRandomRead {
   }
 
   @override
-  Future<List<Uint8List>?> getItemRange(int start,
+  Future<List<Uint8List>?> getRange(int start,
       {int? length, bool forceRefresh = false}) async {
     final out = <Uint8List>[];
     (start, length) = _clampStartLen(start, length);
 
     final chunks = Iterable<int>.generate(length).slices(maxDHTConcurrency).map(
-        (chunk) => chunk
-            .map((pos) => getItem(pos + start, forceRefresh: forceRefresh)));
+        (chunk) =>
+            chunk.map((pos) => get(pos + start, forceRefresh: forceRefresh)));
 
     for (final chunk in chunks) {
       final elems = await chunk.wait;

packages/veilid_support/lib/dht_support/src/dht_short_array/dht_short_array_write.dart

@@ -3,20 +3,27 @@ part of 'dht_short_array.dart';
 ////////////////////////////////////////////////////////////////////////////
 // Writer implementation
 
+abstract class DHTShortArrayWriteOperations
+    implements
+        DHTRandomRead,
+        DHTRandomWrite,
+        DHTInsertRemove,
+        DHTAdd,
+        DHTClear {}
+
 class _DHTShortArrayWrite extends _DHTShortArrayRead
-    implements DHTRandomReadWrite {
+    implements DHTShortArrayWriteOperations {
   _DHTShortArrayWrite._(super.head) : super._();
 
   @override
-  Future<bool> tryAddItem(Uint8List value) =>
-      tryInsertItem(_head.length, value);
+  Future<bool> tryAdd(Uint8List value) => tryInsert(_head.length, value);
 
   @override
-  Future<bool> tryAddItems(List<Uint8List> values) =>
-      tryInsertItems(_head.length, values);
+  Future<bool> tryAddAll(List<Uint8List> values) =>
+      tryInsertAll(_head.length, values);
 
   @override
-  Future<bool> tryInsertItem(int pos, Uint8List value) async {
+  Future<bool> tryInsert(int pos, Uint8List value) async {
     if (pos < 0 || pos > _head.length) {
       throw IndexError.withLength(pos, _head.length);
     }
@@ -36,7 +43,7 @@ class _DHTShortArrayWrite extends _DHTShortArrayRead
   }
 
   @override
-  Future<bool> tryInsertItems(int pos, List<Uint8List> values) async {
+  Future<bool> tryInsertAll(int pos, List<Uint8List> values) async {
     if (pos < 0 || pos > _head.length) {
       throw IndexError.withLength(pos, _head.length);
     }
@@ -92,7 +99,7 @@ class _DHTShortArrayWrite extends _DHTShortArrayRead
   }
 
   @override
-  Future<void> swapItem(int aPos, int bPos) async {
+  Future<void> swap(int aPos, int bPos) async {
     if (aPos < 0 || aPos >= _head.length) {
       throw IndexError.withLength(aPos, _head.length);
     }
@@ -104,7 +111,7 @@ class _DHTShortArrayWrite extends _DHTShortArrayRead
   }
 
   @override
-  Future<void> removeItem(int pos, {Output<Uint8List>? output}) async {
+  Future<void> remove(int pos, {Output<Uint8List>? output}) async {
     if (pos < 0 || pos >= _head.length) {
       throw IndexError.withLength(pos, _head.length);
     }

packages/veilid_support/lib/dht_support/src/interfaces/dht_add.dart

@@ -0,0 +1,41 @@
+import 'dart:typed_data';
+
+import 'package:protobuf/protobuf.dart';
+
+import '../../../veilid_support.dart';
+
+////////////////////////////////////////////////////////////////////////////
+// Add
+abstract class DHTAdd {
+  /// Try to add an item to the DHT container.
+  /// Return true if the element was successfully added, and false if the state
+  /// changed before the element could be added or a newer value was found on
+  /// the network.
+  /// Throws a StateError if the container exceeds its maximum size.
+  Future<bool> tryAdd(Uint8List value);
+
+  /// Try to add a list of items to the DHT container.
+  /// Return true if the elements were successfully added, and false if the
+  /// state changed before the element could be added or a newer value was found
+  /// on the network.
+  /// Throws a StateError if the container exceeds its maximum size.
+  Future<bool> tryAddAll(List<Uint8List> values);
+}
+
+extension DHTAddExt on DHTAdd {
+  /// Convenience function:
+  /// Like tryAddItem but also encodes the input value as JSON and parses the
+  /// returned element as JSON
+  Future<bool> tryAddJson<T>(
+    T newValue,
+  ) =>
+      tryAdd(jsonEncodeBytes(newValue));
+
+  /// Convenience function:
+  /// Like tryAddItem but also encodes the input value as a protobuf object
+  /// and parses the returned element as a protobuf object
+  Future<bool> tryAddProtobuf<T extends GeneratedMessage>(
+    T newValue,
+  ) =>
+      tryAdd(newValue.writeToBuffer());
+}

packages/veilid_support/lib/dht_support/src/interfaces/dht_append_truncate.dart

@@ -1,51 +0,0 @@
-import 'dart:typed_data';
-
-import 'package:protobuf/protobuf.dart';
-
-import '../../../veilid_support.dart';
-
-////////////////////////////////////////////////////////////////////////////
-// Append/truncate interface
-abstract class DHTAppendTruncate {
-  /// Try to add an item to the end of the DHT data structure.
-  /// Return true if the element was successfully added, and false if the state
-  /// changed before the element could be added or a newer value was found on
-  /// the network.
-  /// This may throw an exception if the number elements added exceeds limits.
-  Future<bool> tryAppendItem(Uint8List value);
-
-  /// Try to add a list of items to the end of the DHT data structure.
-  /// Return true if the elements were successfully added, and false if the
-  /// state changed before the element could be added or a newer value was found
-  /// on the network.
-  /// This may throw an exception if the number elements added exceeds limits.
-  Future<bool> tryAppendItems(List<Uint8List> values);
-
-  /// Try to remove a number of items from the head of the DHT data structure.
-  /// Throws StateError if count < 0
-  Future<void> truncate(int count);
-
-  /// Remove all items in the DHT data structure.
-  Future<void> clear();
-}
-
-abstract class DHTAppendTruncateRandomRead
-    implements DHTAppendTruncate, DHTRandomRead {}
-
-extension DHTAppendTruncateExt on DHTAppendTruncate {
-  /// Convenience function:
-  /// Like tryAppendItem but also encodes the input value as JSON and parses the
-  /// returned element as JSON
-  Future<bool> tryAppendItemJson<T>(
-    T newValue,
-  ) =>
-      tryAppendItem(jsonEncodeBytes(newValue));
-
-  /// Convenience function:
-  /// Like tryAppendItem but also encodes the input value as a protobuf object
-  /// and parses the returned element as a protobuf object
-  Future<bool> tryAppendItemProtobuf<T extends GeneratedMessage>(
-    T newValue,
-  ) =>
-      tryAppendItem(newValue.writeToBuffer());
-}

packages/veilid_support/lib/dht_support/src/interfaces/dht_clear.dart

@@ -0,0 +1,7 @@
+////////////////////////////////////////////////////////////////////////////
+// Clear interface
+// ignore: one_member_abstracts
+abstract class DHTClear {
+  /// Remove all items in the DHT container.
+  Future<void> clear();
+}

packages/veilid_support/lib/dht_support/src/interfaces/dht_insert_remove.dart

@@ -0,0 +1,55 @@
+import 'dart:typed_data';
+
+import 'package:protobuf/protobuf.dart';
+
+import '../../../veilid_support.dart';
+
+////////////////////////////////////////////////////////////////////////////
+// Insert/Remove interface
+abstract class DHTInsertRemove {
+  /// Try to insert an item as position 'pos' of the DHT container.
+  /// Return true if the element was successfully inserted, and false if the
+  /// state changed before the element could be inserted or a newer value was
+  /// found on the network.
+  /// Throws an IndexError if the position removed exceeds the length of
+  /// the container.
+  /// Throws a StateError if the container exceeds its maximum size.
+  Future<bool> tryInsert(int pos, Uint8List value);
+
+  /// Try to insert items at position 'pos' of the DHT container.
+  /// Return true if the elements were successfully inserted, and false if the
+  /// state changed before the elements could be inserted or a newer value was
+  /// found on the network.
+  /// Throws an IndexError if the position removed exceeds the length of
+  /// the container.
+  /// Throws a StateError if the container exceeds its maximum size.
+  Future<bool> tryInsertAll(int pos, List<Uint8List> values);
+
+  /// Remove an item at position 'pos' in the DHT container.
+  /// If the remove was successful this returns:
+  ///   * outValue will return the prior contents of the element
+  /// Throws an IndexError if the position removed exceeds the length of
+  /// the container.
+  Future<void> remove(int pos, {Output<Uint8List>? output});
+}
+
+extension DHTInsertRemoveExt on DHTInsertRemove {
+  /// Convenience function:
+  /// Like remove but also parses the returned element as JSON
+  Future<void> removeJson<T>(T Function(dynamic) fromJson, int pos,
+      {Output<T>? output}) async {
+    final outValueBytes = output == null ? null : Output<Uint8List>();
+    await remove(pos, output: outValueBytes);
+    output.mapSave(outValueBytes, (b) => jsonDecodeBytes(fromJson, b));
+  }
+
+  /// Convenience function:
+  /// Like remove but also parses the returned element as JSON
+  Future<void> removeProtobuf<T extends GeneratedMessage>(
+      T Function(List<int>) fromBuffer, int pos,
+      {Output<T>? output}) async {
+    final outValueBytes = output == null ? null : Output<Uint8List>();
+    await remove(pos, output: outValueBytes);
+    output.mapSave(outValueBytes, fromBuffer);
+  }
+}

packages/veilid_support/lib/dht_support/src/interfaces/dht_random_read.dart

@@ -7,23 +7,22 @@ import '../../../veilid_support.dart';
 ////////////////////////////////////////////////////////////////////////////
 // Reader interface
 abstract class DHTRandomRead {
-  /// Returns the number of elements in the DHTArray
-  /// This number will be >= 0 and <= DHTShortArray.maxElements (256)
+  /// Returns the number of elements in the DHT container
   int get length;
 
-  /// Return the item at position 'pos' in the DHTArray. If 'forceRefresh'
+  /// Return the item at position 'pos' in the DHT container. If 'forceRefresh'
   /// is specified, the network will always be checked for newer values
   /// rather than returning the existing locally stored copy of the elements.
-  /// * 'pos' must be >= 0 and < 'length'
-  Future<Uint8List?> getItem(int pos, {bool forceRefresh = false});
+  /// Throws an IndexError if the 'pos' is not within the length
+  /// of the container.
+  Future<Uint8List?> get(int pos, {bool forceRefresh = false});
 
   /// Return a list of a range of items in the DHTArray. If 'forceRefresh'
   /// is specified, the network will always be checked for newer values
   /// rather than returning the existing locally stored copy of the elements.
-  /// * 'start' must be >= 0
-  /// * 'len' must be >= 0 and <= DHTShortArray.maxElements (256) and defaults
-  ///    to the maximum length
-  Future<List<Uint8List>?> getItemRange(int start,
+  /// Throws an IndexError if either 'start' or '(start+length)' is not within
+  /// the length of the container.
+  Future<List<Uint8List>?> getRange(int start,
       {int? length, bool forceRefresh = false});
 
   /// Get a list of the positions that were written offline and not flushed yet
@@ -32,32 +31,32 @@ abstract class DHTRandomRead {
 
 extension DHTRandomReadExt on DHTRandomRead {
   /// Convenience function:
-  /// Like getItem but also parses the returned element as JSON
-  Future<T?> getItemJson<T>(T Function(dynamic) fromJson, int pos,
+  /// Like get but also parses the returned element as JSON
+  Future<T?> getJson<T>(T Function(dynamic) fromJson, int pos,
           {bool forceRefresh = false}) =>
-      getItem(pos, forceRefresh: forceRefresh)
+      get(pos, forceRefresh: forceRefresh)
           .then((out) => jsonDecodeOptBytes(fromJson, out));
 
   /// Convenience function:
-  /// Like getAllItems but also parses the returned elements as JSON
-  Future<List<T>?> getItemRangeJson<T>(T Function(dynamic) fromJson, int start,
+  /// Like getRange but also parses the returned elements as JSON
+  Future<List<T>?> getRangeJson<T>(T Function(dynamic) fromJson, int start,
           {int? length, bool forceRefresh = false}) =>
-      getItemRange(start, length: length, forceRefresh: forceRefresh)
+      getRange(start, length: length, forceRefresh: forceRefresh)
           .then((out) => out?.map(fromJson).toList());
 
   /// Convenience function:
-  /// Like getItem but also parses the returned element as a protobuf object
-  Future<T?> getItemProtobuf<T extends GeneratedMessage>(
+  /// Like get but also parses the returned element as a protobuf object
+  Future<T?> getProtobuf<T extends GeneratedMessage>(
           T Function(List<int>) fromBuffer, int pos,
           {bool forceRefresh = false}) =>
-      getItem(pos, forceRefresh: forceRefresh)
+      get(pos, forceRefresh: forceRefresh)
           .then((out) => (out == null) ? null : fromBuffer(out));
 
   /// Convenience function:
-  /// Like getAllItems but also parses the returned elements as protobuf objects
-  Future<List<T>?> getItemRangeProtobuf<T extends GeneratedMessage>(
+  /// Like getRange but also parses the returned elements as protobuf objects
+  Future<List<T>?> getRangeProtobuf<T extends GeneratedMessage>(
           T Function(List<int>) fromBuffer, int start,
           {int? length, bool forceRefresh = false}) =>
-      getItemRange(start, length: length, forceRefresh: forceRefresh)
+      getRange(start, length: length, forceRefresh: forceRefresh)
           .then((out) => out?.map(fromBuffer).toList());
 }

packages/veilid_support/lib/dht_support/src/interfaces/dht_random_write.dart

@@ -6,8 +6,9 @@ import '../../../veilid_support.dart';
 
 ////////////////////////////////////////////////////////////////////////////
 // Writer interface
+// ignore: one_member_abstracts
 abstract class DHTRandomWrite {
-  /// Try to set an item at position 'pos' of the DHTArray.
+  /// Try to set an item at position 'pos' of the DHT container.
   /// If the set was successful this returns:
   ///   * A boolean true
   ///   * outValue will return the prior contents of the element,
@@ -18,55 +19,15 @@ abstract class DHTRandomWrite {
   ///   * outValue will return the newer value of the element,
   ///     or null if the head record changed.
   ///
-  /// This may throw an exception if the position exceeds the built-in limit of
-  /// 'maxElements = 256' entries.
+  /// Throws an IndexError if the position is not within the length
+  /// of the container.
   Future<bool> tryWriteItem(int pos, Uint8List newValue,
       {Output<Uint8List>? output});
 
-  /// Try to add an item to the end of the DHTArray. Return true if the
-  /// element was successfully added, and false if the state changed before
-  /// the element could be added or a newer value was found on the network.
-  /// This may throw an exception if the number elements added exceeds the
-  /// built-in limit of 'maxElements = 256' entries.
-  Future<bool> tryAddItem(Uint8List value);
-
-  /// Try to add a list of items to the end of the DHTArray. Return true if the
-  /// elements were successfully added, and false if the state changed before
-  /// the elements could be added or a newer value was found on the network.
-  /// This may throw an exception if the number elements added exceeds the
-  /// built-in limit of 'maxElements = 256' entries.
-  Future<bool> tryAddItems(List<Uint8List> values);
-
-  /// Try to insert an item as position 'pos' of the DHTArray.
-  /// Return true if the element was successfully inserted, and false if the
-  /// state changed before the element could be inserted or a newer value was
-  /// found on the network.
-  /// This may throw an exception if the number elements added exceeds the
-  /// built-in limit of 'maxElements = 256' entries.
-  Future<bool> tryInsertItem(int pos, Uint8List value);
-
-  /// Try to insert items at position 'pos' of the DHTArray.
-  /// Return true if the elements were successfully inserted, and false if the
-  /// state changed before the elements could be inserted or a newer value was
-  /// found on the network.
-  /// This may throw an exception if the number elements added exceeds the
-  /// built-in limit of 'maxElements = 256' entries.
-  Future<bool> tryInsertItems(int pos, List<Uint8List> values);
-
   /// Swap items at position 'aPos' and 'bPos' in the DHTArray.
-  /// Throws IndexError if either of the positions swapped exceed
-  /// the length of the list
-  Future<void> swapItem(int aPos, int bPos);
-
-  /// Remove an item at position 'pos' in the DHTArray.
-  /// If the remove was successful this returns:
-  ///   * outValue will return the prior contents of the element
-  /// Throws IndexError if the position removed exceeds the length of
-  /// the list.
-  Future<void> removeItem(int pos, {Output<Uint8List>? output});
-
-  /// Remove all items in the DHTShortArray.
-  Future<void> clear();
+  /// Throws an IndexError if either of the positions swapped exceeds the length
+  /// of the container
+  Future<void> swap(int aPos, int bPos);
 }
 
 extension DHTRandomWriteExt on DHTRandomWrite {
@@ -95,25 +56,4 @@ extension DHTRandomWriteExt on DHTRandomWrite {
     output.mapSave(outValueBytes, fromBuffer);
     return out;
   }
-
-  /// Convenience function:
-  /// Like removeItem but also parses the returned element as JSON
-  Future<void> removeItemJson<T>(T Function(dynamic) fromJson, int pos,
-      {Output<T>? output}) async {
-    final outValueBytes = output == null ? null : Output<Uint8List>();
-    await removeItem(pos, output: outValueBytes);
-    output.mapSave(outValueBytes, (b) => jsonDecodeBytes(fromJson, b));
 }
-
-  /// Convenience function:
-  /// Like removeItem but also parses the returned element as JSON
-  Future<void> removeItemProtobuf<T extends GeneratedMessage>(
-      T Function(List<int>) fromBuffer, int pos,
-      {Output<T>? output}) async {
-    final outValueBytes = output == null ? null : Output<Uint8List>();
-    await removeItem(pos, output: outValueBytes);
-    output.mapSave(outValueBytes, fromBuffer);
-  }
-}
-
-abstract class DHTRandomReadWrite implements DHTRandomRead, DHTRandomWrite {}

packages/veilid_support/lib/dht_support/src/interfaces/dht_truncate.dart

@@ -0,0 +1,8 @@
+////////////////////////////////////////////////////////////////////////////
+// Truncate interface
+// ignore: one_member_abstracts
+abstract class DHTTruncate {
+  /// Remove items from the DHT container to shrink its size to 'newLength'
+  /// Throws StateError if newLength < 0
+  Future<void> truncate(int newLength);
+}

packages/veilid_support/lib/dht_support/src/interfaces/interfaces.dart

@@ -1,4 +1,8 @@
+export 'dht_add.dart';
+export 'dht_clear.dart';
 export 'dht_closeable.dart';
+export 'dht_insert_remove.dart';
 export 'dht_random_read.dart';
 export 'dht_random_write.dart';
+export 'dht_truncate.dart';
 export 'exceptions.dart';

packages/veilid_support/lib/proto/dht.pb.dart

@@ -195,8 +195,109 @@ class DHTShortArray extends $pb.GeneratedMessage {
   $core.List<$core.int> get seqs => $_getList(2);
 }
 
+class DHTDataReference extends $pb.GeneratedMessage {
+  factory DHTDataReference() => create();
+  DHTDataReference._() : super();
+  factory DHTDataReference.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
+  factory DHTDataReference.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
+
+  static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DHTDataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
+    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: $0.TypedKey.create)
+    ..aOM<$0.TypedKey>(2, _omitFieldNames ? '' : 'hash', subBuilder: $0.TypedKey.create)
+    ..hasRequiredFields = false
+  ;
+
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
+  'Will be removed in next major version')
+  DHTDataReference clone() => DHTDataReference()..mergeFromMessage(this);
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
+  'Will be removed in next major version')
+  DHTDataReference copyWith(void Function(DHTDataReference) updates) => super.copyWith((message) => updates(message as DHTDataReference)) as DHTDataReference;
+
+  $pb.BuilderInfo get info_ => _i;
+
+  @$core.pragma('dart2js:noInline')
+  static DHTDataReference create() => DHTDataReference._();
+  DHTDataReference createEmptyInstance() => create();
+  static $pb.PbList<DHTDataReference> createRepeated() => $pb.PbList<DHTDataReference>();
+  @$core.pragma('dart2js:noInline')
+  static DHTDataReference getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<DHTDataReference>(create);
+  static DHTDataReference? _defaultInstance;
+
+  @$pb.TagNumber(1)
+  $0.TypedKey get dhtData => $_getN(0);
+  @$pb.TagNumber(1)
+  set dhtData($0.TypedKey v) { setField(1, v); }
+  @$pb.TagNumber(1)
+  $core.bool hasDhtData() => $_has(0);
+  @$pb.TagNumber(1)
+  void clearDhtData() => clearField(1);
+  @$pb.TagNumber(1)
+  $0.TypedKey ensureDhtData() => $_ensure(0);
+
+  @$pb.TagNumber(2)
+  $0.TypedKey get hash => $_getN(1);
+  @$pb.TagNumber(2)
+  set hash($0.TypedKey v) { setField(2, v); }
+  @$pb.TagNumber(2)
+  $core.bool hasHash() => $_has(1);
+  @$pb.TagNumber(2)
+  void clearHash() => clearField(2);
+  @$pb.TagNumber(2)
+  $0.TypedKey ensureHash() => $_ensure(1);
+}
+
+class BlockStoreDataReference extends $pb.GeneratedMessage {
+  factory BlockStoreDataReference() => create();
+  BlockStoreDataReference._() : super();
+  factory BlockStoreDataReference.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
+  factory BlockStoreDataReference.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
+
+  static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'BlockStoreDataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
+    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'block', subBuilder: $0.TypedKey.create)
+    ..hasRequiredFields = false
+  ;
+
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
+  'Will be removed in next major version')
+  BlockStoreDataReference clone() => BlockStoreDataReference()..mergeFromMessage(this);
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
+  'Will be removed in next major version')
+  BlockStoreDataReference copyWith(void Function(BlockStoreDataReference) updates) => super.copyWith((message) => updates(message as BlockStoreDataReference)) as BlockStoreDataReference;
+
+  $pb.BuilderInfo get info_ => _i;
+
+  @$core.pragma('dart2js:noInline')
+  static BlockStoreDataReference create() => BlockStoreDataReference._();
+  BlockStoreDataReference createEmptyInstance() => create();
+  static $pb.PbList<BlockStoreDataReference> createRepeated() => $pb.PbList<BlockStoreDataReference>();
+  @$core.pragma('dart2js:noInline')
+  static BlockStoreDataReference getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<BlockStoreDataReference>(create);
+  static BlockStoreDataReference? _defaultInstance;
+
+  @$pb.TagNumber(1)
+  $0.TypedKey get block => $_getN(0);
+  @$pb.TagNumber(1)
+  set block($0.TypedKey v) { setField(1, v); }
+  @$pb.TagNumber(1)
+  $core.bool hasBlock() => $_has(0);
+  @$pb.TagNumber(1)
+  void clearBlock() => clearField(1);
+  @$pb.TagNumber(1)
+  $0.TypedKey ensureBlock() => $_ensure(0);
+}
+
 enum DataReference_Kind {
   dhtData, 
+  blockStoreData, 
   notSet
 }
 
@@ -208,11 +309,13 @@ class DataReference extends $pb.GeneratedMessage {
 
   static const $core.Map<$core.int, DataReference_Kind> _DataReference_KindByTag = {
     1 : DataReference_Kind.dhtData,
+    2 : DataReference_Kind.blockStoreData,
     0 : DataReference_Kind.notSet
   };
   static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
-    ..oo(0, [1])
-    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: $0.TypedKey.create)
+    ..oo(0, [1, 2])
+    ..aOM<DHTDataReference>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: DHTDataReference.create)
+    ..aOM<BlockStoreDataReference>(2, _omitFieldNames ? '' : 'blockStoreData', subBuilder: BlockStoreDataReference.create)
     ..hasRequiredFields = false
   ;
 
@@ -241,15 +344,26 @@ class DataReference extends $pb.GeneratedMessage {
   void clearKind() => clearField($_whichOneof(0));
 
   @$pb.TagNumber(1)
-  $0.TypedKey get dhtData => $_getN(0);
+  DHTDataReference get dhtData => $_getN(0);
   @$pb.TagNumber(1)
-  set dhtData($0.TypedKey v) { setField(1, v); }
+  set dhtData(DHTDataReference v) { setField(1, v); }
   @$pb.TagNumber(1)
   $core.bool hasDhtData() => $_has(0);
   @$pb.TagNumber(1)
   void clearDhtData() => clearField(1);
   @$pb.TagNumber(1)
-  $0.TypedKey ensureDhtData() => $_ensure(0);
+  DHTDataReference ensureDhtData() => $_ensure(0);
+
+  @$pb.TagNumber(2)
+  BlockStoreDataReference get blockStoreData => $_getN(1);
+  @$pb.TagNumber(2)
+  set blockStoreData(BlockStoreDataReference v) { setField(2, v); }
+  @$pb.TagNumber(2)
+  $core.bool hasBlockStoreData() => $_has(1);
+  @$pb.TagNumber(2)
+  void clearBlockStoreData() => clearField(2);
+  @$pb.TagNumber(2)
+  BlockStoreDataReference ensureBlockStoreData() => $_ensure(1);
 }
 
 class OwnedDHTRecordPointer extends $pb.GeneratedMessage {

packages/veilid_support/lib/proto/dht.pbjson.dart

@@ -60,11 +60,39 @@ final $typed_data.Uint8List dHTShortArrayDescriptor = $convert.base64Decode(
     'Cg1ESFRTaG9ydEFycmF5EiQKBGtleXMYASADKAsyEC52ZWlsaWQuVHlwZWRLZXlSBGtleXMSFA'
     'oFaW5kZXgYAiABKAxSBWluZGV4EhIKBHNlcXMYAyADKA1SBHNlcXM=');
 
+@$core.Deprecated('Use dHTDataReferenceDescriptor instead')
+const DHTDataReference$json = {
+  '1': 'DHTDataReference',
+  '2': [
+    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'dhtData'},
+    {'1': 'hash', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'hash'},
+  ],
+};
+
+/// Descriptor for `DHTDataReference`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List dHTDataReferenceDescriptor = $convert.base64Decode(
+    'ChBESFREYXRhUmVmZXJlbmNlEisKCGRodF9kYXRhGAEgASgLMhAudmVpbGlkLlR5cGVkS2V5Ug'
+    'dkaHREYXRhEiQKBGhhc2gYAiABKAsyEC52ZWlsaWQuVHlwZWRLZXlSBGhhc2g=');
+
+@$core.Deprecated('Use blockStoreDataReferenceDescriptor instead')
+const BlockStoreDataReference$json = {
+  '1': 'BlockStoreDataReference',
+  '2': [
+    {'1': 'block', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'block'},
+  ],
+};
+
+/// Descriptor for `BlockStoreDataReference`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List blockStoreDataReferenceDescriptor = $convert.base64Decode(
+    'ChdCbG9ja1N0b3JlRGF0YVJlZmVyZW5jZRImCgVibG9jaxgBIAEoCzIQLnZlaWxpZC5UeXBlZE'
+    'tleVIFYmxvY2s=');
+
 @$core.Deprecated('Use dataReferenceDescriptor instead')
 const DataReference$json = {
   '1': 'DataReference',
   '2': [
-    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '9': 0, '10': 'dhtData'},
+    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.dht.DHTDataReference', '9': 0, '10': 'dhtData'},
+    {'1': 'block_store_data', '3': 2, '4': 1, '5': 11, '6': '.dht.BlockStoreDataReference', '9': 0, '10': 'blockStoreData'},
   ],
   '8': [
     {'1': 'kind'},
@@ -73,8 +101,9 @@ const DataReference$json = {
 
 /// Descriptor for `DataReference`. Decode as a `google.protobuf.DescriptorProto`.
 final $typed_data.Uint8List dataReferenceDescriptor = $convert.base64Decode(
-    'Cg1EYXRhUmVmZXJlbmNlEi0KCGRodF9kYXRhGAEgASgLMhAudmVpbGlkLlR5cGVkS2V5SABSB2'
-    'RodERhdGFCBgoEa2luZA==');
+    'Cg1EYXRhUmVmZXJlbmNlEjIKCGRodF9kYXRhGAEgASgLMhUuZGh0LkRIVERhdGFSZWZlcmVuY2'
+    'VIAFIHZGh0RGF0YRJIChBibG9ja19zdG9yZV9kYXRhGAIgASgLMhwuZGh0LkJsb2NrU3RvcmVE'
+    'YXRhUmVmZXJlbmNlSABSDmJsb2NrU3RvcmVEYXRhQgYKBGtpbmQ=');
 
 @$core.Deprecated('Use ownedDHTRecordPointerDescriptor instead')
 const OwnedDHTRecordPointer$json = {

packages/veilid_support/lib/src/identity.dart

@@ -125,13 +125,14 @@ extension IdentityMasterExtension on IdentityMaster {
   }
 
   Future<List<AccountRecordInfo>> readAccountsFromIdentity(
-      {required SharedSecret identitySecret,
-      required String accountKey}) async {
+      {required SecretKey identitySecret, required String accountKey}) async {
     // Read the identity key to get the account keys
     final pool = DHTRecordPool.instance;
 
-    final identityRecordCrypto = await DHTRecordCryptoPrivate.fromSecret(
-        identityRecordKey.kind, identitySecret);
+    final identityRecordCrypto =
+        await DHTRecordPool.privateCryptoFromTypedSecret(
+      TypedKey(kind: identityRecordKey.kind, value: identitySecret),
+    );
 
     late final List<AccountRecordInfo> accountRecordInfo;
     await (await pool.openRecordRead(identityRecordKey,
@@ -157,7 +158,7 @@ extension IdentityMasterExtension on IdentityMaster {
   /// Creates a new Account associated with master identity and store it in the
   /// identity key.
   Future<AccountRecordInfo> addAccountToIdentity<T extends GeneratedMessage>({
-    required SharedSecret identitySecret,
+    required SecretKey identitySecret,
     required String accountKey,
     required Future<T> Function(TypedKey parent) createAccountCallback,
     int maxAccounts = 1,
@@ -234,7 +235,7 @@ class IdentityMasterWithSecrets {
     return (await pool.createRecord(
             debugName:
                 'IdentityMasterWithSecrets::create::IdentityMasterRecord',
-            crypto: const DHTRecordCryptoPublic()))
+            crypto: const VeilidCryptoPublic()))
         .deleteScope((masterRec) async {
       veilidLoggy.debug('Creating identity record');
       // Identity record is private

packages/veilid_support/lib/src/online_element_state.dart

@@ -0,0 +1,12 @@
+import 'package:equatable/equatable.dart';
+import 'package:meta/meta.dart';
+
+@immutable
+class OnlineElementState<T> extends Equatable {
+  const OnlineElementState({required this.value, required this.isOffline});
+  final T value;
+  final bool isOffline;
+
+  @override
+  List<Object?> get props => [value, isOffline];
+}

packages/veilid_support/lib/src/table_db_array.dart

@@ -0,0 +1,802 @@
+import 'dart:async';
+import 'dart:math';
+import 'dart:typed_data';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:charcode/charcode.dart';
+import 'package:equatable/equatable.dart';
+import 'package:meta/meta.dart';
+import 'package:protobuf/protobuf.dart';
+
+import '../veilid_support.dart';
+
+@immutable
+class TableDBArrayUpdate extends Equatable {
+  const TableDBArrayUpdate(
+      {required this.headDelta, required this.tailDelta, required this.length})
+      : assert(length >= 0, 'should never have negative length');
+  final int headDelta;
+  final int tailDelta;
+  final int length;
+
+  @override
+  List<Object?> get props => [headDelta, tailDelta, length];
+}
+
+class _TableDBArrayBase {
+  _TableDBArrayBase({
+    required String table,
+    required VeilidCrypto crypto,
+  })  : _table = table,
+        _crypto = crypto {
+    _initWait.add(_init);
+  }
+
+  // static Future<TableDBArray> make({
+  //   required String table,
+  //   required VeilidCrypto crypto,
+  // }) async {
+  //   final out = TableDBArray(table: table, crypto: crypto);
+  //   await out._initWait();
+  //   return out;
+  // }
+
+  Future<void> initWait() async {
+    await _initWait();
+  }
+
+  Future<void> _init() async {
+    // Load the array details
+    await _mutex.protect(() async {
+      _tableDB = await Veilid.instance.openTableDB(_table, 1);
+      await _loadHead();
+      _initDone = true;
+    });
+  }
+
+  Future<void> close({bool delete = false}) async {
+    // Ensure the init finished
+    await _initWait();
+
+    // Allow multiple attempts to close
+    if (_open) {
+      await _mutex.protect(() async {
+        await _changeStream.close();
+        _tableDB.close();
+        _open = false;
+      });
+    }
+    if (delete) {
+      await Veilid.instance.deleteTableDB(_table);
+    }
+  }
+
+  Future<void> delete() async {
+    await _initWait();
+    if (_open) {
+      throw StateError('should be closed first');
+    }
+    await Veilid.instance.deleteTableDB(_table);
+  }
+
+  Future<StreamSubscription<void>> listen(
+          void Function(TableDBArrayUpdate) onChanged) async =>
+      _changeStream.stream.listen(onChanged);
+
+  ////////////////////////////////////////////////////////////
+  // Public interface
+
+  int get length {
+    if (!_open) {
+      throw StateError('not open');
+    }
+    if (!_initDone) {
+      throw StateError('not initialized');
+    }
+
+    return _length;
+  }
+
+  bool get isOpen => _open;
+
+  Future<void> _add(Uint8List value) async {
+    await _initWait();
+    return _writeTransaction((t) async => _addInner(t, value));
+  }
+
+  Future<void> _addAll(List<Uint8List> values) async {
+    await _initWait();
+    return _writeTransaction((t) async => _addAllInner(t, values));
+  }
+
+  Future<void> _insert(int pos, Uint8List value) async {
+    await _initWait();
+    return _writeTransaction((t) async => _insertInner(t, pos, value));
+  }
+
+  Future<void> _insertAll(int pos, List<Uint8List> values) async {
+    await _initWait();
+    return _writeTransaction((t) async => _insertAllInner(t, pos, values));
+  }
+
+  Future<Uint8List> _get(int pos) async {
+    await _initWait();
+    return _mutex.protect(() async {
+      if (!_open) {
+        throw StateError('not open');
+      }
+      return _getInner(pos);
+    });
+  }
+
+  Future<List<Uint8List>> _getRange(int start, [int? end]) async {
+    await _initWait();
+    return _mutex.protect(() async {
+      if (!_open) {
+        throw StateError('not open');
+      }
+      return _getRangeInner(start, end ?? _length);
+    });
+  }
+
+  Future<void> _remove(int pos, {Output<Uint8List>? out}) async {
+    await _initWait();
+    return _writeTransaction((t) async => _removeInner(t, pos, out: out));
+  }
+
+  Future<void> _removeRange(int start, int end,
+      {Output<List<Uint8List>>? out}) async {
+    await _initWait();
+    return _writeTransaction(
+        (t) async => _removeRangeInner(t, start, end, out: out));
+  }
+
+  Future<void> clear() async {
+    await _initWait();
+    return _writeTransaction((t) async {
+      final keys = await _tableDB.getKeys(0);
+      for (final key in keys) {
+        await t.delete(0, key);
+      }
+      _length = 0;
+      _nextFree = 0;
+      _maxEntry = 0;
+      _dirtyChunks.clear();
+      _chunkCache.clear();
+    });
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Inner interface
+
+  Future<void> _addInner(VeilidTableDBTransaction t, Uint8List value) async {
+    // Allocate an entry to store the value
+    final entry = await _allocateEntry();
+    await _storeEntry(t, entry, value);
+
+    // Put the entry in the index
+    final pos = _length;
+    _length++;
+    _tailDelta++;
+    await _setIndexEntry(pos, entry);
+  }
+
+  Future<void> _addAllInner(
+      VeilidTableDBTransaction t, List<Uint8List> values) async {
+    var pos = _length;
+    _length += values.length;
+    _tailDelta += values.length;
+    for (final value in values) {
+      // Allocate an entry to store the value
+      final entry = await _allocateEntry();
+      await _storeEntry(t, entry, value);
+
+      // Put the entry in the index
+      await _setIndexEntry(pos, entry);
+      pos++;
+    }
+  }
+
+  Future<void> _insertInner(
+      VeilidTableDBTransaction t, int pos, Uint8List value) async {
+    if (pos == _length) {
+      return _addInner(t, value);
+    }
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+    // Allocate an entry to store the value
+    final entry = await _allocateEntry();
+    await _storeEntry(t, entry, value);
+
+    // Put the entry in the index
+    await _insertIndexEntry(pos);
+    await _setIndexEntry(pos, entry);
+  }
+
+  Future<void> _insertAllInner(
+      VeilidTableDBTransaction t, int pos, List<Uint8List> values) async {
+    if (pos == _length) {
+      return _addAllInner(t, values);
+    }
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+    await _insertIndexEntries(pos, values.length);
+    for (final value in values) {
+      // Allocate an entry to store the value
+      final entry = await _allocateEntry();
+      await _storeEntry(t, entry, value);
+
+      // Put the entry in the index
+      await _setIndexEntry(pos, entry);
+      pos++;
+    }
+  }
+
+  Future<Uint8List> _getInner(int pos) async {
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+    final entry = await _getIndexEntry(pos);
+    return (await _loadEntry(entry))!;
+  }
+
+  Future<List<Uint8List>> _getRangeInner(int start, int end) async {
+    final length = end - start;
+    if (length < 0) {
+      throw StateError('length should not be negative');
+    }
+    if (start < 0 || start >= _length) {
+      throw IndexError.withLength(start, _length);
+    }
+    if (end > _length) {
+      throw IndexError.withLength(end, _length);
+    }
+
+    final out = <Uint8List>[];
+    const batchSize = 16;
+
+    for (var pos = start; pos < end;) {
+      var batchLen = min(batchSize, end - pos);
+      final dws = DelayedWaitSet<Uint8List>();
+      while (batchLen > 0) {
+        final entry = await _getIndexEntry(pos);
+        dws.add(() async => (await _loadEntry(entry))!);
+        pos++;
+        batchLen--;
+      }
+      final batchOut = await dws();
+      out.addAll(batchOut);
+    }
+
+    return out;
+  }
+
+  Future<void> _removeInner(VeilidTableDBTransaction t, int pos,
+      {Output<Uint8List>? out}) async {
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+
+    final entry = await _getIndexEntry(pos);
+    if (out != null) {
+      final value = (await _loadEntry(entry))!;
+      out.save(value);
+    }
+
+    await _freeEntry(t, entry);
+    await _removeIndexEntry(pos);
+  }
+
+  Future<void> _removeRangeInner(VeilidTableDBTransaction t, int start, int end,
+      {Output<List<Uint8List>>? out}) async {
+    final length = end - start;
+    if (length < 0) {
+      throw StateError('length should not be negative');
+    }
+    if (start < 0) {
+      throw IndexError.withLength(start, _length);
+    }
+    if (end > _length) {
+      throw IndexError.withLength(end, _length);
+    }
+
+    final outList = <Uint8List>[];
+    for (var pos = start; pos < end; pos++) {
+      final entry = await _getIndexEntry(pos);
+      if (out != null) {
+        final value = (await _loadEntry(entry))!;
+        outList.add(value);
+      }
+      await _freeEntry(t, entry);
+    }
+    if (out != null) {
+      out.save(outList);
+    }
+
+    await _removeIndexEntries(start, length);
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Private implementation
+
+  static final Uint8List _headKey = Uint8List.fromList([$_, $H, $E, $A, $D]);
+  static Uint8List _entryKey(int k) =>
+      (ByteData(4)..setUint32(0, k)).buffer.asUint8List();
+  static Uint8List _chunkKey(int n) =>
+      (ByteData(2)..setUint16(0, n)).buffer.asUint8List();
+
+  Future<T> _writeTransaction<T>(
+          Future<T> Function(VeilidTableDBTransaction) closure) async =>
+      _mutex.protect(() async {
+        if (!_open) {
+          throw StateError('not open');
+        }
+
+        final _oldLength = _length;
+        final _oldNextFree = _nextFree;
+        final _oldMaxEntry = _maxEntry;
+        final _oldHeadDelta = _headDelta;
+        final _oldTailDelta = _tailDelta;
+        try {
+          final out = await transactionScope(_tableDB, (t) async {
+            final out = await closure(t);
+            await _saveHead(t);
+            await _flushDirtyChunks(t);
+            // Send change
+            _changeStream.add(TableDBArrayUpdate(
+                headDelta: _headDelta, tailDelta: _tailDelta, length: _length));
+            _headDelta = 0;
+            _tailDelta = 0;
+            return out;
+          });
+
+          return out;
+        } on Exception {
+          // restore head
+          _length = _oldLength;
+          _nextFree = _oldNextFree;
+          _maxEntry = _oldMaxEntry;
+          _headDelta = _oldHeadDelta;
+          _tailDelta = _oldTailDelta;
+          // invalidate caches because they could have been written to
+          _chunkCache.clear();
+          _dirtyChunks.clear();
+          // propagate exception
+          rethrow;
+        }
+      });
+
+  Future<void> _storeEntry(
+          VeilidTableDBTransaction t, int entry, Uint8List value) async =>
+      t.store(0, _entryKey(entry), await _crypto.encrypt(value));
+
+  Future<Uint8List?> _loadEntry(int entry) async {
+    final encryptedValue = await _tableDB.load(0, _entryKey(entry));
+    return (encryptedValue == null)
+        ? null
+        : await _crypto.decrypt(encryptedValue);
+  }
+
+  Future<int> _getIndexEntry(int pos) async {
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+    final chunkNumber = pos ~/ _indexStride;
+    final chunkOffset = pos % _indexStride;
+
+    final chunk = await _loadIndexChunk(chunkNumber);
+
+    return chunk.buffer.asByteData().getUint32(chunkOffset * 4);
+  }
+
+  Future<void> _setIndexEntry(int pos, int entry) async {
+    if (pos < 0 || pos >= _length) {
+      throw IndexError.withLength(pos, _length);
+    }
+
+    final chunkNumber = pos ~/ _indexStride;
+    final chunkOffset = pos % _indexStride;
+
+    final chunk = await _loadIndexChunk(chunkNumber);
+    chunk.buffer.asByteData().setUint32(chunkOffset * 4, entry);
+
+    _dirtyChunks[chunkNumber] = chunk;
+  }
+
+  Future<void> _insertIndexEntry(int pos) async => _insertIndexEntries(pos, 1);
+
+  Future<void> _insertIndexEntries(int start, int length) async {
+    if (length == 0) {
+      return;
+    }
+    if (length < 0) {
+      throw StateError('length should not be negative');
+    }
+    if (start < 0 || start >= _length) {
+      throw IndexError.withLength(start, _length);
+    }
+
+    // Slide everything over in reverse
+    var src = _length - 1;
+    var dest = src + length;
+
+    (int, Uint8List)? lastSrcChunk;
+    (int, Uint8List)? lastDestChunk;
+    while (src >= start) {
+      final remaining = (src - start) + 1;
+      final srcChunkNumber = src ~/ _indexStride;
+      final srcIndex = src % _indexStride;
+      final srcLength = min(remaining, srcIndex + 1);
+
+      final srcChunk =
+          (lastSrcChunk != null && (lastSrcChunk.$1 == srcChunkNumber))
+              ? lastSrcChunk.$2
+              : await _loadIndexChunk(srcChunkNumber);
+      _dirtyChunks[srcChunkNumber] = srcChunk;
+      lastSrcChunk = (srcChunkNumber, srcChunk);
+
+      final destChunkNumber = dest ~/ _indexStride;
+      final destIndex = dest % _indexStride;
+      final destLength = min(remaining, destIndex + 1);
+
+      final destChunk =
+          (lastDestChunk != null && (lastDestChunk.$1 == destChunkNumber))
+              ? lastDestChunk.$2
+              : await _loadIndexChunk(destChunkNumber);
+      _dirtyChunks[destChunkNumber] = destChunk;
+      lastDestChunk = (destChunkNumber, destChunk);
+
+      final toCopy = min(srcLength, destLength);
+      destChunk.setRange((destIndex - (toCopy - 1)) * 4, (destIndex + 1) * 4,
+          srcChunk, (srcIndex - (toCopy - 1)) * 4);
+
+      dest -= toCopy;
+      src -= toCopy;
+    }
+
+    // Then add to length
+    _length += length;
+    if (start == 0) {
+      _headDelta += length;
+    }
+    _tailDelta += length;
+  }
+
+  Future<void> _removeIndexEntry(int pos) async => _removeIndexEntries(pos, 1);
+
+  Future<void> _removeIndexEntries(int start, int length) async {
+    if (length == 0) {
+      return;
+    }
+    if (length < 0) {
+      throw StateError('length should not be negative');
+    }
+    if (start < 0 || start >= _length) {
+      throw IndexError.withLength(start, _length);
+    }
+    final end = start + length - 1;
+    if (end < 0 || end >= _length) {
+      throw IndexError.withLength(end, _length);
+    }
+
+    // Slide everything over
+    var dest = start;
+    var src = end + 1;
+    (int, Uint8List)? lastSrcChunk;
+    (int, Uint8List)? lastDestChunk;
+    while (src < _length) {
+      final srcChunkNumber = src ~/ _indexStride;
+      final srcIndex = src % _indexStride;
+      final srcLength = _indexStride - srcIndex;
+
+      final srcChunk =
+          (lastSrcChunk != null && (lastSrcChunk.$1 == srcChunkNumber))
+              ? lastSrcChunk.$2
+              : await _loadIndexChunk(srcChunkNumber);
+      _dirtyChunks[srcChunkNumber] = srcChunk;
+      lastSrcChunk = (srcChunkNumber, srcChunk);
+
+      final destChunkNumber = dest ~/ _indexStride;
+      final destIndex = dest % _indexStride;
+      final destLength = _indexStride - destIndex;
+
+      final destChunk =
+          (lastDestChunk != null && (lastDestChunk.$1 == destChunkNumber))
+              ? lastDestChunk.$2
+              : await _loadIndexChunk(destChunkNumber);
+      _dirtyChunks[destChunkNumber] = destChunk;
+      lastDestChunk = (destChunkNumber, destChunk);
+
+      final toCopy = min(srcLength, destLength);
+      destChunk.setRange(
+          destIndex * 4, (destIndex + toCopy) * 4, srcChunk, srcIndex * 4);
+
+      dest += toCopy;
+      src += toCopy;
+    }
+
+    // Then truncate
+    _length -= length;
+    if (start == 0) {
+      _headDelta -= length;
+    }
+    _tailDelta -= length;
+  }
+
+  Future<Uint8List> _loadIndexChunk(int chunkNumber) async {
+    // Get it from the dirty chunks if we have it
+    final dirtyChunk = _dirtyChunks[chunkNumber];
+    if (dirtyChunk != null) {
+      return dirtyChunk;
+    }
+
+    // Get from cache if we have it
+    for (var i = 0; i < _chunkCache.length; i++) {
+      if (_chunkCache[i].$1 == chunkNumber) {
+        // Touch the element
+        final x = _chunkCache.removeAt(i);
+        _chunkCache.add(x);
+        // Return the chunk for this position
+        return x.$2;
+      }
+    }
+
+    // Get chunk from disk
+    var chunk = await _tableDB.load(0, _chunkKey(chunkNumber));
+    chunk ??= Uint8List(_indexStride * 4);
+
+    // Cache the chunk
+    _chunkCache.add((chunkNumber, chunk));
+    if (_chunkCache.length > _chunkCacheLength) {
+      // Trim the LRU cache
+      final (_, _) = _chunkCache.removeAt(0);
+    }
+
+    return chunk;
+  }
+
+  Future<void> _flushDirtyChunks(VeilidTableDBTransaction t) async {
+    for (final ec in _dirtyChunks.entries) {
+      await t.store(0, _chunkKey(ec.key), ec.value);
+    }
+    _dirtyChunks.clear();
+  }
+
+  Future<void> _loadHead() async {
+    assert(_mutex.isLocked, 'should be locked');
+    final headBytes = await _tableDB.load(0, _headKey);
+    if (headBytes == null) {
+      _length = 0;
+      _nextFree = 0;
+      _maxEntry = 0;
+    } else {
+      final b = headBytes.buffer.asByteData();
+      _length = b.getUint32(0);
+      _nextFree = b.getUint32(4);
+      _maxEntry = b.getUint32(8);
+    }
+  }
+
+  Future<void> _saveHead(VeilidTableDBTransaction t) async {
+    assert(_mutex.isLocked, 'should be locked');
+    final b = ByteData(12)
+      ..setUint32(0, _length)
+      ..setUint32(4, _nextFree)
+      ..setUint32(8, _maxEntry);
+    await t.store(0, _headKey, b.buffer.asUint8List());
+  }
+
+  Future<int> _allocateEntry() async {
+    assert(_mutex.isLocked, 'should be locked');
+    if (_nextFree == 0) {
+      return _maxEntry++;
+    }
+    // pop endogenous free list
+    final free = _nextFree;
+    final nextFreeBytes = await _tableDB.load(0, _entryKey(free));
+    _nextFree = nextFreeBytes!.buffer.asByteData().getUint8(0);
+    return free;
+  }
+
+  Future<void> _freeEntry(VeilidTableDBTransaction t, int entry) async {
+    assert(_mutex.isLocked, 'should be locked');
+    // push endogenous free list
+    final b = ByteData(4)..setUint32(0, _nextFree);
+    await t.store(0, _entryKey(entry), b.buffer.asUint8List());
+    _nextFree = entry;
+  }
+
+  final String _table;
+  late final VeilidTableDB _tableDB;
+  var _open = true;
+  var _initDone = false;
+  final VeilidCrypto _crypto;
+  final WaitSet<void> _initWait = WaitSet();
+  final Mutex _mutex = Mutex();
+
+  // Change tracking
+  int _headDelta = 0;
+  int _tailDelta = 0;
+
+  // Head state
+  int _length = 0;
+  int _nextFree = 0;
+  int _maxEntry = 0;
+  static const int _indexStride = 16384;
+  final List<(int, Uint8List)> _chunkCache = [];
+  final Map<int, Uint8List> _dirtyChunks = {};
+  static const int _chunkCacheLength = 3;
+
+  final StreamController<TableDBArrayUpdate> _changeStream =
+      StreamController.broadcast();
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+class TableDBArray extends _TableDBArrayBase {
+  TableDBArray({
+    required super.table,
+    required super.crypto,
+  });
+
+  static Future<TableDBArray> make({
+    required String table,
+    required VeilidCrypto crypto,
+  }) async {
+    final out = TableDBArray(table: table, crypto: crypto);
+    await out._initWait();
+    return out;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Public interface
+
+  Future<void> add(Uint8List value) => _add(value);
+
+  Future<void> addAll(List<Uint8List> values) => _addAll(values);
+
+  Future<void> insert(int pos, Uint8List value) => _insert(pos, value);
+
+  Future<void> insertAll(int pos, List<Uint8List> values) =>
+      _insertAll(pos, values);
+
+  Future<Uint8List?> get(
+    int pos,
+  ) =>
+      _get(pos);
+
+  Future<List<Uint8List>> getRange(int start, [int? end]) =>
+      _getRange(start, end);
+
+  Future<void> remove(int pos, {Output<Uint8List>? out}) =>
+      _remove(pos, out: out);
+
+  Future<void> removeRange(int start, int end,
+          {Output<List<Uint8List>>? out}) =>
+      _removeRange(start, end, out: out);
+}
+//////////////////////////////////////////////////////////////////////////////
+
+class TableDBArrayJson<T> extends _TableDBArrayBase {
+  TableDBArrayJson(
+      {required super.table,
+      required super.crypto,
+      required T Function(dynamic) fromJson})
+      : _fromJson = fromJson;
+
+  static Future<TableDBArrayJson<T>> make<T>(
+      {required String table,
+      required VeilidCrypto crypto,
+      required T Function(dynamic) fromJson}) async {
+    final out =
+        TableDBArrayJson<T>(table: table, crypto: crypto, fromJson: fromJson);
+    await out._initWait();
+    return out;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Public interface
+
+  Future<void> add(T value) => _add(jsonEncodeBytes(value));
+
+  Future<void> addAll(List<T> values) async =>
+      _addAll(values.map(jsonEncodeBytes).toList());
+
+  Future<void> insert(int pos, T value) async =>
+      _insert(pos, jsonEncodeBytes(value));
+
+  Future<void> insertAll(int pos, List<T> values) async =>
+      _insertAll(pos, values.map(jsonEncodeBytes).toList());
+
+  Future<T> get(
+    int pos,
+  ) =>
+      _get(pos).then((out) => jsonDecodeBytes(_fromJson, out));
+
+  Future<List<T>> getRange(int start, [int? end]) =>
+      _getRange(start, end).then((out) => out.map(_fromJson).toList());
+
+  Future<void> remove(int pos, {Output<T>? out}) async {
+    final outJson = (out != null) ? Output<Uint8List>() : null;
+    await _remove(pos, out: outJson);
+    if (outJson != null && outJson.value != null) {
+      out!.save(jsonDecodeBytes(_fromJson, outJson.value!));
+    }
+  }
+
+  Future<void> removeRange(int start, int end, {Output<List<T>>? out}) async {
+    final outJson = (out != null) ? Output<List<Uint8List>>() : null;
+    await _removeRange(start, end, out: outJson);
+    if (outJson != null && outJson.value != null) {
+      out!.save(
+          outJson.value!.map((x) => jsonDecodeBytes(_fromJson, x)).toList());
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  final T Function(dynamic) _fromJson;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+class TableDBArrayProtobuf<T extends GeneratedMessage>
+    extends _TableDBArrayBase {
+  TableDBArrayProtobuf(
+      {required super.table,
+      required super.crypto,
+      required T Function(List<int>) fromBuffer})
+      : _fromBuffer = fromBuffer;
+
+  static Future<TableDBArrayProtobuf<T>> make<T extends GeneratedMessage>(
+      {required String table,
+      required VeilidCrypto crypto,
+      required T Function(List<int>) fromBuffer}) async {
+    final out = TableDBArrayProtobuf<T>(
+        table: table, crypto: crypto, fromBuffer: fromBuffer);
+    await out._initWait();
+    return out;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Public interface
+
+  Future<void> add(T value) => _add(value.writeToBuffer());
+
+  Future<void> addAll(List<T> values) async =>
+      _addAll(values.map((x) => x.writeToBuffer()).toList());
+
+  Future<void> insert(int pos, T value) async =>
+      _insert(pos, value.writeToBuffer());
+
+  Future<void> insertAll(int pos, List<T> values) async =>
+      _insertAll(pos, values.map((x) => x.writeToBuffer()).toList());
+
+  Future<T> get(
+    int pos,
+  ) =>
+      _get(pos).then(_fromBuffer);
+
+  Future<List<T>> getRange(int start, [int? end]) =>
+      _getRange(start, end).then((out) => out.map(_fromBuffer).toList());
+
+  Future<void> remove(int pos, {Output<T>? out}) async {
+    final outProto = (out != null) ? Output<Uint8List>() : null;
+    await _remove(pos, out: outProto);
+    if (outProto != null && outProto.value != null) {
+      out!.save(_fromBuffer(outProto.value!));
+    }
+  }
+
+  Future<void> removeRange(int start, int end, {Output<List<T>>? out}) async {
+    final outProto = (out != null) ? Output<List<Uint8List>>() : null;
+    await _removeRange(start, end, out: outProto);
+    if (outProto != null && outProto.value != null) {
+      out!.save(outProto.value!.map(_fromBuffer).toList());
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  final T Function(List<int>) _fromBuffer;
+}

packages/veilid_support/lib/src/table_db_array_protobuf_cubit.dart

@@ -0,0 +1,197 @@
+import 'dart:async';
+
+import 'package:async_tools/async_tools.dart';
+import 'package:bloc/bloc.dart';
+import 'package:bloc_advanced_tools/bloc_advanced_tools.dart';
+import 'package:equatable/equatable.dart';
+import 'package:fast_immutable_collections/fast_immutable_collections.dart';
+import 'package:meta/meta.dart';
+import 'package:protobuf/protobuf.dart';
+
+import '../../../veilid_support.dart';
+
+@immutable
+class TableDBArrayProtobufStateData<T extends GeneratedMessage>
+    extends Equatable {
+  const TableDBArrayProtobufStateData(
+      {required this.windowElements,
+      required this.length,
+      required this.windowTail,
+      required this.windowCount,
+      required this.follow});
+  // The view of the elements in the dhtlog
+  // Span is from [tail-length, tail)
+  final IList<T> windowElements;
+  // The length of the entire array
+  final int length;
+  // One past the end of the last element
+  final int windowTail;
+  // The total number of elements to try to keep in 'elements'
+  final int windowCount;
+  // If we should have the tail following the array
+  final bool follow;
+
+  @override
+  List<Object?> get props => [windowElements, windowTail, windowCount, follow];
+}
+
+typedef TableDBArrayProtobufState<T extends GeneratedMessage>
+    = AsyncValue<TableDBArrayProtobufStateData<T>>;
+typedef TableDBArrayProtobufBusyState<T extends GeneratedMessage>
+    = BlocBusyState<TableDBArrayProtobufState<T>>;
+
+class TableDBArrayProtobufCubit<T extends GeneratedMessage>
+    extends Cubit<TableDBArrayProtobufBusyState<T>>
+    with BlocBusyWrapper<TableDBArrayProtobufState<T>> {
+  TableDBArrayProtobufCubit({
+    required Future<TableDBArrayProtobuf<T>> Function() open,
+  }) : super(const BlocBusyState(AsyncValue.loading())) {
+    _initWait.add(() async {
+      // Open table db array
+      _array = await open();
+      _wantsCloseArray = true;
+
+      // Make initial state update
+      await _refreshNoWait();
+      _subscription = await _array.listen(_update);
+    });
+  }
+
+  // Set the tail position of the array for pagination.
+  // If tail is 0, the end of the array is used.
+  // If tail is negative, the position is subtracted from the current array
+  // length.
+  // If tail is positive, the position is absolute from the head of the array
+  // If follow is enabled, the tail offset will update when the array changes
+  Future<void> setWindow(
+      {int? tail, int? count, bool? follow, bool forceRefresh = false}) async {
+    await _initWait();
+    if (tail != null) {
+      _tail = tail;
+    }
+    if (count != null) {
+      _count = count;
+    }
+    if (follow != null) {
+      _follow = follow;
+    }
+    await _refreshNoWait(forceRefresh: forceRefresh);
+  }
+
+  Future<void> refresh({bool forceRefresh = false}) async {
+    await _initWait();
+    await _refreshNoWait(forceRefresh: forceRefresh);
+  }
+
+  Future<void> _refreshNoWait({bool forceRefresh = false}) async =>
+      busy((emit) async => _refreshInner(emit, forceRefresh: forceRefresh));
+
+  Future<void> _refreshInner(
+      void Function(AsyncValue<TableDBArrayProtobufStateData<T>>) emit,
+      {bool forceRefresh = false}) async {
+    final avElements = await _loadElements(_tail, _count);
+    final err = avElements.asError;
+    if (err != null) {
+      emit(AsyncValue.error(err.error, err.stackTrace));
+      return;
+    }
+    final loading = avElements.asLoading;
+    if (loading != null) {
+      emit(const AsyncValue.loading());
+      return;
+    }
+    final elements = avElements.asData!.value;
+    emit(AsyncValue.data(TableDBArrayProtobufStateData(
+        windowElements: elements,
+        length: _array.length,
+        windowTail: _tail,
+        windowCount: _count,
+        follow: _follow)));
+  }
+
+  Future<AsyncValue<IList<T>>> _loadElements(
+    int tail,
+    int count,
+  ) async {
+    try {
+      final length = _array.length;
+      if (length == 0) {
+        return AsyncValue.data(IList<T>.empty());
+      }
+      final end = ((tail - 1) % length) + 1;
+      final start = (count < end) ? end - count : 0;
+      final allItems = (await _array.getRange(start, end)).toIList();
+      return AsyncValue.data(allItems);
+    } on Exception catch (e, st) {
+      return AsyncValue.error(e, st);
+    }
+  }
+
+  void _update(TableDBArrayUpdate upd) {
+    // Run at most one background update process
+    // Because this is async, we could get an update while we're
+    // still processing the last one. Only called after init future has run
+    // so we dont have to wait for that here.
+
+    // Accumulate head and tail deltas
+    _headDelta += upd.headDelta;
+    _tailDelta += upd.tailDelta;
+
+    _sspUpdate.busyUpdate<T, TableDBArrayProtobufState<T>>(busy, (emit) async {
+      // apply follow
+      if (_follow) {
+        if (_tail <= 0) {
+          // Negative tail is already following tail changes
+        } else {
+          // Positive tail is measured from the head, so apply deltas
+          _tail = (_tail + _tailDelta - _headDelta) % upd.length;
+        }
+      } else {
+        if (_tail <= 0) {
+          // Negative tail is following tail changes so apply deltas
+          var posTail = _tail + upd.length;
+          posTail = (posTail + _tailDelta - _headDelta) % upd.length;
+          _tail = posTail - upd.length;
+        } else {
+          // Positive tail is measured from head so not following tail
+        }
+      }
+      _headDelta = 0;
+      _tailDelta = 0;
+
+      await _refreshInner(emit);
+    });
+  }
+
+  @override
+  Future<void> close() async {
+    await _initWait();
+    await _subscription?.cancel();
+    _subscription = null;
+    if (_wantsCloseArray) {
+      await _array.close();
+    }
+    await super.close();
+  }
+
+  Future<R?> operate<R>(
+      Future<R?> Function(TableDBArrayProtobuf<T>) closure) async {
+    await _initWait();
+    return closure(_array);
+  }
+
+  final WaitSet<void> _initWait = WaitSet();
+  late final TableDBArrayProtobuf<T> _array;
+  StreamSubscription<void>? _subscription;
+  bool _wantsCloseArray = false;
+  final _sspUpdate = SingleStatelessProcessor();
+
+  // Accumulated deltas since last update
+  var _headDelta = 0;
+  var _tailDelta = 0;
+
+  // Cubit window into the TableDBArray
+  var _tail = 0;
+  var _count = DHTShortArray.maxElements;
+  var _follow = true;
+}

packages/veilid_support/lib/src/veilid_crypto.dart

@@ -0,0 +1,62 @@
+import 'dart:async';
+import 'dart:convert';
+import 'dart:typed_data';
+import '../../../veilid_support.dart';
+
+abstract class VeilidCrypto {
+  Future<Uint8List> encrypt(Uint8List data);
+  Future<Uint8List> decrypt(Uint8List data);
+}
+
+////////////////////////////////////
+/// Encrypted for a specific symmetric key
+class VeilidCryptoPrivate implements VeilidCrypto {
+  VeilidCryptoPrivate._(VeilidCryptoSystem cryptoSystem, SharedSecret secretKey)
+      : _cryptoSystem = cryptoSystem,
+        _secret = secretKey;
+  final VeilidCryptoSystem _cryptoSystem;
+  final SharedSecret _secret;
+
+  static Future<VeilidCryptoPrivate> fromTypedKey(
+      TypedKey typedSecret, String domain) async {
+    final cryptoSystem =
+        await Veilid.instance.getCryptoSystem(typedSecret.kind);
+    final keyMaterial = Uint8List.fromList(
+        [...typedSecret.value.decode(), ...utf8.encode(domain)]);
+    final secretKey = await cryptoSystem.generateHash(keyMaterial);
+    return VeilidCryptoPrivate._(cryptoSystem, secretKey);
+  }
+
+  static Future<VeilidCryptoPrivate> fromTypedKeyPair(
+      TypedKeyPair typedKeyPair, String domain) async {
+    final typedSecret =
+        TypedKey(kind: typedKeyPair.kind, value: typedKeyPair.secret);
+    return fromTypedKey(typedSecret, domain);
+  }
+
+  static Future<VeilidCryptoPrivate> fromSharedSecret(
+      CryptoKind kind, SharedSecret sharedSecret) async {
+    final cryptoSystem = await Veilid.instance.getCryptoSystem(kind);
+    return VeilidCryptoPrivate._(cryptoSystem, sharedSecret);
+  }
+
+  @override
+  Future<Uint8List> encrypt(Uint8List data) =>
+      _cryptoSystem.encryptNoAuthWithNonce(data, _secret);
+
+  @override
+  Future<Uint8List> decrypt(Uint8List data) =>
+      _cryptoSystem.decryptNoAuthWithNonce(data, _secret);
+}
+
+////////////////////////////////////
+/// No encryption
+class VeilidCryptoPublic implements VeilidCrypto {
+  const VeilidCryptoPublic();
+
+  @override
+  Future<Uint8List> encrypt(Uint8List data) async => data;
+
+  @override
+  Future<Uint8List> decrypt(Uint8List data) async => data;
+}

packages/veilid_support/lib/veilid_support.dart

@@ -10,8 +10,12 @@ export 'src/config.dart';
 export 'src/identity.dart';
 export 'src/json_tools.dart';
 export 'src/memory_tools.dart';
+export 'src/online_element_state.dart';
 export 'src/output.dart';
 export 'src/persistent_queue.dart';
 export 'src/protobuf_tools.dart';
 export 'src/table_db.dart';
+export 'src/table_db_array.dart';
+export 'src/table_db_array_protobuf_cubit.dart';
+export 'src/veilid_crypto.dart';
 export 'src/veilid_log.dart' hide veilidLoggy;

packages/veilid_support/pubspec.lock

@@ -36,10 +36,9 @@ packages:
   async_tools:
     dependency: "direct main"
     description:
-      name: async_tools
-      sha256: e783ac6ed5645c86da34240389bb3a000fc5e3ae6589c6a482eb24ece7217681
-      url: "https://pub.dev"
-    source: hosted
+      path: "../../../dart_async_tools"
+      relative: true
+    source: path
     version: "0.1.1"
   bloc:
     dependency: "direct main"
@@ -52,10 +51,9 @@ packages:
   bloc_advanced_tools:
     dependency: "direct main"
     description:
-      name: bloc_advanced_tools
-      sha256: "09f8a121d950575f1f2980c8b10df46b2ac6c72c8cbe48cc145871e5882ed430"
-      url: "https://pub.dev"
-    source: hosted
+      path: "../../../bloc_advanced_tools"
+      relative: true
+    source: path
     version: "0.1.1"
   boolean_selector:
     dependency: transitive
@@ -146,7 +144,7 @@ packages:
     source: hosted
     version: "1.3.0"
   charcode:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: charcode
       sha256: fb98c0f6d12c920a02ee2d998da788bca066ca5f148492b7085ee23372b12306

packages/veilid_support/pubspec.yaml

@@ -10,6 +10,7 @@ dependencies:
   async_tools: ^0.1.1
   bloc: ^8.1.4
   bloc_advanced_tools: ^0.1.1
+  charcode: ^1.3.1
   collection: ^1.18.0
   equatable: ^2.0.5
   fast_immutable_collections: ^10.2.3
@@ -23,6 +24,12 @@ dependencies:
     #   veilid: ^0.0.1
     path: ../../../veilid/veilid-flutter
 
+dependency_overrides:
+  async_tools: 
+    path: ../../../dart_async_tools
+  bloc_advanced_tools: 
+    path: ../../../bloc_advanced_tools
+
 dev_dependencies:
   build_runner: ^2.4.10
   freezed: ^2.5.2

pubspec.lock

@@ -37,10 +37,10 @@ packages:
     dependency: "direct main"
     description:
       name: archive
-      sha256: ecf4273855368121b1caed0d10d4513c7241dfc813f7d3c8933b36622ae9b265
+      sha256: cb6a278ef2dbb298455e1a713bda08524a175630ec643a242c399c932a0a1f7d
       url: "https://pub.dev"
     source: hosted
-    version: "3.5.1"
+    version: "3.6.1"
   args:
     dependency: transitive
     description:
@@ -60,9 +60,10 @@ packages:
   async_tools:
     dependency: "direct main"
     description:
-      path: "../dart_async_tools"
-      relative: true
-    source: path
+      name: async_tools
+      sha256: e783ac6ed5645c86da34240389bb3a000fc5e3ae6589c6a482eb24ece7217681
+      url: "https://pub.dev"
+    source: hosted
     version: "0.1.1"
   awesome_extensions:
     dependency: "direct main"
@@ -99,10 +100,11 @@ packages:
   bloc_advanced_tools:
     dependency: "direct main"
     description:
-      path: "../bloc_advanced_tools"
-      relative: true
-    source: path
-    version: "0.1.1"
+      name: bloc_advanced_tools
+      sha256: "0cf9b3a73a67addfe22ec3f97a1ac240f6ad53870d6b21a980260f390d7901cd"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.1.2"
   blurry_modal_progress_hud:
     dependency: "direct main"
     description:
@@ -155,18 +157,18 @@ packages:
     dependency: "direct dev"
     description:
       name: build_runner
-      sha256: "1414d6d733a85d8ad2f1dfcb3ea7945759e35a123cb99ccfac75d0758f75edfa"
+      sha256: "644dc98a0f179b872f612d3eb627924b578897c629788e858157fa5e704ca0c7"
       url: "https://pub.dev"
     source: hosted
-    version: "2.4.10"
+    version: "2.4.11"
   build_runner_core:
     dependency: transitive
     description:
       name: build_runner_core
-      sha256: "4ae8ffe5ac758da294ecf1802f2aff01558d8b1b00616aa7538ea9a8a5d50799"
+      sha256: e3c79f69a64bdfcd8a776a3c28db4eb6e3fb5356d013ae5eb2e52007706d5dbe
       url: "https://pub.dev"
     source: hosted
-    version: "7.3.0"
+    version: "7.3.1"
   built_collection:
     dependency: transitive
     description:
@@ -219,10 +221,10 @@ packages:
     dependency: transitive
     description:
       name: camera_android
-      sha256: b350ac087f111467e705b2b76cc1322f7f5bdc122aa83b4b243b0872f390d229
+      sha256: "3af7f0b55f184d392d2eec238aaa30552ebeef2915e5e094f5488bf50d6d7ca2"
       url: "https://pub.dev"
     source: hosted
-    version: "0.10.9+2"
+    version: "0.10.9+3"
   camera_avfoundation:
     dependency: transitive
     description:
@@ -251,10 +253,10 @@ packages:
     dependency: "direct main"
     description:
       name: change_case
-      sha256: "47c48c36f95f20c6d0ba03efabceff261d05026cca322cc2c4c01c343371b5bb"
+      sha256: "99cfdf2018c627c8a3af5a23ea4c414eb69c75c31322d23b9660ebc3cf30b514"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "2.1.0"
   characters:
     dependency: transitive
     description:
@@ -403,10 +405,10 @@ packages:
     dependency: "direct main"
     description:
       name: fast_immutable_collections
-      sha256: "533806a7f0c624c2e479d05d3fdce4c87109a7cd0db39b8cc3830d3a2e8dedc7"
+      sha256: c3c73f4f989d3302066e4ec94e6ec73b5dc872592d02194f49f1352d64126b8c
       url: "https://pub.dev"
     source: hosted
-    version: "10.2.3"
+    version: "10.2.4"
   ffi:
     dependency: transitive
     description:
@@ -471,19 +473,20 @@ packages:
   flutter_chat_ui:
     dependency: "direct main"
     description:
-      name: flutter_chat_ui
-      sha256: "40fb37acc328dd179eadc3d67bf8bd2d950dc0da34464aa8d48e8707e0234c09"
-      url: "https://pub.dev"
-    source: hosted
+      path: "."
+      ref: main
+      resolved-ref: "6712d897e25041de38fb53ec06dc7a12cc6bff7d"
+      url: "https://gitlab.com/veilid/flutter-chat-ui.git"
+    source: git
     version: "1.6.13"
   flutter_form_builder:
     dependency: "direct main"
     description:
       name: flutter_form_builder
-      sha256: "560eb5e367d81170c6ade1e7ae63ecc5167936ae2cdfaae8a345e91bce19d2f2"
+      sha256: "447f8808f68070f7df968e8063aada3c9d2e90e789b5b70f3b44e4b315212656"
       url: "https://pub.dev"
     source: hosted
-    version: "9.2.1"
+    version: "9.3.0"
   flutter_hooks:
     dependency: "direct main"
     description:
@@ -533,10 +536,10 @@ packages:
     dependency: transitive
     description:
       name: flutter_plugin_android_lifecycle
-      sha256: "8cf40eebf5dec866a6d1956ad7b4f7016e6c0cc69847ab946833b7d43743809f"
+      sha256: c6b0b4c05c458e1c01ad9bcc14041dd7b1f6783d487be4386f793f47a8a4d03e
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.19"
+    version: "2.0.20"
   flutter_shaders:
     dependency: transitive
     description:
@@ -573,10 +576,10 @@ packages:
     dependency: "direct main"
     description:
       name: flutter_translate
-      sha256: "8b1c449bf6d17753e6f188185f735ebc0a328d21d745878a43be66857de8ebb3"
+      sha256: bc09db690098879e3f90eb3aac3499e5282f32d5f9d8f1cc597d67bbc1e065ef
       url: "https://pub.dev"
     source: hosted
-    version: "4.0.4"
+    version: "4.1.0"
   flutter_web_plugins:
     dependency: transitive
     description: flutter
@@ -586,10 +589,10 @@ packages:
     dependency: "direct main"
     description:
       name: form_builder_validators
-      sha256: "19aa5282b7cede82d0025ab031a98d0554b84aa2ba40f12013471a3b3e22bf02"
+      sha256: "475853a177bfc832ec12551f752fd0001278358a6d42d2364681ff15f48f67cf"
       url: "https://pub.dev"
     source: hosted
-    version: "9.1.0"
+    version: "10.0.1"
   freezed:
     dependency: "direct dev"
     description:
@@ -634,10 +637,10 @@ packages:
     dependency: "direct main"
     description:
       name: go_router
-      sha256: aa073287b8f43553678e6fa9e8bb9c83212ff76e09542129a8099bbc8db4df65
+      sha256: abec47eb8c8c36ebf41d0a4c64dbbe7f956e39a012b3aafc530e951bdc12fe3f
       url: "https://pub.dev"
     source: hosted
-    version: "14.1.2"
+    version: "14.1.4"
   graphs:
     dependency: transitive
     description:
@@ -706,10 +709,10 @@ packages:
     dependency: "direct main"
     description:
       name: image
-      sha256: "4c68bfd5ae83e700b5204c1e74451e7bf3cf750e6843c6e158289cf56bda018e"
+      sha256: "2237616a36c0d69aef7549ab439b833fb7f9fb9fc861af2cc9ac3eedddd69ca8"
       url: "https://pub.dev"
     source: hosted
-    version: "4.1.7"
+    version: "4.2.0"
   intl:
     dependency: "direct main"
     description:
@@ -826,10 +829,10 @@ packages:
     dependency: "direct main"
     description:
       name: motion_toast
-      sha256: "4763b2aa3499d0bf00ffd9737479b73141d0397f532542893156efb4a5ac1994"
+      sha256: "8dc8af93c606d0a08f2592591164f4a761099c5470e589f25689de6c601f124e"
       url: "https://pub.dev"
     source: hosted
-    version: "2.9.1"
+    version: "2.10.0"
   nested:
     dependency: transitive
     description:
@@ -890,10 +893,10 @@ packages:
     dependency: transitive
     description:
       name: path_provider_android
-      sha256: a248d8146ee5983446bf03ed5ea8f6533129a12b11f12057ad1b4a67a2b3b41d
+      sha256: "9c96da072b421e98183f9ea7464898428e764bc0ce5567f27ec8693442e72514"
       url: "https://pub.dev"
     source: hosted
-    version: "2.2.4"
+    version: "2.2.5"
   path_provider_foundation:
     dependency: transitive
     description:
@@ -1018,10 +1021,10 @@ packages:
     dependency: transitive
     description:
       name: pubspec_parse
-      sha256: c63b2876e58e194e4b0828fcb080ad0e06d051cb607a6be51a9e084f47cb9367
+      sha256: c799b721d79eb6ee6fa56f00c04b472dcd44a30d258fac2174a6ec57302678f8
       url: "https://pub.dev"
     source: hosted
-    version: "1.2.3"
+    version: "1.3.0"
   qr:
     dependency: transitive
     description:
@@ -1095,7 +1098,7 @@ packages:
     source: hosted
     version: "0.1.9"
   scroll_to_index:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: scroll_to_index
       sha256: b707546e7500d9f070d63e5acf74fd437ec7eeeb68d3412ef7b0afada0b4f176
@@ -1106,10 +1109,10 @@ packages:
     dependency: "direct main"
     description:
       name: searchable_listview
-      sha256: dfa6358f5e097f45b5b51a160cb6189e112e3abe0f728f4740349cd3b6575617
+      sha256: "5e547f2a3f88f99a798cfe64882cfce51496d8f3177bea4829dd7bcf3fa8910d"
       url: "https://pub.dev"
     source: hosted
-    version: "2.13.0"
+    version: "2.14.0"
   share_plus:
     dependency: "direct main"
     description:
@@ -1138,10 +1141,10 @@ packages:
     dependency: transitive
     description:
       name: shared_preferences_android
-      sha256: "1ee8bf911094a1b592de7ab29add6f826a7331fb854273d55918693d5364a1f2"
+      sha256: "93d0ec9dd902d85f326068e6a899487d1f65ffcd5798721a95330b26c8131577"
       url: "https://pub.dev"
     source: hosted
-    version: "2.2.2"
+    version: "2.2.3"
   shared_preferences_foundation:
     dependency: transitive
     description:
@@ -1219,6 +1222,15 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.0.0"
+  sorted_list:
+    dependency: "direct main"
+    description:
+      path: "."
+      ref: main
+      resolved-ref: "090eb9be48ab85ff064a0a1d8175b4a72d79b139"
+      url: "https://gitlab.com/veilid/dart-sorted-list-improved.git"
+    source: git
+    version: "1.0.0"
   source_gen:
     dependency: transitive
     description:
@@ -1383,26 +1395,26 @@ packages:
     dependency: transitive
     description:
       name: universal_platform
-      sha256: d315be0f6641898b280ffa34e2ddb14f3d12b1a37882557869646e0cc363d0cc
+      sha256: "64e16458a0ea9b99260ceb5467a214c1f298d647c659af1bff6d3bf82536b1ec"
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.0+1"
+    version: "1.1.0"
   url_launcher:
     dependency: transitive
     description:
       name: url_launcher
-      sha256: "6ce1e04375be4eed30548f10a315826fd933c1e493206eab82eed01f438c8d2e"
+      sha256: "21b704ce5fa560ea9f3b525b43601c678728ba46725bab9b01187b4831377ed3"
       url: "https://pub.dev"
     source: hosted
-    version: "6.2.6"
+    version: "6.3.0"
   url_launcher_android:
     dependency: transitive
     description:
       name: url_launcher_android
-      sha256: "17cd5e205ea615e2c6ea7a77323a11712dffa0720a8a90540db57a01347f9ad9"
+      sha256: ceb2625f0c24ade6ef6778d1de0b2e44f2db71fded235eb52295247feba8c5cf
       url: "https://pub.dev"
     source: hosted
-    version: "6.3.2"
+    version: "6.3.3"
   url_launcher_ios:
     dependency: transitive
     description:
@@ -1533,10 +1545,10 @@ packages:
     dependency: transitive
     description:
       name: web_socket
-      sha256: "217f49b5213796cb508d6a942a5dc604ce1cb6a0a6b3d8cb3f0c314f0ecea712"
+      sha256: "24301d8c293ce6fe327ffe6f59d8fd8834735f0ec36e4fd383ec7ff8a64aa078"
       url: "https://pub.dev"
     source: hosted
-    version: "0.1.4"
+    version: "0.1.5"
   web_socket_channel:
     dependency: transitive
     description:
@@ -1619,4 +1631,4 @@ packages:
     version: "1.1.2"
 sdks:
   dart: ">=3.4.0 <4.0.0"
-  flutter: ">=3.19.1"
+  flutter: ">=3.22.1"

pubspec.yaml

@@ -5,35 +5,38 @@ version: 0.2.0+10
 
 environment:
   sdk: '>=3.2.0 <4.0.0'  
-  flutter: '>=3.19.1'
+  flutter: '>=3.22.1'
 
 dependencies:
   animated_theme_switcher: ^2.0.10
   ansicolor: ^2.0.2
-  archive: ^3.5.1
+  archive: ^3.6.1
   async_tools: ^0.1.1
-  awesome_extensions: ^2.0.14
+  awesome_extensions: ^2.0.16
   badges: ^3.1.2
   basic_utils: ^5.7.0
   bloc: ^8.1.4
-  bloc_advanced_tools: ^0.1.1
+  bloc_advanced_tools: ^0.1.2
   blurry_modal_progress_hud: ^1.1.1
-  change_case: ^2.0.1
+  change_case: ^2.1.0
   charcode: ^1.3.1
   circular_profile_avatar: ^2.0.5
   circular_reveal_animation: ^2.0.1
   cool_dropdown: ^2.1.0
   cupertino_icons: ^1.0.8
   equatable: ^2.0.5
-  fast_immutable_collections: ^10.2.2
+  fast_immutable_collections: ^10.2.4
   fixnum: ^1.1.0
   flutter:
     sdk: flutter
   flutter_animate: ^4.5.0
   flutter_bloc: ^8.1.5
   flutter_chat_types: ^3.6.2
-  flutter_chat_ui: ^1.6.12
-  flutter_form_builder: ^9.2.1
+  flutter_chat_ui:
+    git:
+      url: https://gitlab.com/veilid/flutter-chat-ui.git
+      ref: main
+  flutter_form_builder: ^9.3.0
   flutter_hooks: ^0.20.5
   flutter_localizations:
     sdk: flutter
@@ -41,18 +44,18 @@ dependencies:
   flutter_slidable: ^3.1.0
   flutter_spinkit: ^5.2.1
   flutter_svg: ^2.0.10+1
-  flutter_translate: ^4.0.4
-  form_builder_validators: ^9.1.0
+  flutter_translate: ^4.1.0
+  form_builder_validators: ^10.0.1
   freezed_annotation: ^2.4.1
-  go_router: ^14.1.2
+  go_router: ^14.1.4
   hydrated_bloc: ^9.1.5
-  image: ^4.1.7
-  intl: ^0.18.1
+  image: ^4.2.0
+  intl: ^0.19.0
   json_annotation: ^4.9.0
   loggy: ^2.0.3
-  meta: ^1.11.0
+  meta: ^1.12.0
   mobile_scanner: ^5.1.1
-  motion_toast: ^2.9.1
+  motion_toast: ^2.10.0
   pasteboard: ^0.2.0
   path: ^1.9.0
   path_provider: ^2.1.3
@@ -65,10 +68,15 @@ dependencies:
   quickalert: ^1.1.0
   radix_colors: ^1.0.4
   reorderable_grid: ^1.0.10
-  searchable_listview: ^2.12.0
+  scroll_to_index: ^3.0.1
+  searchable_listview: ^2.14.0
   share_plus: ^9.0.0
   shared_preferences: ^2.2.3
   signal_strength_indicator: ^0.4.1
+  sorted_list:
+    git:
+      url: https://gitlab.com/veilid/dart-sorted-list-improved.git
+      ref: main
   split_view: ^3.2.1
   stack_trace: ^1.11.1
   stream_transform: ^2.1.0
@@ -79,21 +87,20 @@ dependencies:
     path: ../veilid/veilid-flutter
   veilid_support:
     path: packages/veilid_support
-  window_manager: ^0.3.8
+  window_manager: ^0.3.9
   xterm: ^4.0.0
   zxing2: ^0.2.3
 
-dependency_overrides:
-  async_tools: 
-    path: ../dart_async_tools
-  bloc_advanced_tools: 
-    path: ../bloc_advanced_tools
-  # REMOVE ONCE form_builder_validators HAS A FIX UPSTREAM
-  intl: 0.19.0   
-
+# dependency_overrides:
+#   async_tools: 
+#     path: ../dart_async_tools
+#   bloc_advanced_tools: 
+#     path: ../bloc_advanced_tools
+#   flutter_chat_ui:
+#     path: ../flutter_chat_ui
 
 dev_dependencies:
-  build_runner: ^2.4.9
+  build_runner: ^2.4.11
   freezed: ^2.5.2
   icons_launcher: ^2.1.7
   json_serializable: ^6.8.0