everything but reconcile

lib/chat/cubits/single_contact_messages_cubit.dart

@@ -16,7 +16,7 @@ class RenderStateElement {
   RenderStateElement(
       {required this.message,
       required this.isLocal,
-      this.reconciled = false,
+      this.reconciledTimestamp,
       this.sent = false,
       this.sentOffline = false});
 
@@ -28,7 +28,7 @@ class RenderStateElement {
     if (sent && !sentOffline) {
       return MessageSendState.delivered;
     }
-    if (reconciled) {
+    if (reconciledTimestamp != null) {
       return MessageSendState.sent;
     }
     return MessageSendState.sending;
@@ -36,7 +36,7 @@ class RenderStateElement {
 
   proto.Message message;
   bool isLocal;
-  bool reconciled;
+  Timestamp? reconciledTimestamp;
   bool sent;
   bool sentOffline;
 }
@@ -68,7 +68,6 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   Future<void> close() async {
     await _initWait();
 
-    await _unreconciledMessagesQueue.close();
     await _sendingMessagesQueue.close();
     await _sentSubscription?.cancel();
     await _rcvdSubscription?.cancel();
@@ -81,13 +80,6 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
   // Initialize everything
   Future<void> _init() async {
-    // Late initialization of queues with closures
-    _unreconciledMessagesQueue = PersistentQueue<proto.Message>(
-      table: 'SingleContactUnreconciledMessages',
-      key: _remoteConversationRecordKey.toString(),
-      fromBuffer: proto.Message.fromBuffer,
-      closure: _processUnreconciledMessages,
-    );
     _sendingMessagesQueue = PersistentQueue<proto.Message>(
       table: 'SingleContactSendingMessages',
       key: _remoteConversationRecordKey.toString(),
@@ -160,13 +152,14 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
     _reconciledMessagesCubit = TableDBArrayCubit(
         open: () async => TableDBArray.make(table: tableName, crypto: crypto),
-        decodeElement: proto.Message.fromBuffer);
+        decodeElement: proto.ReconciledMessage.fromBuffer);
     _reconciledSubscription =
         _reconciledMessagesCubit!.stream.listen(_updateReconciledMessagesState);
     _updateReconciledMessagesState(_reconciledMessagesCubit!.state);
   }
 
   ////////////////////////////////////////////////////////////////////////////
+  // Public interface
 
   // Set the tail position of the log for pagination.
   // If tail is 0, the end of the log is used.
@@ -181,7 +174,14 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         tail: tail, count: count, follow: follow, forceRefresh: forceRefresh);
   }
 
+  // Set a user-visible 'text' message with possible attachments
+  void sendTextMessage({required proto.Message_Text messageText}) {
+    final message = proto.Message()..text = messageText;
+    _sendMessage(message: message);
+  }
+
   ////////////////////////////////////////////////////////////////////////////
+  // Internal implementation
 
   // Called when the sent messages cubit gets a change
   // This will re-render when messages are sent from another machine
@@ -191,10 +191,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
       return;
     }
 
-    await _reconcileMessages(sentMessages, _sentMessagesCubit);
+    _reconcileMessages(sentMessages, _sentMessagesCubit!);
-
-    // Update the view
-    _renderState();
   }
 
   // Called when the received messages cubit gets a change
@@ -204,61 +201,16 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
       return;
     }
 
-    await _reconcileMessages(rcvdMessages, _rcvdMessagesCubit);
+    _reconcileMessages(rcvdMessages, _rcvdMessagesCubit!);
-
-    singleFuture(_rcvdMessagesCubit!, () async {
-      // Get the timestamp of our most recent reconciled message
-      final lastReconciledMessageTs =
-          await _reconciledMessagesCubit!.operate((arr) async {
-        final len = arr.length;
-        if (len == 0) {
-          return null;
-        } else {
-          final lastMessage =
-              await arr.getProtobuf(proto.Message.fromBuffer, len - 1);
-          if (lastMessage == null) {
-            throw StateError('should have gotten last message');
-          }
-          return lastMessage.timestamp;
-        }
-      });
-
-      // Find oldest message we have not yet reconciled
-
-      // // Go through all the ones from the cubit state first since we've already
-      // // gotten them from the DHT
-      // for (var rn = rcvdMessages.elements.length; rn >= 0; rn--) {
-      //   //
-      // }
-
-      // // Add remote messages updates to queue to process asynchronously
-      // // Ignore offline state because remote messages are always fully delivered
-      // // This may happen once per client but should be idempotent
-      // _unreconciledMessagesQueue.addAllSync(rcvdMessages.map((x) => x.value));
-
-      // Update the view
-      _renderState();
-    });
   }
 
   // Called when the reconciled messages window gets a change
   void _updateReconciledMessagesState(
-      TableDBArrayBusyState<proto.Message> avmessages) {
+      TableDBArrayBusyState<proto.ReconciledMessage> avmessages) {
     // Update the view
     _renderState();
   }
 
-  // Async process to reconcile messages sent or received in the background
-  Future<void> _processUnreconciledMessages(
-      IList<proto.Message> messages) async {
-    // await _reconciledMessagesCubit!
-    //     .operateAppendEventual((reconciledMessagesWriter) async {
-    //   await _reconcileMessagesInner(
-    //       reconciledMessagesWriter: reconciledMessagesWriter,
-    //       messages: messages);
-    // });
-  }
-
   Future<Uint8List> _hashSignature(proto.Signature signature) async =>
       (await _localMessagesCryptoSystem
               .generateHash(signature.toVeilid().decode()))
@@ -317,6 +269,43 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         writer.tryAddAll(messages.map((m) => m.writeToBuffer()).toList()));
   }
 
+  void _reconcileMessages(DHTLogStateData<proto.Message> inputMessages,
+      DHTLogCubit<proto.Message> inputMessagesCubit) {
+    singleFuture(_reconciledMessagesCubit!, () async {
+      // Get the timestamp of our most recent reconciled message
+      final lastReconciledMessageTs =
+          await _reconciledMessagesCubit!.operate((arr) async {
+        final len = arr.length;
+        if (len == 0) {
+          return null;
+        } else {
+          final lastMessage =
+              await arr.getProtobuf(proto.Message.fromBuffer, len - 1);
+          if (lastMessage == null) {
+            throw StateError('should have gotten last message');
+          }
+          return lastMessage.timestamp;
+        }
+      });
+
+      // Find oldest message we have not yet reconciled
+
+      // // Go through all the ones from the cubit state first since we've already
+      // // gotten them from the DHT
+      // for (var rn = rcvdMessages.elements.length; rn >= 0; rn--) {
+      //   //
+      // }
+
+      // // Add remote messages updates to queue to process asynchronously
+      // // Ignore offline state because remote messages are always fully delivered
+      // // This may happen once per client but should be idempotent
+      // _unreconciledMessagesQueue.addAllSync(rcvdMessages.map((x) => x.value));
+
+      // Update the view
+      _renderState();
+    });
+  }
+
   Future<void> _reconcileMessagesInner(
       {required DHTLogWriteOperations reconciledMessagesWriter,
       required IList<proto.Message> messages}) async {
@@ -380,15 +369,11 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
   // Produce a state for this cubit from the input cubits and queues
   void _renderState() {
-    //  xxx move into a singlefuture
-
     // Get all reconciled messages
     final reconciledMessages =
         _reconciledMessagesCubit?.state.state.asData?.value;
     // Get all sent messages
     final sentMessages = _sentMessagesCubit?.state.state.asData?.value;
-    // Get all items in the unreconciled queue
-    final unreconciledMessages = _unreconciledMessagesQueue.queue;
     // Get all items in the unsent queue
     final sendingMessages = _sendingMessagesQueue.queue;
 
@@ -400,31 +385,30 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
     // Generate state for each message
     final sentMessagesMap =
-        IMap<Int64, DHTLogElementState<proto.Message>>.fromValues(
+        IMap<String, DHTLogElementState<proto.Message>>.fromValues(
-      keyMapper: (x) => x.value.timestamp,
+      keyMapper: (x) => x.value.uniqueIdString,
       values: sentMessages.elements,
     );
-    final reconciledMessagesMap = IMap<Int64, proto.Message>.fromValues(
+    final reconciledMessagesMap =
-      keyMapper: (x) => x.timestamp,
+        IMap<String, proto.ReconciledMessage>.fromValues(
+      keyMapper: (x) => x.content.uniqueIdString,
       values: reconciledMessages.elements,
     );
-    final sendingMessagesMap = IMap<Int64, proto.Message>.fromValues(
+    final sendingMessagesMap = IMap<String, proto.Message>.fromValues(
-      keyMapper: (x) => x.timestamp,
+      keyMapper: (x) => x.uniqueIdString,
       values: sendingMessages,
     );
-    final unreconciledMessagesMap = IMap<Int64, proto.Message>.fromValues(
-      keyMapper: (x) => x.timestamp,
-      values: unreconciledMessages,
-    );
 
-    final renderedElements = <Int64, RenderStateElement>{};
+    final renderedElements = <String, RenderStateElement>{};
 
     for (final m in reconciledMessagesMap.entries) {
       renderedElements[m.key] = RenderStateElement(
-          message: m.value.value,
+        message: m.value.content,
-          isLocal: m.value.value.author.toVeilid() != _remoteIdentityPublicKey,
+        isLocal: m.value.content.author.toVeilid() ==
-          reconciled: true,
+            _activeAccountInfo.localAccount.identityMaster
-          reconciledOffline: m.value.isOffline);
+                .identityPublicTypedKey(),
+        reconciledTimestamp: Timestamp.fromInt64(m.value.reconciledTime),
+      );
     }
     for (final m in sentMessagesMap.entries) {
       renderedElements.putIfAbsent(
@@ -436,17 +420,6 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
         ..sent = true
         ..sentOffline = m.value.isOffline;
     }
-    for (final m in unreconciledMessagesMap.entries) {
-      renderedElements
-          .putIfAbsent(
-              m.key,
-              () => RenderStateElement(
-                    message: m.value,
-                    isLocal:
-                        m.value.author.toVeilid() != _remoteIdentityPublicKey,
-                  ))
-          .reconciled = false;
-    }
     for (final m in sendingMessagesMap.entries) {
       renderedElements
           .putIfAbsent(
@@ -465,24 +438,25 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
     final renderedState = messageKeys
         .map((x) => MessageState(
             content: x.value.message,
-            timestamp: Timestamp.fromInt64(x.key),
+            sentTimestamp: Timestamp.fromInt64(x.value.message.timestamp),
+            reconciledTimestamp: x.value.reconciledTimestamp,
             sendState: x.value.sendState))
         .toIList();
 
     // Emit the rendered state
-
     emit(AsyncValue.data(renderedState));
   }
 
-  void sendTextMessage({required proto.Message_Text messageText}) {
+  void _sendMessage({required proto.Message message}) {
-    final message = proto.Message()
+    // Add common fields
-      ..id = generateNextId()
+    // id and signature will get set by _processMessageToSend
+    message
       ..author = _activeAccountInfo.localAccount.identityMaster
           .identityPublicTypedKey()
           .toProto()
-      ..timestamp = Veilid.instance.now().toInt64()
+      ..timestamp = Veilid.instance.now().toInt64();
-      ..text = messageText;
 
+    // Put in the queue
     _sendingMessagesQueue.addSync(message);
 
     // Update the view
@@ -490,6 +464,7 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
   }
 
   /////////////////////////////////////////////////////////////////////////
+  // Static utility functions
 
   static Future<void> cleanupAndDeleteMessages(
       {required TypedKey localConversationRecordKey}) async {
@@ -518,13 +493,12 @@ class SingleContactMessagesCubit extends Cubit<SingleContactMessagesState> {
 
   DHTLogCubit<proto.Message>? _sentMessagesCubit;
   DHTLogCubit<proto.Message>? _rcvdMessagesCubit;
-  TableDBArrayCubit<proto.Message>? _reconciledMessagesCubit;
+  TableDBArrayCubit<proto.ReconciledMessage>? _reconciledMessagesCubit;
 
-  late final PersistentQueue<proto.Message> _unreconciledMessagesQueue; xxx can we eliminate this? and make rcvd messages cubit listener work like sent?
   late final PersistentQueue<proto.Message> _sendingMessagesQueue;
 
   StreamSubscription<DHTLogBusyState<proto.Message>>? _sentSubscription;
   StreamSubscription<DHTLogBusyState<proto.Message>>? _rcvdSubscription;
-  StreamSubscription<TableDBArrayBusyState<proto.Message>>?
+  StreamSubscription<TableDBArrayBusyState<proto.ReconciledMessage>>?
       _reconciledSubscription;
 }

lib/chat/models/message_state.dart

@@ -28,8 +28,10 @@ class MessageState with _$MessageState {
     // Content of the message
     @JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
     required proto.Message content,
-    // Received or delivered timestamp
+    // Sent timestamp
-    required Timestamp timestamp,
+    required Timestamp sentTimestamp,
+    // Reconciled timestamp
+    required Timestamp? reconciledTimestamp,
     // The state of the message
     required MessageSendState? sendState,
   }) = _MessageState;
@@ -37,11 +39,3 @@ class MessageState with _$MessageState {
   factory MessageState.fromJson(dynamic json) =>
       _$MessageStateFromJson(json as Map<String, dynamic>);
 }
-
-extension MessageStateExt on MessageState {
-  Uint8List get uniqueId {
-    final author = content.author.toVeilid().decode();
-    final id = content.id;
-    return author..addAll(id);
-  }
-}

lib/chat/models/message_state.freezed.dart

@@ -23,8 +23,10 @@ mixin _$MessageState {
 // Content of the message
   @JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
   proto.Message get content =>
-      throw _privateConstructorUsedError; // Received or delivered timestamp
+      throw _privateConstructorUsedError; // Sent timestamp
-  Timestamp get timestamp =>
+  Timestamp get sentTimestamp =>
+      throw _privateConstructorUsedError; // Reconciled timestamp
+  Timestamp? get reconciledTimestamp =>
       throw _privateConstructorUsedError; // The state of the message
   MessageSendState? get sendState => throw _privateConstructorUsedError;
 
@@ -43,7 +45,8 @@ abstract class $MessageStateCopyWith<$Res> {
   $Res call(
       {@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
       proto.Message content,
-      Timestamp timestamp,
+      Timestamp sentTimestamp,
+      Timestamp? reconciledTimestamp,
       MessageSendState? sendState});
 }
 
@@ -61,7 +64,8 @@ class _$MessageStateCopyWithImpl<$Res, $Val extends MessageState>
   @override
   $Res call({
     Object? content = null,
-    Object? timestamp = null,
+    Object? sentTimestamp = null,
+    Object? reconciledTimestamp = freezed,
     Object? sendState = freezed,
   }) {
     return _then(_value.copyWith(
@@ -69,10 +73,14 @@ class _$MessageStateCopyWithImpl<$Res, $Val extends MessageState>
           ? _value.content
           : content // ignore: cast_nullable_to_non_nullable
               as proto.Message,
-      timestamp: null == timestamp
+      sentTimestamp: null == sentTimestamp
-          ? _value.timestamp
+          ? _value.sentTimestamp
-          : timestamp // ignore: cast_nullable_to_non_nullable
+          : sentTimestamp // ignore: cast_nullable_to_non_nullable
               as Timestamp,
+      reconciledTimestamp: freezed == reconciledTimestamp
+          ? _value.reconciledTimestamp
+          : reconciledTimestamp // ignore: cast_nullable_to_non_nullable
+              as Timestamp?,
       sendState: freezed == sendState
           ? _value.sendState
           : sendState // ignore: cast_nullable_to_non_nullable
@@ -92,7 +100,8 @@ abstract class _$$MessageStateImplCopyWith<$Res>
   $Res call(
       {@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
       proto.Message content,
-      Timestamp timestamp,
+      Timestamp sentTimestamp,
+      Timestamp? reconciledTimestamp,
       MessageSendState? sendState});
 }
 
@@ -108,7 +117,8 @@ class __$$MessageStateImplCopyWithImpl<$Res>
   @override
   $Res call({
     Object? content = null,
-    Object? timestamp = null,
+    Object? sentTimestamp = null,
+    Object? reconciledTimestamp = freezed,
     Object? sendState = freezed,
   }) {
     return _then(_$MessageStateImpl(
@@ -116,10 +126,14 @@ class __$$MessageStateImplCopyWithImpl<$Res>
           ? _value.content
           : content // ignore: cast_nullable_to_non_nullable
               as proto.Message,
-      timestamp: null == timestamp
+      sentTimestamp: null == sentTimestamp
-          ? _value.timestamp
+          ? _value.sentTimestamp
-          : timestamp // ignore: cast_nullable_to_non_nullable
+          : sentTimestamp // ignore: cast_nullable_to_non_nullable
               as Timestamp,
+      reconciledTimestamp: freezed == reconciledTimestamp
+          ? _value.reconciledTimestamp
+          : reconciledTimestamp // ignore: cast_nullable_to_non_nullable
+              as Timestamp?,
       sendState: freezed == sendState
           ? _value.sendState
           : sendState // ignore: cast_nullable_to_non_nullable
@@ -134,7 +148,8 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
   const _$MessageStateImpl(
       {@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
       required this.content,
-      required this.timestamp,
+      required this.sentTimestamp,
+      required this.reconciledTimestamp,
       required this.sendState});
 
   factory _$MessageStateImpl.fromJson(Map<String, dynamic> json) =>
@@ -144,16 +159,19 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
   @override
   @JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
   final proto.Message content;
-// Received or delivered timestamp
+// Sent timestamp
   @override
-  final Timestamp timestamp;
+  final Timestamp sentTimestamp;
+// Reconciled timestamp
+  @override
+  final Timestamp? reconciledTimestamp;
 // The state of the message
   @override
   final MessageSendState? sendState;
 
   @override
   String toString({DiagnosticLevel minLevel = DiagnosticLevel.info}) {
-    return 'MessageState(content: $content, timestamp: $timestamp, sendState: $sendState)';
+    return 'MessageState(content: $content, sentTimestamp: $sentTimestamp, reconciledTimestamp: $reconciledTimestamp, sendState: $sendState)';
   }
 
   @override
@@ -162,7 +180,8 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
     properties
       ..add(DiagnosticsProperty('type', 'MessageState'))
       ..add(DiagnosticsProperty('content', content))
-      ..add(DiagnosticsProperty('timestamp', timestamp))
+      ..add(DiagnosticsProperty('sentTimestamp', sentTimestamp))
+      ..add(DiagnosticsProperty('reconciledTimestamp', reconciledTimestamp))
       ..add(DiagnosticsProperty('sendState', sendState));
   }
 
@@ -172,15 +191,18 @@ class _$MessageStateImpl with DiagnosticableTreeMixin implements _MessageState {
         (other.runtimeType == runtimeType &&
             other is _$MessageStateImpl &&
             (identical(other.content, content) || other.content == content) &&
-            (identical(other.timestamp, timestamp) ||
+            (identical(other.sentTimestamp, sentTimestamp) ||
-                other.timestamp == timestamp) &&
+                other.sentTimestamp == sentTimestamp) &&
+            (identical(other.reconciledTimestamp, reconciledTimestamp) ||
+                other.reconciledTimestamp == reconciledTimestamp) &&
             (identical(other.sendState, sendState) ||
                 other.sendState == sendState));
   }
 
   @JsonKey(ignore: true)
   @override
-  int get hashCode => Object.hash(runtimeType, content, timestamp, sendState);
+  int get hashCode => Object.hash(
+      runtimeType, content, sentTimestamp, reconciledTimestamp, sendState);
 
   @JsonKey(ignore: true)
   @override
@@ -200,7 +222,8 @@ abstract class _MessageState implements MessageState {
   const factory _MessageState(
       {@JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
       required final proto.Message content,
-      required final Timestamp timestamp,
+      required final Timestamp sentTimestamp,
+      required final Timestamp? reconciledTimestamp,
       required final MessageSendState? sendState}) = _$MessageStateImpl;
 
   factory _MessageState.fromJson(Map<String, dynamic> json) =
@@ -209,8 +232,10 @@ abstract class _MessageState implements MessageState {
   @override // Content of the message
   @JsonKey(fromJson: proto.messageFromJson, toJson: proto.messageToJson)
   proto.Message get content;
-  @override // Received or delivered timestamp
+  @override // Sent timestamp
-  Timestamp get timestamp;
+  Timestamp get sentTimestamp;
+  @override // Reconciled timestamp
+  Timestamp? get reconciledTimestamp;
   @override // The state of the message
   MessageSendState? get sendState;
   @override

lib/chat/models/message_state.g.dart

@@ -9,7 +9,10 @@ part of 'message_state.dart';
 _$MessageStateImpl _$$MessageStateImplFromJson(Map<String, dynamic> json) =>
     _$MessageStateImpl(
       content: messageFromJson(json['content'] as Map<String, dynamic>),
-      timestamp: Timestamp.fromJson(json['timestamp']),
+      sentTimestamp: Timestamp.fromJson(json['sent_timestamp']),
+      reconciledTimestamp: json['reconciled_timestamp'] == null
+          ? null
+          : Timestamp.fromJson(json['reconciled_timestamp']),
       sendState: json['send_state'] == null
           ? null
           : MessageSendState.fromJson(json['send_state']),
@@ -18,6 +21,7 @@ _$MessageStateImpl _$$MessageStateImplFromJson(Map<String, dynamic> json) =>
 Map<String, dynamic> _$$MessageStateImplToJson(_$MessageStateImpl instance) =>
     <String, dynamic>{
       'content': messageToJson(instance.content),
-      'timestamp': instance.timestamp.toJson(),
+      'sent_timestamp': instance.sentTimestamp.toJson(),
+      'reconciled_timestamp': instance.reconciledTimestamp?.toJson(),
       'send_state': instance.sendState?.toJson(),
     };

lib/chat/views/chat_component.dart

@@ -104,7 +104,7 @@ class ChatComponent extends StatelessWidget {
 
   /////////////////////////////////////////////////////////////////////
 
-  types.Message? messageToChatMessage(MessageState message) {
+  types.Message? messageStateToChatMessage(MessageState message) {
     final isLocal = message.content.author.toVeilid() == _localUserIdentityKey;
 
     types.Status? status;
@@ -125,8 +125,9 @@ class ChatComponent extends StatelessWidget {
         final contextText = message.content.text;
         final textMessage = types.TextMessage(
             author: isLocal ? _localUser : _remoteUser,
-            createdAt: (message.timestamp.value ~/ BigInt.from(1000)).toInt(),
+            createdAt:
-            id: base64UrlNoPadEncode(message.uniqueId),
+                (message.sentTimestamp.value ~/ BigInt.from(1000)).toInt(),
+            id: message.content.uniqueIdString,
             text: contextText.text,
             showStatus: status != null,
             status: status);
@@ -219,7 +220,7 @@ class ChatComponent extends StatelessWidget {
     final chatMessages = <types.Message>[];
     final tsSet = <String>{};
     for (final message in messages) {
-      final chatMessage = messageToChatMessage(message);
+      final chatMessage = messageStateToChatMessage(message);
       if (chatMessage == null) {
         continue;
       }

lib/proto/extensions.dart

@@ -1,3 +1,7 @@
+import 'dart:typed_data';
+
+import 'package:veilid_support/veilid_support.dart';
+
 import 'proto.dart' as proto;
 
 proto.Message messageFromJson(Map<String, dynamic> j) =>
@@ -10,3 +14,13 @@ proto.ReconciledMessage reconciledMessageFromJson(Map<String, dynamic> j) =>
 
 Map<String, dynamic> reconciledMessageToJson(proto.ReconciledMessage m) =>
     m.writeToJsonMap();
+
+extension MessageExt on proto.Message {
+  Uint8List get uniqueIdBytes {
+    final author = this.author.toVeilid().decode();
+    final id = this.id;
+    return Uint8List.fromList([...author, ...id]);
+  }
+
+  String get uniqueIdString => base64UrlNoPadEncode(uniqueIdBytes);
+}

packages/veilid_support/lib/proto/dht.pb.dart

@@ -195,8 +195,109 @@ class DHTShortArray extends $pb.GeneratedMessage {
   $core.List<$core.int> get seqs => $_getList(2);
 }
 
+class DHTDataReference extends $pb.GeneratedMessage {
+  factory DHTDataReference() => create();
+  DHTDataReference._() : super();
+  factory DHTDataReference.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
+  factory DHTDataReference.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
+
+  static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DHTDataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
+    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: $0.TypedKey.create)
+    ..aOM<$0.TypedKey>(2, _omitFieldNames ? '' : 'hash', subBuilder: $0.TypedKey.create)
+    ..hasRequiredFields = false
+  ;
+
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
+  'Will be removed in next major version')
+  DHTDataReference clone() => DHTDataReference()..mergeFromMessage(this);
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
+  'Will be removed in next major version')
+  DHTDataReference copyWith(void Function(DHTDataReference) updates) => super.copyWith((message) => updates(message as DHTDataReference)) as DHTDataReference;
+
+  $pb.BuilderInfo get info_ => _i;
+
+  @$core.pragma('dart2js:noInline')
+  static DHTDataReference create() => DHTDataReference._();
+  DHTDataReference createEmptyInstance() => create();
+  static $pb.PbList<DHTDataReference> createRepeated() => $pb.PbList<DHTDataReference>();
+  @$core.pragma('dart2js:noInline')
+  static DHTDataReference getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<DHTDataReference>(create);
+  static DHTDataReference? _defaultInstance;
+
+  @$pb.TagNumber(1)
+  $0.TypedKey get dhtData => $_getN(0);
+  @$pb.TagNumber(1)
+  set dhtData($0.TypedKey v) { setField(1, v); }
+  @$pb.TagNumber(1)
+  $core.bool hasDhtData() => $_has(0);
+  @$pb.TagNumber(1)
+  void clearDhtData() => clearField(1);
+  @$pb.TagNumber(1)
+  $0.TypedKey ensureDhtData() => $_ensure(0);
+
+  @$pb.TagNumber(2)
+  $0.TypedKey get hash => $_getN(1);
+  @$pb.TagNumber(2)
+  set hash($0.TypedKey v) { setField(2, v); }
+  @$pb.TagNumber(2)
+  $core.bool hasHash() => $_has(1);
+  @$pb.TagNumber(2)
+  void clearHash() => clearField(2);
+  @$pb.TagNumber(2)
+  $0.TypedKey ensureHash() => $_ensure(1);
+}
+
+class BlockStoreDataReference extends $pb.GeneratedMessage {
+  factory BlockStoreDataReference() => create();
+  BlockStoreDataReference._() : super();
+  factory BlockStoreDataReference.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
+  factory BlockStoreDataReference.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
+
+  static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'BlockStoreDataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
+    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'block', subBuilder: $0.TypedKey.create)
+    ..hasRequiredFields = false
+  ;
+
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
+  'Will be removed in next major version')
+  BlockStoreDataReference clone() => BlockStoreDataReference()..mergeFromMessage(this);
+  @$core.Deprecated(
+  'Using this can add significant overhead to your binary. '
+  'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
+  'Will be removed in next major version')
+  BlockStoreDataReference copyWith(void Function(BlockStoreDataReference) updates) => super.copyWith((message) => updates(message as BlockStoreDataReference)) as BlockStoreDataReference;
+
+  $pb.BuilderInfo get info_ => _i;
+
+  @$core.pragma('dart2js:noInline')
+  static BlockStoreDataReference create() => BlockStoreDataReference._();
+  BlockStoreDataReference createEmptyInstance() => create();
+  static $pb.PbList<BlockStoreDataReference> createRepeated() => $pb.PbList<BlockStoreDataReference>();
+  @$core.pragma('dart2js:noInline')
+  static BlockStoreDataReference getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<BlockStoreDataReference>(create);
+  static BlockStoreDataReference? _defaultInstance;
+
+  @$pb.TagNumber(1)
+  $0.TypedKey get block => $_getN(0);
+  @$pb.TagNumber(1)
+  set block($0.TypedKey v) { setField(1, v); }
+  @$pb.TagNumber(1)
+  $core.bool hasBlock() => $_has(0);
+  @$pb.TagNumber(1)
+  void clearBlock() => clearField(1);
+  @$pb.TagNumber(1)
+  $0.TypedKey ensureBlock() => $_ensure(0);
+}
+
 enum DataReference_Kind {
   dhtData, 
+  blockStoreData, 
   notSet
 }
 
@@ -208,11 +309,13 @@ class DataReference extends $pb.GeneratedMessage {
 
   static const $core.Map<$core.int, DataReference_Kind> _DataReference_KindByTag = {
     1 : DataReference_Kind.dhtData,
+    2 : DataReference_Kind.blockStoreData,
     0 : DataReference_Kind.notSet
   };
   static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'DataReference', package: const $pb.PackageName(_omitMessageNames ? '' : 'dht'), createEmptyInstance: create)
-    ..oo(0, [1])
+    ..oo(0, [1, 2])
-    ..aOM<$0.TypedKey>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: $0.TypedKey.create)
+    ..aOM<DHTDataReference>(1, _omitFieldNames ? '' : 'dhtData', subBuilder: DHTDataReference.create)
+    ..aOM<BlockStoreDataReference>(2, _omitFieldNames ? '' : 'blockStoreData', subBuilder: BlockStoreDataReference.create)
     ..hasRequiredFields = false
   ;
 
@@ -241,15 +344,26 @@ class DataReference extends $pb.GeneratedMessage {
   void clearKind() => clearField($_whichOneof(0));
 
   @$pb.TagNumber(1)
-  $0.TypedKey get dhtData => $_getN(0);
+  DHTDataReference get dhtData => $_getN(0);
   @$pb.TagNumber(1)
-  set dhtData($0.TypedKey v) { setField(1, v); }
+  set dhtData(DHTDataReference v) { setField(1, v); }
   @$pb.TagNumber(1)
   $core.bool hasDhtData() => $_has(0);
   @$pb.TagNumber(1)
   void clearDhtData() => clearField(1);
   @$pb.TagNumber(1)
-  $0.TypedKey ensureDhtData() => $_ensure(0);
+  DHTDataReference ensureDhtData() => $_ensure(0);
+
+  @$pb.TagNumber(2)
+  BlockStoreDataReference get blockStoreData => $_getN(1);
+  @$pb.TagNumber(2)
+  set blockStoreData(BlockStoreDataReference v) { setField(2, v); }
+  @$pb.TagNumber(2)
+  $core.bool hasBlockStoreData() => $_has(1);
+  @$pb.TagNumber(2)
+  void clearBlockStoreData() => clearField(2);
+  @$pb.TagNumber(2)
+  BlockStoreDataReference ensureBlockStoreData() => $_ensure(1);
 }
 
 class OwnedDHTRecordPointer extends $pb.GeneratedMessage {

packages/veilid_support/lib/proto/dht.pbjson.dart

@@ -60,11 +60,39 @@ final $typed_data.Uint8List dHTShortArrayDescriptor = $convert.base64Decode(
     'Cg1ESFRTaG9ydEFycmF5EiQKBGtleXMYASADKAsyEC52ZWlsaWQuVHlwZWRLZXlSBGtleXMSFA'
     'oFaW5kZXgYAiABKAxSBWluZGV4EhIKBHNlcXMYAyADKA1SBHNlcXM=');
 
+@$core.Deprecated('Use dHTDataReferenceDescriptor instead')
+const DHTDataReference$json = {
+  '1': 'DHTDataReference',
+  '2': [
+    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'dhtData'},
+    {'1': 'hash', '3': 2, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'hash'},
+  ],
+};
+
+/// Descriptor for `DHTDataReference`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List dHTDataReferenceDescriptor = $convert.base64Decode(
+    'ChBESFREYXRhUmVmZXJlbmNlEisKCGRodF9kYXRhGAEgASgLMhAudmVpbGlkLlR5cGVkS2V5Ug'
+    'dkaHREYXRhEiQKBGhhc2gYAiABKAsyEC52ZWlsaWQuVHlwZWRLZXlSBGhhc2g=');
+
+@$core.Deprecated('Use blockStoreDataReferenceDescriptor instead')
+const BlockStoreDataReference$json = {
+  '1': 'BlockStoreDataReference',
+  '2': [
+    {'1': 'block', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '10': 'block'},
+  ],
+};
+
+/// Descriptor for `BlockStoreDataReference`. Decode as a `google.protobuf.DescriptorProto`.
+final $typed_data.Uint8List blockStoreDataReferenceDescriptor = $convert.base64Decode(
+    'ChdCbG9ja1N0b3JlRGF0YVJlZmVyZW5jZRImCgVibG9jaxgBIAEoCzIQLnZlaWxpZC5UeXBlZE'
+    'tleVIFYmxvY2s=');
+
 @$core.Deprecated('Use dataReferenceDescriptor instead')
 const DataReference$json = {
   '1': 'DataReference',
   '2': [
-    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.veilid.TypedKey', '9': 0, '10': 'dhtData'},
+    {'1': 'dht_data', '3': 1, '4': 1, '5': 11, '6': '.dht.DHTDataReference', '9': 0, '10': 'dhtData'},
+    {'1': 'block_store_data', '3': 2, '4': 1, '5': 11, '6': '.dht.BlockStoreDataReference', '9': 0, '10': 'blockStoreData'},
   ],
   '8': [
     {'1': 'kind'},
@@ -73,8 +101,9 @@ const DataReference$json = {
 
 /// Descriptor for `DataReference`. Decode as a `google.protobuf.DescriptorProto`.
 final $typed_data.Uint8List dataReferenceDescriptor = $convert.base64Decode(
-    'Cg1EYXRhUmVmZXJlbmNlEi0KCGRodF9kYXRhGAEgASgLMhAudmVpbGlkLlR5cGVkS2V5SABSB2'
+    'Cg1EYXRhUmVmZXJlbmNlEjIKCGRodF9kYXRhGAEgASgLMhUuZGh0LkRIVERhdGFSZWZlcmVuY2'
-    'RodERhdGFCBgoEa2luZA==');
+    'VIAFIHZGh0RGF0YRJIChBibG9ja19zdG9yZV9kYXRhGAIgASgLMhwuZGh0LkJsb2NrU3RvcmVE'
+    'YXRhUmVmZXJlbmNlSABSDmJsb2NrU3RvcmVEYXRhQgYKBGtpbmQ=');
 
 @$core.Deprecated('Use ownedDHTRecordPointerDescriptor instead')
 const OwnedDHTRecordPointer$json = {