Git-Mirrors/veilidchat
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilidchat.git synced 2024-12-25 07:39:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

tabledbarray tests pass

Browse Source
This commit is contained in:
Christien Rioux 2024-05-27 13:17:33 -04:00
parent ab65956433
commit 17f6dfce46
3 changed files with 207 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
packages/veilid_support/example/integration_test/app_test.dart
View File

@ -40,77 +40,7 @@ void main() {
group('TableDBArray Tests', () { group('TableDBArray Tests', () {
// test('create/delete TableDBArray', testTableDBArrayCreateDelete); // test('create/delete TableDBArray', testTableDBArrayCreateDelete);
// group('TableDBArray Add/Get Tests', () { group('TableDBArray Add/Get Tests', () {
// for (final params in [
// //
// (99, 3, 15),
// (100, 4, 16),
// (101, 5, 17),
// //
// (511, 3, 127),
// (512, 4, 128),
// (513, 5, 129),
// //
// (4095, 3, 1023),
// (4096, 4, 1024),
// (4097, 5, 1025),
// //
// (65535, 3, 16383),
// (65536, 4, 16384),
// (65537, 5, 16385),
// ]) {
// final count = params.$1;
// final singles = params.$2;
// final batchSize = params.$3;
// test(
// // timeout: const Timeout(Duration(seconds: 480)),
// 'add/remove TableDBArray count = $count batchSize=$batchSize',
// makeTestTableDBArrayAddGetClear(
// count: count,
// singles: singles,
// batchSize: batchSize,
// crypto: const VeilidCryptoPublic()),
// );
// }
// });
// group('TableDBArray Insert Tests', () {
// for (final params in [
// //
// (99, 3, 15),
// (100, 4, 16),
// (101, 5, 17),
// //
// (511, 3, 127),
// (512, 4, 128),
// (513, 5, 129),
// //
// (4095, 3, 1023),
// (4096, 4, 1024),
// (4097, 5, 1025),
// //
// (65535, 3, 16383),
// (65536, 4, 16384),
// (65537, 5, 16385),
// ]) {
// final count = params.$1;
// final singles = params.$2;
// final batchSize = params.$3;
// test(
// // timeout: const Timeout(Duration(seconds: 480)),
// 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
// makeTestTableDBArrayInsert(
// count: count,
// singles: singles,
// batchSize: batchSize,
// crypto: const VeilidCryptoPublic()),
// );
// }
// });
group('TableDBArray Remove Tests', () {
for (final params in [ for (final params in [
// //
(99, 3, 15), (99, 3, 15),
@ -134,7 +64,77 @@ void main() {
final batchSize = params.$3; final batchSize = params.$3;
test( test(
// timeout: const Timeout(Duration(seconds: 480)), timeout: const Timeout(Duration(seconds: 480)),
'add/remove TableDBArray count = $count batchSize=$batchSize',
makeTestTableDBArrayAddGetClear(
count: count,
singles: singles,
batchSize: batchSize,
crypto: const VeilidCryptoPublic()),
);
}
});
group('TableDBArray Insert Tests', () {
for (final params in [
//
(99, 3, 15),
(100, 4, 16),
(101, 5, 17),
//
(511, 3, 127),
(512, 4, 128),
(513, 5, 129),
//
(4095, 3, 1023),
(4096, 4, 1024),
(4097, 5, 1025),
//
(65535, 3, 16383),
(65536, 4, 16384),
(65537, 5, 16385),
]) {
final count = params.$1;
final singles = params.$2;
final batchSize = params.$3;
test(
timeout: const Timeout(Duration(seconds: 480)),
'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
makeTestTableDBArrayInsert(
count: count,
singles: singles,
batchSize: batchSize,
crypto: const VeilidCryptoPublic()),
);
}
});
group('TableDBArray Remove Tests', () {
for (final params in [
//
(99, 3, 15),
(100, 4, 16),
(101, 5, 17),
//
(511, 3, 127),
(512, 4, 128),
(513, 5, 129),
//
(4095, 3, 1023),
(4096, 4, 1024),
(4097, 5, 1025),
//
(16383, 3, 4095),
(16384, 4, 4096),
(16385, 5, 4097),
]) {
final count = params.$1;
final singles = params.$2;
final batchSize = params.$3;
test(
timeout: const Timeout(Duration(seconds: 480)),
'remove TableDBArray count=$count singles=$singles batchSize=$batchSize', 'remove TableDBArray count=$count singles=$singles batchSize=$batchSize',
makeTestTableDBArrayRemove( makeTestTableDBArrayRemove(
count: count, count: count,
@ -147,50 +147,50 @@ void main() {
}); });
}); });
// group('DHT Support Tests', () { group('DHT Support Tests', () {
// setUpAll(updateProcessorFixture.setUp); setUpAll(updateProcessorFixture.setUp);
// setUpAll(tickerFixture.setUp); setUpAll(tickerFixture.setUp);
// tearDownAll(tickerFixture.tearDown); tearDownAll(tickerFixture.tearDown);
// tearDownAll(updateProcessorFixture.tearDown); tearDownAll(updateProcessorFixture.tearDown);
// test('create pool', testDHTRecordPoolCreate); test('create pool', testDHTRecordPoolCreate);
// group('DHTRecordPool Tests', () { group('DHTRecordPool Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp); setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown); tearDownAll(dhtRecordPoolFixture.tearDown);
// test('create/delete record', testDHTRecordCreateDelete); test('create/delete record', testDHTRecordCreateDelete);
// test('record scopes', testDHTRecordScopes); test('record scopes', testDHTRecordScopes);
// test('create/delete deep record', testDHTRecordDeepCreateDelete); test('create/delete deep record', testDHTRecordDeepCreateDelete);
// }); });
// group('DHTShortArray Tests', () { group('DHTShortArray Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp); setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown); tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create shortarray stride=$stride', test('create shortarray stride=$stride',
// makeTestDHTShortArrayCreateDelete(stride: stride)); makeTestDHTShortArrayCreateDelete(stride: stride));
// test('add shortarray stride=$stride', test('add shortarray stride=$stride',
// makeTestDHTShortArrayAdd(stride: stride)); makeTestDHTShortArrayAdd(stride: stride));
// } }
// }); });
// group('DHTLog Tests', () { group('DHTLog Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp); setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown); tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create log stride=$stride', test('create log stride=$stride',
// makeTestDHTLogCreateDelete(stride: stride)); makeTestDHTLogCreateDelete(stride: stride));
// test( test(
// timeout: const Timeout(Duration(seconds: 480)), timeout: const Timeout(Duration(seconds: 480)),
// 'add/truncate log stride=$stride', 'add/truncate log stride=$stride',
// makeTestDHTLogAddTruncate(stride: stride), makeTestDHTLogAddTruncate(stride: stride),
// ); );
// } }
// }); });
// }); });
}); });
}); });
} }

91
packages/veilid_support/example/integration_test/test_table_db_array.dart
View File

@ -84,7 +84,8 @@ Future<void> Function() makeTestTableDBArrayAddGetClear(
{ {
for (var n = batchSize; n < count; n += batchSize) { for (var n = batchSize; n < count; n += batchSize) {
final toGet = min(batchSize, count - n); final toGet = min(batchSize, count - n);
expect(await arr.getRange(n, toGet), equals(makeDataBatch(n, toGet))); expect(await arr.getRange(n, n + toGet),
equals(makeDataBatch(n, toGet)));
} }
} }
@ -140,7 +141,7 @@ Future<void> Function() makeTestTableDBArrayInsert(
{ {
for (var n = batchSize; n < count; n += batchSize) { for (var n = batchSize; n < count; n += batchSize) {
final toGet = min(batchSize, count - n); final toGet = min(batchSize, count - n);
expect(await arr.getRange(n, toGet), expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet))); equals(match.sublist(n, n + toGet)));
} }
} }
@ -154,7 +155,6 @@ Future<void> Function() makeTestTableDBArrayInsert(
await arr.close(delete: true); await arr.close(delete: true);
}; };
Future<void> Function() makeTestTableDBArrayRemove( Future<void> Function() makeTestTableDBArrayRemove(
{required int count, {required int count,
required int singles, required int singles,
@ -164,42 +164,79 @@ Future<void> Function() makeTestTableDBArrayRemove(
final arr = await TableDBArray.make(table: 'testArray', crypto: crypto); final arr = await TableDBArray.make(table: 'testArray', crypto: crypto);
final match = <Uint8List>[]; final match = <Uint8List>[];
xxx removal test
print('inserting');
{ {
for (var n = 0; n < count;) { final rems = [
final start = n; (0, 0),
var toAdd = min(batchSize, count - n); (0, 1),
for (var s = 0; s < min(singles, toAdd); s++) { (0, batchSize),
final data = makeData(n); (1, batchSize - 1),
await arr.insert(start, data); (batchSize, 1),
match.insert(start, data); (batchSize + 1, batchSize),
toAdd--; (batchSize - 1, batchSize + 1)
n++; ];
} for (final rem in rems) {
print('adding ');
{
for (var n = match.length; n < count;) {
final toAdd = min(batchSize, count - n);
final data = makeDataBatch(n, toAdd); final data = makeDataBatch(n, toAdd);
await arr.insertAll(start, data); await arr.addAll(data);
match.insertAll(start, data); match.addAll(data);
n += toAdd; n += toAdd;
print(' $n/$count'); print(' $n/$count');
} }
expect(arr.length, equals(match.length));
} }
print('get singles');
{ {
for (var n = 0; n < batchSize; n++) { final start = rem.$1;
expect(await arr.get(n), equals(match[n])); final length = rem.$2;
} print('removing start=$start length=$length');
}
final out = Output<List<Uint8List>>();
await arr.removeRange(start, start + length, out: out);
expect(out.value, equals(match.sublist(start, start + length)));
match.removeRange(start, start + length);
expect(arr.length, equals(match.length));
print('get batch'); print('get batch');
{ {
for (var n = batchSize; n < count; n += batchSize) { final checkCount = match.length;
final toGet = min(batchSize, count - n); for (var n = 0; n < checkCount;) {
expect(await arr.getRange(n, toGet), final toGet = min(batchSize, checkCount - n);
expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet))); equals(match.sublist(n, n + toGet)));
n += toGet;
print(' $n/$checkCount');
}
}
}
{
final start = match.length - rem.$1 - rem.$2;
final length = rem.$2;
print('removing from end start=$start length=$length');
final out = Output<List<Uint8List>>();
await arr.removeRange(start, start + length, out: out);
expect(out.value, equals(match.sublist(start, start + length)));
match.removeRange(start, start + length);
expect(arr.length, equals(match.length));
print('get batch');
{
final checkCount = match.length;
for (var n = 0; n < checkCount;) {
final toGet = min(batchSize, checkCount - n);
expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet)));
n += toGet;
print(' $n/$checkCount');
}
expect(arr.length, equals(match.length));
}
}
} }
} }

43
packages/veilid_support/lib/src/table_db_array.dart
View File

@ -112,13 +112,13 @@ class TableDBArray {
}); });
} }
Future<List<Uint8List>> getRange(int start, int length) async { Future<List<Uint8List>> getRange(int start, int end) async {
await _initWait(); await _initWait();
return _mutex.protect(() async { return _mutex.protect(() async {
if (!_open) { if (!_open) {
throw StateError('not open'); throw StateError('not open');
} }
return _getRangeInner(start, length); return _getRangeInner(start, end);
}); });
} }
@ -127,11 +127,11 @@ class TableDBArray {
return _writeTransaction((t) async => _removeInner(t, pos, out: out)); return _writeTransaction((t) async => _removeInner(t, pos, out: out));
} }
Future<void> removeRange(int start, int length, Future<void> removeRange(int start, int end,
{Output<List<Uint8List>>? out}) async { {Output<List<Uint8List>>? out}) async {
await _initWait(); await _initWait();
return _writeTransaction( return _writeTransaction(
(t) async => _removeRangeInner(t, start, length, out: out)); (t) async => _removeRangeInner(t, start, end, out: out));
} }
Future<void> clear() async { Future<void> clear() async {
@ -223,23 +223,34 @@ class TableDBArray {
return (await _loadEntry(entry))!; return (await _loadEntry(entry))!;
} }
Future<List<Uint8List>> _getRangeInner(int start, int length) async { Future<List<Uint8List>> _getRangeInner(int start, int end) async {
final length = end - start;
if (length < 0) { if (length < 0) {
throw StateError('length should not be negative'); throw StateError('length should not be negative');
} }
if (start < 0 || start >= _length) { if (start < 0 || start >= _length) {
throw IndexError.withLength(start, _length); throw IndexError.withLength(start, _length);
} }
if ((start + length) > _length) { if (end > _length) {
throw IndexError.withLength(start + length, _length); throw IndexError.withLength(end, _length);
} }
final out = <Uint8List>[]; final out = <Uint8List>[];
for (var pos = start; pos < (start + length); pos++) { const batchSize = 16;
for (var pos = start; pos < end;) {
var batchLen = min(batchSize, end - pos);
final dws = DelayedWaitSet<Uint8List>();
while (batchLen > 0) {
final entry = await _getIndexEntry(pos); final entry = await _getIndexEntry(pos);
final value = (await _loadEntry(entry))!; dws.add(() async => (await _loadEntry(entry))!);
out.add(value); pos++;
batchLen--;
} }
final batchOut = await dws();
out.addAll(batchOut);
}
return out; return out;
} }
@ -259,21 +270,21 @@ class TableDBArray {
await _removeIndexEntry(pos); await _removeIndexEntry(pos);
} }
Future<void> _removeRangeInner( Future<void> _removeRangeInner(VeilidTableDBTransaction t, int start, int end,
VeilidTableDBTransaction t, int start, int length,
{Output<List<Uint8List>>? out}) async { {Output<List<Uint8List>>? out}) async {
final length = end - start;
if (length < 0) { if (length < 0) {
throw StateError('length should not be negative'); throw StateError('length should not be negative');
} }
if (start < 0 || start >= _length) { if (start < 0) {
throw IndexError.withLength(start, _length); throw IndexError.withLength(start, _length);
} }
if ((start + length) > _length) { if (end > _length) {
throw IndexError.withLength(start + length, _length); throw IndexError.withLength(end, _length);
} }
final outList = <Uint8List>[]; final outList = <Uint8List>[];
for (var pos = start; pos < (start + length); pos++) { for (var pos = start; pos < end; pos++) {
final entry = await _getIndexEntry(pos); final entry = await _getIndexEntry(pos);
if (out != null) { if (out != null) {
final value = (await _loadEntry(entry))!; final value = (await _loadEntry(entry))!;