Git-Mirrors/veilidchat
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilidchat.git synced 2024-12-24 23:29:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

tabledbarray tests pass

Browse Source
This commit is contained in:
Christien Rioux 2024-05-27 13:17:33 -04:00
parent ab65956433
commit 17f6dfce46
3 changed files with 207 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
packages/veilid_support/example/integration_test/app_test.dart
View File

@ -40,77 +40,7 @@ void main() {
group('TableDBArray Tests', () {
// test('create/delete TableDBArray', testTableDBArrayCreateDelete);
// group('TableDBArray Add/Get Tests', () {
// for (final params in [
// //
// (99, 3, 15),
// (100, 4, 16),
// (101, 5, 17),
// //
// (511, 3, 127),
// (512, 4, 128),
// (513, 5, 129),
// //
// (4095, 3, 1023),
// (4096, 4, 1024),
// (4097, 5, 1025),
// //
// (65535, 3, 16383),
// (65536, 4, 16384),
// (65537, 5, 16385),
// ]) {
// final count = params.$1;
// final singles = params.$2;
// final batchSize = params.$3;
// test(
// // timeout: const Timeout(Duration(seconds: 480)),
// 'add/remove TableDBArray count = $count batchSize=$batchSize',
// makeTestTableDBArrayAddGetClear(
// count: count,
// singles: singles,
// batchSize: batchSize,
// crypto: const VeilidCryptoPublic()),
// );
// }
// });
// group('TableDBArray Insert Tests', () {
// for (final params in [
// //
// (99, 3, 15),
// (100, 4, 16),
// (101, 5, 17),
// //
// (511, 3, 127),
// (512, 4, 128),
// (513, 5, 129),
// //
// (4095, 3, 1023),
// (4096, 4, 1024),
// (4097, 5, 1025),
// //
// (65535, 3, 16383),
// (65536, 4, 16384),
// (65537, 5, 16385),
// ]) {
// final count = params.$1;
// final singles = params.$2;
// final batchSize = params.$3;
// test(
// // timeout: const Timeout(Duration(seconds: 480)),
// 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
// makeTestTableDBArrayInsert(
// count: count,
// singles: singles,
// batchSize: batchSize,
// crypto: const VeilidCryptoPublic()),
// );
// }
// });
group('TableDBArray Remove Tests', () {
group('TableDBArray Add/Get Tests', () {
for (final params in [
//
(99, 3, 15),
@ -134,7 +64,77 @@ void main() {
final batchSize = params.$3;
test(
// timeout: const Timeout(Duration(seconds: 480)),
timeout: const Timeout(Duration(seconds: 480)),
'add/remove TableDBArray count = $count batchSize=$batchSize',
makeTestTableDBArrayAddGetClear(
count: count,
singles: singles,
batchSize: batchSize,
crypto: const VeilidCryptoPublic()),
);
}
});
group('TableDBArray Insert Tests', () {
for (final params in [
//
(99, 3, 15),
(100, 4, 16),
(101, 5, 17),
//
(511, 3, 127),
(512, 4, 128),
(513, 5, 129),
//
(4095, 3, 1023),
(4096, 4, 1024),
(4097, 5, 1025),
//
(65535, 3, 16383),
(65536, 4, 16384),
(65537, 5, 16385),
]) {
final count = params.$1;
final singles = params.$2;
final batchSize = params.$3;
test(
timeout: const Timeout(Duration(seconds: 480)),
'insert TableDBArray count=$count singles=$singles batchSize=$batchSize',
makeTestTableDBArrayInsert(
count: count,
singles: singles,
batchSize: batchSize,
crypto: const VeilidCryptoPublic()),
);
}
});
group('TableDBArray Remove Tests', () {
for (final params in [
//
(99, 3, 15),
(100, 4, 16),
(101, 5, 17),
//
(511, 3, 127),
(512, 4, 128),
(513, 5, 129),
//
(4095, 3, 1023),
(4096, 4, 1024),
(4097, 5, 1025),
//
(16383, 3, 4095),
(16384, 4, 4096),
(16385, 5, 4097),
]) {
final count = params.$1;
final singles = params.$2;
final batchSize = params.$3;
test(
timeout: const Timeout(Duration(seconds: 480)),
'remove TableDBArray count=$count singles=$singles batchSize=$batchSize',
makeTestTableDBArrayRemove(
count: count,
@ -147,50 +147,50 @@ void main() {
});
});
// group('DHT Support Tests', () {
// setUpAll(updateProcessorFixture.setUp);
// setUpAll(tickerFixture.setUp);
// tearDownAll(tickerFixture.tearDown);
// tearDownAll(updateProcessorFixture.tearDown);
group('DHT Support Tests', () {
setUpAll(updateProcessorFixture.setUp);
setUpAll(tickerFixture.setUp);
tearDownAll(tickerFixture.tearDown);
tearDownAll(updateProcessorFixture.tearDown);
// test('create pool', testDHTRecordPoolCreate);
test('create pool', testDHTRecordPoolCreate);
// group('DHTRecordPool Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
group('DHTRecordPool Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
// test('create/delete record', testDHTRecordCreateDelete);
// test('record scopes', testDHTRecordScopes);
// test('create/delete deep record', testDHTRecordDeepCreateDelete);
// });
test('create/delete record', testDHTRecordCreateDelete);
test('record scopes', testDHTRecordScopes);
test('create/delete deep record', testDHTRecordDeepCreateDelete);
});
// group('DHTShortArray Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
group('DHTShortArray Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create shortarray stride=$stride',
// makeTestDHTShortArrayCreateDelete(stride: stride));
// test('add shortarray stride=$stride',
// makeTestDHTShortArrayAdd(stride: stride));
// }
// });
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create shortarray stride=$stride',
makeTestDHTShortArrayCreateDelete(stride: stride));
test('add shortarray stride=$stride',
makeTestDHTShortArrayAdd(stride: stride));
}
});
// group('DHTLog Tests', () {
// setUpAll(dhtRecordPoolFixture.setUp);
// tearDownAll(dhtRecordPoolFixture.tearDown);
group('DHTLog Tests', () {
setUpAll(dhtRecordPoolFixture.setUp);
tearDownAll(dhtRecordPoolFixture.tearDown);
// for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
// test('create log stride=$stride',
// makeTestDHTLogCreateDelete(stride: stride));
// test(
// timeout: const Timeout(Duration(seconds: 480)),
// 'add/truncate log stride=$stride',
// makeTestDHTLogAddTruncate(stride: stride),
// );
// }
// });
// });
for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) {
test('create log stride=$stride',
makeTestDHTLogCreateDelete(stride: stride));
test(
timeout: const Timeout(Duration(seconds: 480)),
'add/truncate log stride=$stride',
makeTestDHTLogAddTruncate(stride: stride),
);
}
});
});
});
});
}

91
packages/veilid_support/example/integration_test/test_table_db_array.dart
View File

@ -84,7 +84,8 @@ Future<void> Function() makeTestTableDBArrayAddGetClear(
{
for (var n = batchSize; n < count; n += batchSize) {
final toGet = min(batchSize, count - n);
expect(await arr.getRange(n, toGet), equals(makeDataBatch(n, toGet)));
expect(await arr.getRange(n, n + toGet),
equals(makeDataBatch(n, toGet)));
}
}
@ -140,7 +141,7 @@ Future<void> Function() makeTestTableDBArrayInsert(
{
for (var n = batchSize; n < count; n += batchSize) {
final toGet = min(batchSize, count - n);
expect(await arr.getRange(n, toGet),
expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet)));
}
}
@ -154,7 +155,6 @@ Future<void> Function() makeTestTableDBArrayInsert(
await arr.close(delete: true);
};
Future<void> Function() makeTestTableDBArrayRemove(
{required int count,
required int singles,
@ -164,42 +164,79 @@ Future<void> Function() makeTestTableDBArrayRemove(
final arr = await TableDBArray.make(table: 'testArray', crypto: crypto);
final match = <Uint8List>[];
xxx removal test
print('inserting');
{
for (var n = 0; n < count;) {
final start = n;
var toAdd = min(batchSize, count - n);
for (var s = 0; s < min(singles, toAdd); s++) {
final data = makeData(n);
await arr.insert(start, data);
match.insert(start, data);
toAdd--;
n++;
}
final rems = [
(0, 0),
(0, 1),
(0, batchSize),
(1, batchSize - 1),
(batchSize, 1),
(batchSize + 1, batchSize),
(batchSize - 1, batchSize + 1)
];
for (final rem in rems) {
print('adding ');
{
for (var n = match.length; n < count;) {
final toAdd = min(batchSize, count - n);
final data = makeDataBatch(n, toAdd);
await arr.insertAll(start, data);
match.insertAll(start, data);
await arr.addAll(data);
match.addAll(data);
n += toAdd;
print(' $n/$count');
}
expect(arr.length, equals(match.length));
}
print('get singles');
{
for (var n = 0; n < batchSize; n++) {
expect(await arr.get(n), equals(match[n]));
}
}
final start = rem.$1;
final length = rem.$2;
print('removing start=$start length=$length');
final out = Output<List<Uint8List>>();
await arr.removeRange(start, start + length, out: out);
expect(out.value, equals(match.sublist(start, start + length)));
match.removeRange(start, start + length);
expect(arr.length, equals(match.length));
print('get batch');
{
for (var n = batchSize; n < count; n += batchSize) {
final toGet = min(batchSize, count - n);
expect(await arr.getRange(n, toGet),
final checkCount = match.length;
for (var n = 0; n < checkCount;) {
final toGet = min(batchSize, checkCount - n);
expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet)));
n += toGet;
print(' $n/$checkCount');
}
}
}
{
final start = match.length - rem.$1 - rem.$2;
final length = rem.$2;
print('removing from end start=$start length=$length');
final out = Output<List<Uint8List>>();
await arr.removeRange(start, start + length, out: out);
expect(out.value, equals(match.sublist(start, start + length)));
match.removeRange(start, start + length);
expect(arr.length, equals(match.length));
print('get batch');
{
final checkCount = match.length;
for (var n = 0; n < checkCount;) {
final toGet = min(batchSize, checkCount - n);
expect(await arr.getRange(n, n + toGet),
equals(match.sublist(n, n + toGet)));
n += toGet;
print(' $n/$checkCount');
}
expect(arr.length, equals(match.length));
}
}
}
}

43
packages/veilid_support/lib/src/table_db_array.dart
View File

@ -112,13 +112,13 @@ class TableDBArray {
});
}
Future<List<Uint8List>> getRange(int start, int length) async {
Future<List<Uint8List>> getRange(int start, int end) async {
await _initWait();
return _mutex.protect(() async {
if (!_open) {
throw StateError('not open');
}
return _getRangeInner(start, length);
return _getRangeInner(start, end);
});
}
@ -127,11 +127,11 @@ class TableDBArray {
return _writeTransaction((t) async => _removeInner(t, pos, out: out));
}
Future<void> removeRange(int start, int length,
Future<void> removeRange(int start, int end,
{Output<List<Uint8List>>? out}) async {
await _initWait();
return _writeTransaction(
(t) async => _removeRangeInner(t, start, length, out: out));
(t) async => _removeRangeInner(t, start, end, out: out));
}
Future<void> clear() async {
@ -223,23 +223,34 @@ class TableDBArray {
return (await _loadEntry(entry))!;
}
Future<List<Uint8List>> _getRangeInner(int start, int length) async {
Future<List<Uint8List>> _getRangeInner(int start, int end) async {
final length = end - start;
if (length < 0) {
throw StateError('length should not be negative');
}
if (start < 0 || start >= _length) {
throw IndexError.withLength(start, _length);
}
if ((start + length) > _length) {
throw IndexError.withLength(start + length, _length);
if (end > _length) {
throw IndexError.withLength(end, _length);
}
final out = <Uint8List>[];
for (var pos = start; pos < (start + length); pos++) {
const batchSize = 16;
for (var pos = start; pos < end;) {
var batchLen = min(batchSize, end - pos);
final dws = DelayedWaitSet<Uint8List>();
while (batchLen > 0) {
final entry = await _getIndexEntry(pos);
final value = (await _loadEntry(entry))!;
out.add(value);
dws.add(() async => (await _loadEntry(entry))!);
pos++;
batchLen--;
}
final batchOut = await dws();
out.addAll(batchOut);
}
return out;
}
@ -259,21 +270,21 @@ class TableDBArray {
await _removeIndexEntry(pos);
}
Future<void> _removeRangeInner(
VeilidTableDBTransaction t, int start, int length,
Future<void> _removeRangeInner(VeilidTableDBTransaction t, int start, int end,
{Output<List<Uint8List>>? out}) async {
final length = end - start;
if (length < 0) {
throw StateError('length should not be negative');
}
if (start < 0 || start >= _length) {
if (start < 0) {
throw IndexError.withLength(start, _length);
}
if ((start + length) > _length) {
throw IndexError.withLength(start + length, _length);
if (end > _length) {
throw IndexError.withLength(end, _length);
}
final outList = <Uint8List>[];
for (var pos = start; pos < (start + length); pos++) {
for (var pos = start; pos < end; pos++) {
final entry = await _getIndexEntry(pos);
if (out != null) {
final value = (await _loadEntry(entry))!;