From 17f6dfce466585062c0e94343c2d5358d26b523e Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Mon, 27 May 2024 13:17:33 -0400 Subject: [PATCH] tabledbarray tests pass --- .../example/integration_test/app_test.dart | 218 +++++++++--------- .../integration_test/test_table_db_array.dart | 103 ++++++--- .../lib/src/table_db_array.dart | 45 ++-- 3 files changed, 207 insertions(+), 159 deletions(-) diff --git a/packages/veilid_support/example/integration_test/app_test.dart b/packages/veilid_support/example/integration_test/app_test.dart index b7a807f..6912fd3 100644 --- a/packages/veilid_support/example/integration_test/app_test.dart +++ b/packages/veilid_support/example/integration_test/app_test.dart @@ -40,77 +40,7 @@ void main() { group('TableDBArray Tests', () { // test('create/delete TableDBArray', testTableDBArrayCreateDelete); - // group('TableDBArray Add/Get Tests', () { - // for (final params in [ - // // - // (99, 3, 15), - // (100, 4, 16), - // (101, 5, 17), - // // - // (511, 3, 127), - // (512, 4, 128), - // (513, 5, 129), - // // - // (4095, 3, 1023), - // (4096, 4, 1024), - // (4097, 5, 1025), - // // - // (65535, 3, 16383), - // (65536, 4, 16384), - // (65537, 5, 16385), - // ]) { - // final count = params.$1; - // final singles = params.$2; - // final batchSize = params.$3; - - // test( - // // timeout: const Timeout(Duration(seconds: 480)), - // 'add/remove TableDBArray count = $count batchSize=$batchSize', - // makeTestTableDBArrayAddGetClear( - // count: count, - // singles: singles, - // batchSize: batchSize, - // crypto: const VeilidCryptoPublic()), - // ); - // } - // }); - - // group('TableDBArray Insert Tests', () { - // for (final params in [ - // // - // (99, 3, 15), - // (100, 4, 16), - // (101, 5, 17), - // // - // (511, 3, 127), - // (512, 4, 128), - // (513, 5, 129), - // // - // (4095, 3, 1023), - // (4096, 4, 1024), - // (4097, 5, 1025), - // // - // (65535, 3, 16383), - // (65536, 4, 16384), - // (65537, 5, 16385), - // ]) { - // final count = params.$1; - // final singles = params.$2; - // final batchSize = params.$3; - - // test( - // // timeout: const Timeout(Duration(seconds: 480)), - // 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize', - // makeTestTableDBArrayInsert( - // count: count, - // singles: singles, - // batchSize: batchSize, - // crypto: const VeilidCryptoPublic()), - // ); - // } - // }); - - group('TableDBArray Remove Tests', () { + group('TableDBArray Add/Get Tests', () { for (final params in [ // (99, 3, 15), @@ -134,7 +64,77 @@ void main() { final batchSize = params.$3; test( - // timeout: const Timeout(Duration(seconds: 480)), + timeout: const Timeout(Duration(seconds: 480)), + 'add/remove TableDBArray count = $count batchSize=$batchSize', + makeTestTableDBArrayAddGetClear( + count: count, + singles: singles, + batchSize: batchSize, + crypto: const VeilidCryptoPublic()), + ); + } + }); + + group('TableDBArray Insert Tests', () { + for (final params in [ + // + (99, 3, 15), + (100, 4, 16), + (101, 5, 17), + // + (511, 3, 127), + (512, 4, 128), + (513, 5, 129), + // + (4095, 3, 1023), + (4096, 4, 1024), + (4097, 5, 1025), + // + (65535, 3, 16383), + (65536, 4, 16384), + (65537, 5, 16385), + ]) { + final count = params.$1; + final singles = params.$2; + final batchSize = params.$3; + + test( + timeout: const Timeout(Duration(seconds: 480)), + 'insert TableDBArray count=$count singles=$singles batchSize=$batchSize', + makeTestTableDBArrayInsert( + count: count, + singles: singles, + batchSize: batchSize, + crypto: const VeilidCryptoPublic()), + ); + } + }); + + group('TableDBArray Remove Tests', () { + for (final params in [ + // + (99, 3, 15), + (100, 4, 16), + (101, 5, 17), + // + (511, 3, 127), + (512, 4, 128), + (513, 5, 129), + // + (4095, 3, 1023), + (4096, 4, 1024), + (4097, 5, 1025), + // + (16383, 3, 4095), + (16384, 4, 4096), + (16385, 5, 4097), + ]) { + final count = params.$1; + final singles = params.$2; + final batchSize = params.$3; + + test( + timeout: const Timeout(Duration(seconds: 480)), 'remove TableDBArray count=$count singles=$singles batchSize=$batchSize', makeTestTableDBArrayRemove( count: count, @@ -147,50 +147,50 @@ void main() { }); }); - // group('DHT Support Tests', () { - // setUpAll(updateProcessorFixture.setUp); - // setUpAll(tickerFixture.setUp); - // tearDownAll(tickerFixture.tearDown); - // tearDownAll(updateProcessorFixture.tearDown); + group('DHT Support Tests', () { + setUpAll(updateProcessorFixture.setUp); + setUpAll(tickerFixture.setUp); + tearDownAll(tickerFixture.tearDown); + tearDownAll(updateProcessorFixture.tearDown); - // test('create pool', testDHTRecordPoolCreate); + test('create pool', testDHTRecordPoolCreate); - // group('DHTRecordPool Tests', () { - // setUpAll(dhtRecordPoolFixture.setUp); - // tearDownAll(dhtRecordPoolFixture.tearDown); + group('DHTRecordPool Tests', () { + setUpAll(dhtRecordPoolFixture.setUp); + tearDownAll(dhtRecordPoolFixture.tearDown); - // test('create/delete record', testDHTRecordCreateDelete); - // test('record scopes', testDHTRecordScopes); - // test('create/delete deep record', testDHTRecordDeepCreateDelete); - // }); + test('create/delete record', testDHTRecordCreateDelete); + test('record scopes', testDHTRecordScopes); + test('create/delete deep record', testDHTRecordDeepCreateDelete); + }); - // group('DHTShortArray Tests', () { - // setUpAll(dhtRecordPoolFixture.setUp); - // tearDownAll(dhtRecordPoolFixture.tearDown); + group('DHTShortArray Tests', () { + setUpAll(dhtRecordPoolFixture.setUp); + tearDownAll(dhtRecordPoolFixture.tearDown); - // for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { - // test('create shortarray stride=$stride', - // makeTestDHTShortArrayCreateDelete(stride: stride)); - // test('add shortarray stride=$stride', - // makeTestDHTShortArrayAdd(stride: stride)); - // } - // }); + for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { + test('create shortarray stride=$stride', + makeTestDHTShortArrayCreateDelete(stride: stride)); + test('add shortarray stride=$stride', + makeTestDHTShortArrayAdd(stride: stride)); + } + }); - // group('DHTLog Tests', () { - // setUpAll(dhtRecordPoolFixture.setUp); - // tearDownAll(dhtRecordPoolFixture.tearDown); + group('DHTLog Tests', () { + setUpAll(dhtRecordPoolFixture.setUp); + tearDownAll(dhtRecordPoolFixture.tearDown); - // for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { - // test('create log stride=$stride', - // makeTestDHTLogCreateDelete(stride: stride)); - // test( - // timeout: const Timeout(Duration(seconds: 480)), - // 'add/truncate log stride=$stride', - // makeTestDHTLogAddTruncate(stride: stride), - // ); - // } - // }); - // }); + for (final stride in [256, 16 /*64, 32, 16, 8, 4, 2, 1 */]) { + test('create log stride=$stride', + makeTestDHTLogCreateDelete(stride: stride)); + test( + timeout: const Timeout(Duration(seconds: 480)), + 'add/truncate log stride=$stride', + makeTestDHTLogAddTruncate(stride: stride), + ); + } + }); + }); }); }); } diff --git a/packages/veilid_support/example/integration_test/test_table_db_array.dart b/packages/veilid_support/example/integration_test/test_table_db_array.dart index 81607a6..e67bc39 100644 --- a/packages/veilid_support/example/integration_test/test_table_db_array.dart +++ b/packages/veilid_support/example/integration_test/test_table_db_array.dart @@ -84,7 +84,8 @@ Future Function() makeTestTableDBArrayAddGetClear( { for (var n = batchSize; n < count; n += batchSize) { final toGet = min(batchSize, count - n); - expect(await arr.getRange(n, toGet), equals(makeDataBatch(n, toGet))); + expect(await arr.getRange(n, n + toGet), + equals(makeDataBatch(n, toGet))); } } @@ -140,7 +141,7 @@ Future Function() makeTestTableDBArrayInsert( { for (var n = batchSize; n < count; n += batchSize) { final toGet = min(batchSize, count - n); - expect(await arr.getRange(n, toGet), + expect(await arr.getRange(n, n + toGet), equals(match.sublist(n, n + toGet))); } } @@ -154,7 +155,6 @@ Future Function() makeTestTableDBArrayInsert( await arr.close(delete: true); }; - Future Function() makeTestTableDBArrayRemove( {required int count, required int singles, @@ -164,42 +164,79 @@ Future Function() makeTestTableDBArrayRemove( final arr = await TableDBArray.make(table: 'testArray', crypto: crypto); final match = []; -xxx removal test - print('inserting'); + { - for (var n = 0; n < count;) { - final start = n; - var toAdd = min(batchSize, count - n); - for (var s = 0; s < min(singles, toAdd); s++) { - final data = makeData(n); - await arr.insert(start, data); - match.insert(start, data); - toAdd--; - n++; + final rems = [ + (0, 0), + (0, 1), + (0, batchSize), + (1, batchSize - 1), + (batchSize, 1), + (batchSize + 1, batchSize), + (batchSize - 1, batchSize + 1) + ]; + for (final rem in rems) { + print('adding '); + { + for (var n = match.length; n < count;) { + final toAdd = min(batchSize, count - n); + final data = makeDataBatch(n, toAdd); + await arr.addAll(data); + match.addAll(data); + n += toAdd; + print(' $n/$count'); + } + expect(arr.length, equals(match.length)); } - final data = makeDataBatch(n, toAdd); - await arr.insertAll(start, data); - match.insertAll(start, data); - n += toAdd; + { + final start = rem.$1; + final length = rem.$2; + print('removing start=$start length=$length'); - print(' $n/$count'); - } - } + final out = Output>(); + await arr.removeRange(start, start + length, out: out); + expect(out.value, equals(match.sublist(start, start + length))); + match.removeRange(start, start + length); + expect(arr.length, equals(match.length)); - print('get singles'); - { - for (var n = 0; n < batchSize; n++) { - expect(await arr.get(n), equals(match[n])); - } - } + print('get batch'); + { + final checkCount = match.length; + for (var n = 0; n < checkCount;) { + final toGet = min(batchSize, checkCount - n); + expect(await arr.getRange(n, n + toGet), + equals(match.sublist(n, n + toGet))); + n += toGet; + print(' $n/$checkCount'); + } + } + } - print('get batch'); - { - for (var n = batchSize; n < count; n += batchSize) { - final toGet = min(batchSize, count - n); - expect(await arr.getRange(n, toGet), - equals(match.sublist(n, n + toGet))); + { + final start = match.length - rem.$1 - rem.$2; + final length = rem.$2; + print('removing from end start=$start length=$length'); + + final out = Output>(); + await arr.removeRange(start, start + length, out: out); + expect(out.value, equals(match.sublist(start, start + length))); + match.removeRange(start, start + length); + expect(arr.length, equals(match.length)); + + print('get batch'); + { + final checkCount = match.length; + for (var n = 0; n < checkCount;) { + final toGet = min(batchSize, checkCount - n); + expect(await arr.getRange(n, n + toGet), + equals(match.sublist(n, n + toGet))); + n += toGet; + print(' $n/$checkCount'); + } + expect(arr.length, equals(match.length)); + } + } } } diff --git a/packages/veilid_support/lib/src/table_db_array.dart b/packages/veilid_support/lib/src/table_db_array.dart index dbabd3a..51b15b8 100644 --- a/packages/veilid_support/lib/src/table_db_array.dart +++ b/packages/veilid_support/lib/src/table_db_array.dart @@ -112,13 +112,13 @@ class TableDBArray { }); } - Future> getRange(int start, int length) async { + Future> getRange(int start, int end) async { await _initWait(); return _mutex.protect(() async { if (!_open) { throw StateError('not open'); } - return _getRangeInner(start, length); + return _getRangeInner(start, end); }); } @@ -127,11 +127,11 @@ class TableDBArray { return _writeTransaction((t) async => _removeInner(t, pos, out: out)); } - Future removeRange(int start, int length, + Future removeRange(int start, int end, {Output>? out}) async { await _initWait(); return _writeTransaction( - (t) async => _removeRangeInner(t, start, length, out: out)); + (t) async => _removeRangeInner(t, start, end, out: out)); } Future clear() async { @@ -223,23 +223,34 @@ class TableDBArray { return (await _loadEntry(entry))!; } - Future> _getRangeInner(int start, int length) async { + Future> _getRangeInner(int start, int end) async { + final length = end - start; if (length < 0) { throw StateError('length should not be negative'); } if (start < 0 || start >= _length) { throw IndexError.withLength(start, _length); } - if ((start + length) > _length) { - throw IndexError.withLength(start + length, _length); + if (end > _length) { + throw IndexError.withLength(end, _length); } final out = []; - for (var pos = start; pos < (start + length); pos++) { - final entry = await _getIndexEntry(pos); - final value = (await _loadEntry(entry))!; - out.add(value); + const batchSize = 16; + + for (var pos = start; pos < end;) { + var batchLen = min(batchSize, end - pos); + final dws = DelayedWaitSet(); + while (batchLen > 0) { + final entry = await _getIndexEntry(pos); + dws.add(() async => (await _loadEntry(entry))!); + pos++; + batchLen--; + } + final batchOut = await dws(); + out.addAll(batchOut); } + return out; } @@ -259,21 +270,21 @@ class TableDBArray { await _removeIndexEntry(pos); } - Future _removeRangeInner( - VeilidTableDBTransaction t, int start, int length, + Future _removeRangeInner(VeilidTableDBTransaction t, int start, int end, {Output>? out}) async { + final length = end - start; if (length < 0) { throw StateError('length should not be negative'); } - if (start < 0 || start >= _length) { + if (start < 0) { throw IndexError.withLength(start, _length); } - if ((start + length) > _length) { - throw IndexError.withLength(start + length, _length); + if (end > _length) { + throw IndexError.withLength(end, _length); } final outList = []; - for (var pos = start; pos < (start + length); pos++) { + for (var pos = start; pos < end; pos++) { final entry = await _getIndexEntry(pos); if (out != null) { final value = (await _loadEntry(entry))!;