Git-Mirrors/monero
1
0
Fork 0
mirror of https://github.com/monero-project/monero.git synced 2024-10-01 11:49:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

fcmp++ tests: init tree in db once, then copy db for tests

Browse Source 
- speeds up trim_tree test by 60%+
This commit is contained in:
j-berman 2024-09-09 14:44:17 -07:00
parent 9784ced3db
commit f47c60ec63
2 changed files with 120 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
tests/unit_tests/curve_trees.cpp
View File

@ -865,6 +865,8 @@ static bool grow_tree_db(const std::size_t expected_old_n_leaf_tuples,
std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db)
{
cryptonote::db_wtxn_guard guard(test_db.m_db);
CHECK_AND_ASSERT_MES(test_db.m_db->get_num_leaf_tuples() == (uint64_t)(expected_old_n_leaf_tuples),
false, "unexpected starting n leaf tuples in db");
@ -875,50 +877,24 @@ static bool grow_tree_db(const std::size_t expected_old_n_leaf_tuples,
return test_db.m_db->audit_tree(expected_old_n_leaf_tuples + n_leaves);
}
//----------------------------------------------------------------------------------------------------------------------
static bool grow_and_extend_tree_db(const std::size_t init_leaves,
const std::size_t ext_leaves,
std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db)
{
cryptonote::db_wtxn_guard guard(test_db.m_db);
LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then extending by " << ext_leaves << " leaves");
CHECK_AND_ASSERT_MES(grow_tree_db(0, init_leaves, curve_trees, test_db), false,
"failed to add initial leaves to db");
MDEBUG("Successfully added initial " << init_leaves << " leaves to db, extending by "
<< ext_leaves << " leaves");
CHECK_AND_ASSERT_MES(grow_tree_db(init_leaves, ext_leaves, curve_trees, test_db), false,
"failed to extend tree in db");
MDEBUG("Successfully extended tree in db by " << ext_leaves << " leaves");
return true;
}
//----------------------------------------------------------------------------------------------------------------------
static bool trim_tree_db(const std::size_t init_leaves,
static bool trim_tree_db(const std::size_t expected_old_n_leaf_tuples,
const std::size_t trim_leaves,
std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db)
{
cryptonote::db_wtxn_guard guard(test_db.m_db);
LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then trimming by " << trim_leaves << " leaves");
CHECK_AND_ASSERT_THROW_MES(expected_old_n_leaf_tuples >= trim_leaves, "cannot trim more leaves than exist");
CHECK_AND_ASSERT_THROW_MES(trim_leaves > 0, "must be trimming some leaves");
auto init_outputs = generate_random_leaves(*curve_trees, 0, init_leaves);
LOG_PRINT_L1("Trimming " << trim_leaves << " leaf tuples from tree with "
<< expected_old_n_leaf_tuples << " leaves in db");
test_db.m_db->grow_tree(std::move(init_outputs));
CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(init_leaves), false,
"failed to add initial leaves to db");
CHECK_AND_ASSERT_MES(test_db.m_db->get_num_leaf_tuples() == (uint64_t)(expected_old_n_leaf_tuples),
false, "trimming unexpected starting n leaf tuples in db");
MDEBUG("Successfully added initial " << init_leaves << " leaves to db, trimming by "
<< trim_leaves << " leaves");
// Can use 0 from trim_block_id since it's unused in tests
// Can use 0 for trim_block_id since it's unused in tests
test_db.m_db->trim_tree(trim_leaves, 0);
CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(init_leaves - trim_leaves), false,
CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(expected_old_n_leaf_tuples - trim_leaves), false,
"failed to trim tree in db");
MDEBUG("Successfully trimmed tree in db by " << trim_leaves << " leaves");
@ -944,10 +920,25 @@ static bool trim_tree_db(const std::size_t init_leaves,
\
unit_test::BlockchainLMDBTest test_db; \
//----------------------------------------------------------------------------------------------------------------------
#define BEGIN_INIT_TREE_ITER(curve_trees) \
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves) \
{ \
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves"); \
\
/* Init tree in memory */ \
CurveTreesGlobalTree global_tree(*curve_trees); \
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves)); \
\
/* Init tree in db */ \
INIT_BLOCKCHAIN_LMDB_TEST_DB(test_db, curve_trees); \
ASSERT_TRUE(grow_tree_db(0, init_leaves, curve_trees, test_db)); \
//----------------------------------------------------------------------------------------------------------------------
#define END_INIT_TREE_ITER(curve_trees) \
}; \
//----------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------
// Test
//----------------------------------------------------------------------------------------------------------------------
// TODO: init tree in db once, then extend a copy of that db
TEST(curve_trees, grow_tree)
{
// Use lower values for chunk width than prod so that we can quickly test a many-layer deep tree
@ -962,25 +953,24 @@ TEST(curve_trees, grow_tree)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves)
BEGIN_INIT_TREE_ITER(curve_trees)
// Then extend the tree with ext_leaves
for (std::size_t ext_leaves = 1; (init_leaves + ext_leaves) <= min_leaves_needed_for_tree_depth; ++ext_leaves)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves, ext_leaves));
// Then extend the tree with ext_leaves
for (std::size_t ext_leaves = 1; (init_leaves + ext_leaves) <= min_leaves_needed_for_tree_depth; ++ext_leaves)
{
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves, ext_leaves));
// Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees);
ASSERT_TRUE(grow_and_extend_tree_db(init_leaves, ext_leaves, curve_trees, test_db));
}
// Tree in db
// Copy the already existing db
unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
ASSERT_TRUE(grow_tree_db(init_leaves, ext_leaves, curve_trees, copy_db));
}
END_INIT_TREE_ITER()
}
//----------------------------------------------------------------------------------------------------------------------
TEST(curve_trees, trim_tree)
@ -997,28 +987,27 @@ TEST(curve_trees, trim_tree)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves)
BEGIN_INIT_TREE_ITER(curve_trees)
// Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
if (trim_leaves > init_leaves)
continue;
// Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
{
if (trim_leaves > init_leaves)
continue;
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy));
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy));
// Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees);
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, curve_trees, test_db));
}
// Tree in db
// Copy the already existing db
unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, copy_db));
}
END_INIT_TREE_ITER()
}
//----------------------------------------------------------------------------------------------------------------------
TEST(curve_trees, trim_tree_then_grow)
@ -1038,32 +1027,29 @@ TEST(curve_trees, trim_tree_then_grow)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves)
BEGIN_INIT_TREE_ITER(curve_trees)
// Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
if (trim_leaves > init_leaves)
continue;
// Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
{
if (trim_leaves > init_leaves)
continue;
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy));
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves - trim_leaves, grow_after_trim));
// Tree in memory
// Copy the already existing global tree
CurveTreesGlobalTree tree_copy(global_tree);
ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy));
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves - trim_leaves, grow_after_trim));
// Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees);
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, curve_trees, test_db));
cryptonote::db_wtxn_guard guard(test_db.m_db);
const std::size_t expected_n_leaves = init_leaves - trim_leaves + grow_after_trim;
ASSERT_TRUE(grow_tree_db(grow_after_trim, expected_n_leaves, curve_trees, test_db));
}
// Tree in db
// Copy the already existing db
unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, copy_db));
ASSERT_TRUE(grow_tree_db(init_leaves - trim_leaves, grow_after_trim, curve_trees, copy_db));
}
END_INIT_TREE_ITER()
}
//----------------------------------------------------------------------------------------------------------------------
// Make sure the result of hash_trim is the same as the equivalent hash_grow excluding the trimmed children

48
tests/unit_tests/unit_tests_utils.h
View File

@ -75,13 +75,21 @@ namespace unit_test
class BlockchainLMDBTest
{
public:
BlockchainLMDBTest() : m_temp_db_dir(boost::filesystem::temp_directory_path().string() + "/monero-lmdb-tests/")
BlockchainLMDBTest(bool is_copy = false) :
m_temp_db_dir(boost::filesystem::temp_directory_path().string() + "/monero-lmdb-tests/"),
m_is_copy{is_copy}
{}
~BlockchainLMDBTest()
{
delete m_db;
remove_files();
if (m_temp_db_dir.find("/monero-lmdb-tests/") == std::string::npos)
{
LOG_ERROR("unexpected temp db dir");
return;
}
if (!m_is_copy)
boost::filesystem::remove_all(m_temp_db_dir);
}
void init_new_db(std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> curve_trees)
@ -94,6 +102,7 @@ namespace unit_test
MDEBUG("Creating test db at path " << dir_path);
ASSERT_NO_THROW(this->m_db->open(dir_path));
m_cur_dir_path = dir_path;
}
void init_hardfork(cryptonote::HardFork *hardfork)
@ -102,18 +111,45 @@ namespace unit_test
this->m_db->set_hard_fork(hardfork);
}
void remove_files()
BlockchainLMDBTest *copy_db(std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> curve_trees)
{
boost::filesystem::remove_all(m_temp_db_dir);
CHECK_AND_ASSERT_THROW_MES(this->m_db != nullptr, "expected non-null m_db");
CHECK_AND_ASSERT_THROW_MES(this->m_cur_dir_path != "", "expected cur dir path set");
const boost::filesystem::path lmdb_data_path = boost::filesystem::path(m_cur_dir_path + "/data.mdb");
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::exists(lmdb_data_path), "did not find lmdb data file");
// Close db, copy db file, open copy, then reopen the db
this->m_db->close();
const auto temp_db_path = boost::filesystem::unique_path();
const std::string dest_path = m_temp_db_dir + temp_db_path.string();
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::create_directories(dest_path),
"failed to create new db dirs");
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::copy_file(lmdb_data_path, dest_path + "/data.mdb"),
"failed to copy db data");
// Open db copy
BlockchainLMDBTest *copy_db = new BlockchainLMDBTest(true/*is_copy*/);
copy_db->m_db = new cryptonote::BlockchainLMDB(true/*batch_transactions*/, curve_trees);
copy_db->m_db->open(dest_path);
copy_db->m_cur_dir_path = dest_path;
// Reopen original db so it's ready for use
this->m_db->open(m_cur_dir_path);
return copy_db;
}
cryptonote::BlockchainDB* m_db{nullptr};
const std::string m_temp_db_dir;
std::string m_cur_dir_path{""};
const bool m_is_copy{false};
};
}
#define INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees) \
test_db.init_new_db(curve_trees); \
#define INIT_BLOCKCHAIN_LMDB_TEST_DB(test_db, curve_trees) \
if (curve_trees != nullptr) \
test_db.init_new_db(curve_trees); \
auto hardfork = cryptonote::HardFork(*test_db.m_db, 1, 0); \
test_db.init_hardfork(&hardfork); \
auto scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \