Git-Mirrors/monero
1
0
Fork 0
mirror of https://github.com/monero-project/monero.git synced 2025-09-23 09:44:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

fcmp++ tests: init tree in db once, then copy db for tests

Browse source 
- speeds up trim_tree test by 60%+
This commit is contained in:
j-berman 2024-09-09 14:44:17 -07:00
parent 9784ced3db
commit f47c60ec63
2 changed files with 120 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

112
tests/unit_tests/curve_trees.cpp
View file

@ -865,6 +865,8 @@ static bool grow_tree_db(const std::size_t expected_old_n_leaf_tuples,
std::shared_ptr<CurveTreesV1> curve_trees, std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db) unit_test::BlockchainLMDBTest &test_db)
{ {
cryptonote::db_wtxn_guard guard(test_db.m_db);
CHECK_AND_ASSERT_MES(test_db.m_db->get_num_leaf_tuples() == (uint64_t)(expected_old_n_leaf_tuples), CHECK_AND_ASSERT_MES(test_db.m_db->get_num_leaf_tuples() == (uint64_t)(expected_old_n_leaf_tuples),
false, "unexpected starting n leaf tuples in db"); false, "unexpected starting n leaf tuples in db");
@ -875,50 +877,24 @@ static bool grow_tree_db(const std::size_t expected_old_n_leaf_tuples,
return test_db.m_db->audit_tree(expected_old_n_leaf_tuples + n_leaves); return test_db.m_db->audit_tree(expected_old_n_leaf_tuples + n_leaves);
} }
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
static bool grow_and_extend_tree_db(const std::size_t init_leaves, static bool trim_tree_db(const std::size_t expected_old_n_leaf_tuples,
const std::size_t ext_leaves,
std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db)
{
cryptonote::db_wtxn_guard guard(test_db.m_db);
LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then extending by " << ext_leaves << " leaves");
CHECK_AND_ASSERT_MES(grow_tree_db(0, init_leaves, curve_trees, test_db), false,
"failed to add initial leaves to db");
MDEBUG("Successfully added initial " << init_leaves << " leaves to db, extending by "
<< ext_leaves << " leaves");
CHECK_AND_ASSERT_MES(grow_tree_db(init_leaves, ext_leaves, curve_trees, test_db), false,
"failed to extend tree in db");
MDEBUG("Successfully extended tree in db by " << ext_leaves << " leaves");
return true;
}
//----------------------------------------------------------------------------------------------------------------------
static bool trim_tree_db(const std::size_t init_leaves,
const std::size_t trim_leaves, const std::size_t trim_leaves,
std::shared_ptr<CurveTreesV1> curve_trees,
unit_test::BlockchainLMDBTest &test_db) unit_test::BlockchainLMDBTest &test_db)
{ {
cryptonote::db_wtxn_guard guard(test_db.m_db); cryptonote::db_wtxn_guard guard(test_db.m_db);
LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then trimming by " << trim_leaves << " leaves"); CHECK_AND_ASSERT_THROW_MES(expected_old_n_leaf_tuples >= trim_leaves, "cannot trim more leaves than exist");
CHECK_AND_ASSERT_THROW_MES(trim_leaves > 0, "must be trimming some leaves");
auto init_outputs = generate_random_leaves(*curve_trees, 0, init_leaves); LOG_PRINT_L1("Trimming " << trim_leaves << " leaf tuples from tree with "
<< expected_old_n_leaf_tuples << " leaves in db");
test_db.m_db->grow_tree(std::move(init_outputs)); CHECK_AND_ASSERT_MES(test_db.m_db->get_num_leaf_tuples() == (uint64_t)(expected_old_n_leaf_tuples),
CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(init_leaves), false, false, "trimming unexpected starting n leaf tuples in db");
"failed to add initial leaves to db");
MDEBUG("Successfully added initial " << init_leaves << " leaves to db, trimming by " // Can use 0 for trim_block_id since it's unused in tests
<< trim_leaves << " leaves");
// Can use 0 from trim_block_id since it's unused in tests
test_db.m_db->trim_tree(trim_leaves, 0); test_db.m_db->trim_tree(trim_leaves, 0);
CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(init_leaves - trim_leaves), false, CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(expected_old_n_leaf_tuples - trim_leaves), false,
"failed to trim tree in db"); "failed to trim tree in db");
MDEBUG("Successfully trimmed tree in db by " << trim_leaves << " leaves"); MDEBUG("Successfully trimmed tree in db by " << trim_leaves << " leaves");
@ -944,10 +920,25 @@ static bool trim_tree_db(const std::size_t init_leaves,
\ \
unit_test::BlockchainLMDBTest test_db; \ unit_test::BlockchainLMDBTest test_db; \
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
#define BEGIN_INIT_TREE_ITER(curve_trees) \
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves) \
{ \
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves"); \
\
/* Init tree in memory */ \
CurveTreesGlobalTree global_tree(*curve_trees); \
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves)); \
\
/* Init tree in db */ \
INIT_BLOCKCHAIN_LMDB_TEST_DB(test_db, curve_trees); \
ASSERT_TRUE(grow_tree_db(0, init_leaves, curve_trees, test_db)); \
//----------------------------------------------------------------------------------------------------------------------
#define END_INIT_TREE_ITER(curve_trees) \
}; \
//----------------------------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
// Test // Test
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
// TODO: init tree in db once, then extend a copy of that db
TEST(curve_trees, grow_tree) TEST(curve_trees, grow_tree)
{ {
// Use lower values for chunk width than prod so that we can quickly test a many-layer deep tree // Use lower values for chunk width than prod so that we can quickly test a many-layer deep tree
@ -962,11 +953,7 @@ TEST(curve_trees, grow_tree)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth); INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves // First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves) BEGIN_INIT_TREE_ITER(curve_trees)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
// Then extend the tree with ext_leaves // Then extend the tree with ext_leaves
for (std::size_t ext_leaves = 1; (init_leaves + ext_leaves) <= min_leaves_needed_for_tree_depth; ++ext_leaves) for (std::size_t ext_leaves = 1; (init_leaves + ext_leaves) <= min_leaves_needed_for_tree_depth; ++ext_leaves)
@ -977,10 +964,13 @@ TEST(curve_trees, grow_tree)
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves, ext_leaves)); ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves, ext_leaves));
// Tree in db // Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees); // Copy the already existing db
ASSERT_TRUE(grow_and_extend_tree_db(init_leaves, ext_leaves, curve_trees, test_db)); unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
} INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
ASSERT_TRUE(grow_tree_db(init_leaves, ext_leaves, curve_trees, copy_db));
} }
END_INIT_TREE_ITER()
} }
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
TEST(curve_trees, trim_tree) TEST(curve_trees, trim_tree)
@ -997,11 +987,7 @@ TEST(curve_trees, trim_tree)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth); INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves // First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves) BEGIN_INIT_TREE_ITER(curve_trees)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
// Then trim by trim_leaves // Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves) for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
@ -1015,10 +1001,13 @@ TEST(curve_trees, trim_tree)
ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy)); ASSERT_TRUE(trim_tree_in_memory(init_leaves, trim_leaves, tree_copy));
// Tree in db // Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees); // Copy the already existing db
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, curve_trees, test_db)); unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
} INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, copy_db));
} }
END_INIT_TREE_ITER()
} }
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
TEST(curve_trees, trim_tree_then_grow) TEST(curve_trees, trim_tree_then_grow)
@ -1038,11 +1027,7 @@ TEST(curve_trees, trim_tree_then_grow)
INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth); INIT_CURVE_TREES_TEST(helios_chunk_width, selene_chunk_width, tree_depth);
// First initialize the tree with init_leaves // First initialize the tree with init_leaves
for (std::size_t init_leaves = 1; init_leaves <= min_leaves_needed_for_tree_depth; ++init_leaves) BEGIN_INIT_TREE_ITER(curve_trees)
{
LOG_PRINT_L1("Initializing tree with " << init_leaves << " leaves in memory");
CurveTreesGlobalTree global_tree(*curve_trees);
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, global_tree, 0, init_leaves));
// Then trim by trim_leaves // Then trim by trim_leaves
for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves) for (std::size_t trim_leaves = 1; trim_leaves <= min_leaves_needed_for_tree_depth; ++trim_leaves)
@ -1057,13 +1042,14 @@ TEST(curve_trees, trim_tree_then_grow)
ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves - trim_leaves, grow_after_trim)); ASSERT_TRUE(grow_tree_in_memory(*curve_trees, tree_copy, init_leaves - trim_leaves, grow_after_trim));
// Tree in db // Tree in db
INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees); // Copy the already existing db
ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, curve_trees, test_db)); unit_test::BlockchainLMDBTest copy_db = *test_db.copy_db(curve_trees);
cryptonote::db_wtxn_guard guard(test_db.m_db); INIT_BLOCKCHAIN_LMDB_TEST_DB(copy_db, nullptr);
const std::size_t expected_n_leaves = init_leaves - trim_leaves + grow_after_trim; ASSERT_TRUE(trim_tree_db(init_leaves, trim_leaves, copy_db));
ASSERT_TRUE(grow_tree_db(grow_after_trim, expected_n_leaves, curve_trees, test_db)); ASSERT_TRUE(grow_tree_db(init_leaves - trim_leaves, grow_after_trim, curve_trees, copy_db));
}
} }
END_INIT_TREE_ITER()
} }
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
// Make sure the result of hash_trim is the same as the equivalent hash_grow excluding the trimmed children // Make sure the result of hash_trim is the same as the equivalent hash_grow excluding the trimmed children

46
tests/unit_tests/unit_tests_utils.h
View file

@ -75,13 +75,21 @@ namespace unit_test
class BlockchainLMDBTest class BlockchainLMDBTest
{ {
public: public:
BlockchainLMDBTest() : m_temp_db_dir(boost::filesystem::temp_directory_path().string() + "/monero-lmdb-tests/") BlockchainLMDBTest(bool is_copy = false) :
m_temp_db_dir(boost::filesystem::temp_directory_path().string() + "/monero-lmdb-tests/"),
m_is_copy{is_copy}
{} {}
~BlockchainLMDBTest() ~BlockchainLMDBTest()
{ {
delete m_db; delete m_db;
remove_files(); if (m_temp_db_dir.find("/monero-lmdb-tests/") == std::string::npos)
{
LOG_ERROR("unexpected temp db dir");
return;
}
if (!m_is_copy)
boost::filesystem::remove_all(m_temp_db_dir);
} }
void init_new_db(std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> curve_trees) void init_new_db(std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> curve_trees)
@ -94,6 +102,7 @@ namespace unit_test
MDEBUG("Creating test db at path " << dir_path); MDEBUG("Creating test db at path " << dir_path);
ASSERT_NO_THROW(this->m_db->open(dir_path)); ASSERT_NO_THROW(this->m_db->open(dir_path));
m_cur_dir_path = dir_path;
} }
void init_hardfork(cryptonote::HardFork *hardfork) void init_hardfork(cryptonote::HardFork *hardfork)
@ -102,17 +111,44 @@ namespace unit_test
this->m_db->set_hard_fork(hardfork); this->m_db->set_hard_fork(hardfork);
} }
void remove_files() BlockchainLMDBTest *copy_db(std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> curve_trees)
{ {
boost::filesystem::remove_all(m_temp_db_dir); CHECK_AND_ASSERT_THROW_MES(this->m_db != nullptr, "expected non-null m_db");
CHECK_AND_ASSERT_THROW_MES(this->m_cur_dir_path != "", "expected cur dir path set");
const boost::filesystem::path lmdb_data_path = boost::filesystem::path(m_cur_dir_path + "/data.mdb");
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::exists(lmdb_data_path), "did not find lmdb data file");
// Close db, copy db file, open copy, then reopen the db
this->m_db->close();
const auto temp_db_path = boost::filesystem::unique_path();
const std::string dest_path = m_temp_db_dir + temp_db_path.string();
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::create_directories(dest_path),
"failed to create new db dirs");
CHECK_AND_ASSERT_THROW_MES(boost::filesystem::copy_file(lmdb_data_path, dest_path + "/data.mdb"),
"failed to copy db data");
// Open db copy
BlockchainLMDBTest *copy_db = new BlockchainLMDBTest(true/*is_copy*/);
copy_db->m_db = new cryptonote::BlockchainLMDB(true/*batch_transactions*/, curve_trees);
copy_db->m_db->open(dest_path);
copy_db->m_cur_dir_path = dest_path;
// Reopen original db so it's ready for use
this->m_db->open(m_cur_dir_path);
return copy_db;
} }
cryptonote::BlockchainDB* m_db{nullptr}; cryptonote::BlockchainDB* m_db{nullptr};
const std::string m_temp_db_dir; const std::string m_temp_db_dir;
std::string m_cur_dir_path{""};
const bool m_is_copy{false};
}; };
} }
#define INIT_BLOCKCHAIN_LMDB_TEST_DB(curve_trees) \ #define INIT_BLOCKCHAIN_LMDB_TEST_DB(test_db, curve_trees) \
if (curve_trees != nullptr) \
test_db.init_new_db(curve_trees); \ test_db.init_new_db(curve_trees); \
auto hardfork = cryptonote::HardFork(*test_db.m_db, 1, 0); \ auto hardfork = cryptonote::HardFork(*test_db.m_db, 1, 0); \
test_db.init_hardfork(&hardfork); \ test_db.init_hardfork(&hardfork); \