From 0a604a9786a1228b3d8fbfa846b31f81c7da473d Mon Sep 17 00:00:00 2001 From: j-berman Date: Wed, 4 Sep 2024 21:13:48 -0700 Subject: [PATCH] fcmp++: Enable trimming to empty tree --- src/blockchain_db/lmdb/db_lmdb.cpp | 91 ++++++++++++++++++++---------- src/fcmp_pp/curve_trees.cpp | 12 +++- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 95eb7d285..70b719e19 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -921,6 +921,8 @@ void BlockchainLMDB::remove_block() // must use h now; deleting from m_block_info will invalidate it mdb_block_info *bi = (mdb_block_info *)h.mv_data; + const uint64_t block_id = bi->bi_height; + const uint64_t old_n_leaf_tuples = bi->bi_n_leaf_tuples; blk_height bh = {bi->bi_hash, 0}; h.mv_data = (void *)&bh; h.mv_size = sizeof(bh); @@ -937,12 +939,10 @@ void BlockchainLMDB::remove_block() // Get n_leaf_tuples from the new tip so we can trim the curve trees tree to the new tip const uint64_t new_n_leaf_tuples = get_top_block_n_leaf_tuples(); - const uint64_t old_n_leaf_tuples = bi->bi_n_leaf_tuples; - if (new_n_leaf_tuples > old_n_leaf_tuples) throw1(DB_ERROR("Unexpected: more leaf tuples are in prev block, tree is expected to only grow")); - - this->trim_tree(old_n_leaf_tuples - new_n_leaf_tuples, bi->bi_height/*trim_block_id*/); + const uint64_t trim_n_leaf_tuples = old_n_leaf_tuples - new_n_leaf_tuples; + this->trim_tree(trim_n_leaf_tuples, block_id/*trim_block_id*/); } uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) @@ -1523,7 +1523,7 @@ void BlockchainLMDB::trim_tree(const uint64_t trim_n_leaf_tuples, const uint64_t CURSOR(layers) const uint64_t old_n_leaf_tuples = this->get_num_leaf_tuples(); - CHECK_AND_ASSERT_THROW_MES(old_n_leaf_tuples > trim_n_leaf_tuples, "cannot trim more leaves than exist"); + CHECK_AND_ASSERT_THROW_MES(old_n_leaf_tuples >= trim_n_leaf_tuples, "cannot trim more leaves than exist"); CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set"); const auto trim_instructions = m_curve_trees->get_trim_instructions(old_n_leaf_tuples, trim_n_leaf_tuples); @@ -1542,8 +1542,6 @@ void BlockchainLMDB::trim_tree(const uint64_t trim_n_leaf_tuples, const uint64_t CHECK_AND_ASSERT_THROW_MES((tree_reduction.new_total_leaf_tuples + trim_n_leaf_tuples) == old_n_leaf_tuples, "unexpected new total leaves"); - MDEBUG("Trimming " << trim_n_leaf_tuples << " leaf tuples"); - // Trim the leaves // TODO: trim_leaves MDB_val_set(k_block_id, trim_block_id); @@ -1579,12 +1577,13 @@ void BlockchainLMDB::trim_tree(const uint64_t trim_n_leaf_tuples, const uint64_t // TODO: trim_layers const auto &c2_layer_reductions = tree_reduction.c2_layer_reductions; const auto &c1_layer_reductions = tree_reduction.c1_layer_reductions; - CHECK_AND_ASSERT_THROW_MES(!c2_layer_reductions.empty(), "empty c2 layer reductions"); + + const std::size_t n_layers = c2_layer_reductions.size() + c1_layer_reductions.size(); bool use_c2 = true; uint64_t c2_idx = 0; uint64_t c1_idx = 0; - for (uint64_t i = 0; i < (c2_layer_reductions.size() + c1_layer_reductions.size()); ++i) + for (uint64_t i = 0; i < n_layers; ++i) { if (use_c2) { @@ -1606,29 +1605,51 @@ void BlockchainLMDB::trim_tree(const uint64_t trim_n_leaf_tuples, const uint64_t // Trim any remaining layers in layers after the root // TODO: trim_leftovers_after_root - const uint64_t expected_root_idx = c2_layer_reductions.size() + c1_layer_reductions.size() - 1; - while (1) + if (n_layers > 0) { - MDB_val k, v; - int result = mdb_cursor_get(m_cur_layers, &k, &v, MDB_LAST); - if (result != MDB_SUCCESS) - throw0(DB_ERROR(lmdb_error("Failed to get last elem: ", result).c_str())); - - const uint64_t last_layer_idx = *(uint64_t *)k.mv_data; - - if (last_layer_idx > expected_root_idx) + const uint64_t expected_root_idx = n_layers - 1; + while (1) { - // Delete all elements in layers after the root + MDB_val k, v; + int result = mdb_cursor_get(m_cur_layers, &k, &v, MDB_LAST); + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to get last elem: ", result).c_str())); + + const uint64_t last_layer_idx = *(uint64_t *)k.mv_data; + + if (last_layer_idx > expected_root_idx) + { + // Delete all elements in layers after the root + result = mdb_cursor_del(m_cur_layers, MDB_NODUPDATA); + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Error removing elems after root: ", result).c_str())); + } + else if (last_layer_idx < expected_root_idx) + { + throw0(DB_ERROR("Encountered unexpected last elem in tree before the root")); + } + else // last_layer_idx == expected_root_idx + { + // We've trimmed all layers past the root, we're done + break; + } + } + } + else // n_layers == 0 + { + // We're removing everything + while (1) + { + MDB_val k, v; + int result = mdb_cursor_get(m_cur_layers, &k, &v, MDB_LAST); + if (result == MDB_NOTFOUND) + break; + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to get last elem: ", result).c_str())); + result = mdb_cursor_del(m_cur_layers, MDB_NODUPDATA); - } - else if (last_layer_idx < expected_root_idx) - { - throw0(DB_ERROR("Encountered unexpected last elem in tree before the root")); - } - else // last_layer_idx == expected_root_idx - { - // We've trimmed all layers past the root, we're done - break; + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Error removing elems after root: ", result).c_str())); } } } @@ -1842,6 +1863,11 @@ fcmp_pp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim BlockchainLMDB::get_ const std::vector &trim_instructions) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + fcmp_pp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim last_chunk_children_to_trim; + if (trim_instructions.empty()) + return last_chunk_children_to_trim; + check_open(); CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set"); @@ -1849,7 +1875,6 @@ fcmp_pp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim BlockchainLMDB::get_ TXN_PREFIX_RDONLY(); RCURSOR(layers) - fcmp_pp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim last_chunk_children_to_trim; auto &c1_last_children_out = last_chunk_children_to_trim.c1_children; auto &c2_last_children_out = last_chunk_children_to_trim.c2_children; @@ -1965,12 +1990,16 @@ fcmp_pp::curve_trees::CurveTreesV1::LastHashes BlockchainLMDB::get_last_hashes_t const std::vector &trim_instructions) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + fcmp_pp::curve_trees::CurveTreesV1::LastHashes last_hashes_out; + if (trim_instructions.empty()) + return last_hashes_out; + check_open(); TXN_PREFIX_RDONLY(); RCURSOR(layers) - fcmp_pp::curve_trees::CurveTreesV1::LastHashes last_hashes_out; // Traverse the tree layer-by-layer starting at the layer closest to leaf layer uint64_t layer_idx = 0; diff --git a/src/fcmp_pp/curve_trees.cpp b/src/fcmp_pp/curve_trees.cpp index 3ed54a35c..98d18cf61 100644 --- a/src/fcmp_pp/curve_trees.cpp +++ b/src/fcmp_pp/curve_trees.cpp @@ -812,11 +812,14 @@ std::vector CurveTrees::get_trim_instructions( const uint64_t old_n_leaf_tuples, const uint64_t trim_n_leaf_tuples) const { - CHECK_AND_ASSERT_THROW_MES(old_n_leaf_tuples > trim_n_leaf_tuples, "cannot trim more leaves than exist"); + CHECK_AND_ASSERT_THROW_MES(old_n_leaf_tuples >= trim_n_leaf_tuples, "cannot trim more leaves than exist"); CHECK_AND_ASSERT_THROW_MES(trim_n_leaf_tuples > 0, "must be trimming some leaves"); std::vector trim_instructions; + if (old_n_leaf_tuples == trim_n_leaf_tuples) + return trim_instructions; + // Get trim instructions for the leaf layer { const uint64_t old_total_leaves = old_n_leaf_tuples * LEAF_TUPLE_SIZE; @@ -865,7 +868,12 @@ typename CurveTrees::TreeReduction CurveTrees::get_tree_reductio { TreeReduction tree_reduction_out; - CHECK_AND_ASSERT_THROW_MES(!trim_instructions.empty(), "missing trim instructions"); + if (trim_instructions.empty()) + { + tree_reduction_out.new_total_leaf_tuples = 0; + return tree_reduction_out; + } + CHECK_AND_ASSERT_THROW_MES((trim_instructions[0].new_total_children % LEAF_TUPLE_SIZE) == 0, "unexpected new total leaves"); const uint64_t new_total_leaf_tuples = trim_instructions[0].new_total_children / LEAF_TUPLE_SIZE;