diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index b5840240e..e34149e48 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -179,7 +179,7 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) +std::vector BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { const transaction &tx = txp.first; @@ -223,7 +223,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash); - std::vector amount_output_indices(tx.vout.size()); + std::vector output_indices(tx.vout.size()); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -231,21 +231,35 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair { // miner v2 txes have their coinbase output in one single out to save space, // and we store them as rct outputs with an identity mask + // note: tx_outs_to_leaf_tuples in curve_trees.cpp mirrors this logic if (miner_tx && tx.version == 2) { cryptonote::tx_out vout = tx.vout[i]; rct::key commitment = rct::zeroCommit(vout.amount); vout.amount = 0; - amount_output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time, + output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time, &commitment); } else { - amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time, + output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time, tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL); } } + + std::vector amount_output_indices; + std::vector output_ids; + amount_output_indices.reserve(output_indices.size()); + output_ids.reserve(output_indices.size()); + for (const auto &o_idx : output_indices) + { + amount_output_indices.push_back(o_idx.amount_index); + output_ids.push_back(o_idx.output_id); + } + add_tx_amount_output_indices(tx_id, amount_output_indices); + + return output_ids; } uint64_t BlockchainDB::add_block( const std::pair& blck @@ -273,9 +287,12 @@ uint64_t BlockchainDB::add_block( const std::pair& blck time1 = epee::misc_utils::get_tick_count(); + std::vector> output_ids; + output_ids.reserve(1 + txs.size()); + uint64_t num_rct_outs = 0; blobdata miner_bd = tx_to_blob(blk.miner_tx); - add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd))); + output_ids.push_back(add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)))); if (blk.miner_tx.version == 2) num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; @@ -283,7 +300,7 @@ uint64_t BlockchainDB::add_block( const std::pair& blck for (const std::pair& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; - add_transaction(blk_hash, tx, &tx_hash); + output_ids.push_back(add_transaction(blk_hash, tx, &tx_hash)); for (const auto &vout: tx.first.vout) { if (vout.amount == 0) @@ -297,20 +314,22 @@ uint64_t BlockchainDB::add_block( const std::pair& blck // When adding a block, we also need to add all the leaf tuples included in // the block to a table keeping track of locked leaf tuples. Once those leaf // tuples unlock, we use them to grow the tree. - std::multimap leaf_tuples_by_unlock_height; + std::multimap leaf_tuples_by_unlock_height; // Get miner tx's leaf tuples fcmp::curve_trees::curve_trees_v1.tx_outs_to_leaf_tuples( blk.miner_tx, + output_ids[0], prev_height, true/*miner_tx*/, leaf_tuples_by_unlock_height); // Get all other txs' leaf tuples - for (const auto &txp : txs) + for (std::size_t i = 0; i < txs.size(); ++i) { fcmp::curve_trees::curve_trees_v1.tx_outs_to_leaf_tuples( - txp.first, + txs[i].first, + output_ids[i+1], prev_height, false/*miner_tx*/, leaf_tuples_by_unlock_height); diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 522fe5838..05d95bdac 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -188,6 +188,14 @@ struct txpool_tx_meta_t } }; +/** + * @brief a struct containing output indexes for convenience + */ +struct output_indexes_t +{ + uint64_t amount_index; + uint64_t output_id; +}; #define DBF_SAFE 1 #define DBF_FAST 2 @@ -408,7 +416,7 @@ private: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) = 0; /** @@ -473,8 +481,9 @@ private: * future, this tracking (of the number, at least) should be moved to * this class, as it is necessary and the same among all BlockchainDB. * - * It returns an amount output index, which is the index of the output - * for its specified amount. + * It returns the output indexes, which contains an amount output index (the + * index of the output for its specified amount) and output id (the global + * index of the output among all outputs of any amount). * * This data should be stored in such a manner that the only thing needed to * reverse the process is the tx_out. @@ -487,9 +496,9 @@ private: * @param local_index index of the output in its transaction * @param unlock_time unlock time/height of the output * @param commitment the rct commitment to the output amount - * @return amount output index + * @return output indexes */ - virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0; + virtual output_indexes_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0; /** * @brief store amount output indices for a tx's outputs @@ -570,8 +579,10 @@ protected: * @param tx the transaction to add * @param tx_hash_ptr the hash of the transaction, if already calculated * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated + * + * @return the global output ids of all outputs inserted */ - void add_transaction(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); + std::vector add_transaction(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -1396,17 +1407,6 @@ public: */ virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; - // returns the total number of global outputs - /** - * @brief fetches the total number of global outputs - * - * The subclass should return a count of all outputs, - * or zero if there are none. - * * - * @return the number of global outputs - */ - virtual uint64_t get_num_global_outputs() const = 0; - /** * @brief return index of the first element (should be hidden, but isn't) * @@ -1780,7 +1780,7 @@ public: // TODO: description and make private virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, - const std::vector &new_leaves) = 0; + const std::vector &new_leaves) = 0; virtual void trim_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, const uint64_t trim_n_leaf_tuples) = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index a2f3470cd..dc2057120 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -216,7 +216,7 @@ namespace * * spent_keys input hash - * - * locked_leaves block ID [{leaf tuple}...] + * locked_leaves block ID [{output ID, leaf tuple}...] * leaves leaf_idx {leaf tuple} * layers layer_idx [{child_chunk_idx, child_chunk_hash}...] * @@ -817,7 +817,7 @@ estim: } void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - uint64_t num_rct_outs, const crypto::hash& blk_hash, const std::multimap &leaf_tuples_by_unlock_height) + uint64_t num_rct_outs, const crypto::hash& blk_hash, const std::multimap &leaf_tuples_by_unlock_height) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -848,10 +848,9 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l // Grow the tree with outputs that unlock at this block height const auto unlocked_leaf_tuples = this->get_locked_leaf_tuples_at_height(m_height); - // TODO: double check consistent order for inserting outputs into the tree this->grow_tree(fcmp::curve_trees::curve_trees_v1, unlocked_leaf_tuples); - // TODO: remove unlocked_leaf_tuples from the locked outputs table + // TODO: remove locked from the locked outputs table int result = 0; @@ -1119,7 +1118,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); } -uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, +output_indexes_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, @@ -1183,7 +1182,10 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); - return ok.amount_index; + return output_indexes_t{ + .amount_index = ok.amount_index, + .output_id = ok.output_id + }; } void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, @@ -1362,12 +1364,11 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) } void BlockchainLMDB::grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, - const std::vector &new_leaves) + const std::vector &new_leaves) { if (new_leaves.empty()) return; - // TODO: block_wtxn_start like pop_block, then call BlockchainDB::grow_tree LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; @@ -1970,6 +1971,8 @@ bool BlockchainLMDB::audit_tree(const fcmp::curve_trees::CurveTreesV1 &curve_tre const uint64_t actual_n_leaf_tuples = this->get_num_leaf_tuples(); CHECK_AND_ASSERT_MES(actual_n_leaf_tuples == expected_n_leaf_tuples, false, "unexpected num leaf tuples"); + MDEBUG("Auditing tree with " << actual_n_leaf_tuples << " leaf tuples"); + if (actual_n_leaf_tuples == 0) { // Make sure layers table is also empty @@ -2203,7 +2206,7 @@ bool BlockchainLMDB::audit_layer(const C_CHILD &c_child, chunk_width); } -std::vector BlockchainLMDB::get_locked_leaf_tuples_at_height( +std::vector BlockchainLMDB::get_locked_leaf_tuples_at_height( const uint64_t height) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2216,7 +2219,7 @@ std::vector BlockchainLMDB::get_lock MDB_val v_tuple; // Get all the locked outputs at that height - std::vector leaf_tuples; + std::vector leaf_tuples; // TODO: double check this gets all leaf tuples when it does multiple iters MDB_cursor_op op = MDB_SET; @@ -2233,8 +2236,8 @@ std::vector BlockchainLMDB::get_lock if (h != height) throw0(DB_ERROR(("Height " + std::to_string(h) + " not the expected" + std::to_string(height)).c_str())); - const auto range_begin = ((const fcmp::curve_trees::CurveTreesV1::LeafTuple*)v_tuple.mv_data); - const auto range_end = range_begin + v_tuple.mv_size / sizeof(fcmp::curve_trees::CurveTreesV1::LeafTuple); + const auto range_begin = ((const fcmp::curve_trees::CurveTreesV1::LeafTupleContext*)v_tuple.mv_data); + const auto range_end = range_begin + v_tuple.mv_size / sizeof(fcmp::curve_trees::CurveTreesV1::LeafTupleContext); auto it = range_begin; @@ -4399,27 +4402,6 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -uint64_t BlockchainLMDB::get_num_global_outputs() const -{ - LOG_PRINT_L3("BlockchainLMDB:: " << __func__); - check_open(); - - TXN_PREFIX_RDONLY(); - RCURSOR(output_amounts); - - MDB_stat db_stats; - int result = mdb_stat(m_txn, m_output_amounts, &db_stats); - uint64_t count = 0; - if (result != MDB_NOTFOUND) - { - if (result) - throw0(DB_ERROR(lmdb_error("Failed to query m_output_amounts: ", result).c_str())); - count = db_stats.ms_entries; - } - TXN_POSTFIX_RDONLY(); - return count; -} - output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -6688,6 +6670,9 @@ void BlockchainLMDB::migrate_5_6() // ... Could also require outputs be inserted all-or-nothing first, and then can pick up where left off for the tree // if any of leaves, layers, or block_infn tables exist, then locked_leaves migration should be complete + // TODO: I can keep track of the contiguous output_id inserted in a separate table used strictly for this migration + // On next run, read all outputs until we reach the highest contiguous output_id, then continue from there + do { // 1. Set up locked outputs table @@ -6711,7 +6696,7 @@ void BlockchainLMDB::migrate_5_6() MDB_cursor_op op = MDB_FIRST; - const uint64_t n_outputs = this->get_num_global_outputs(); + const uint64_t n_outputs = this->num_outputs(); i = 0; while (1) @@ -6763,23 +6748,25 @@ void BlockchainLMDB::migrate_5_6() uint64_t amount = *(const uint64_t*)k.mv_data; output_data_t output_data; + fcmp::curve_trees::CurveTreesV1::LeafTupleContext tuple_context; if (amount == 0) { const outkey *okp = (const outkey *)v.mv_data; output_data = okp->data; + tuple_context.output_id = okp->output_id; } else { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; memcpy(&output_data, &okp->data, sizeof(pre_rct_output_data_t)); output_data.commitment = rct::zeroCommit(amount); + tuple_context.output_id = okp->output_id; } // Convert the output into a leaf tuple - fcmp::curve_trees::CurveTreesV1::LeafTuple leaf_tuple; try { - leaf_tuple = fcmp::curve_trees::curve_trees_v1.output_to_leaf_tuple( + tuple_context.leaf_tuple = fcmp::curve_trees::curve_trees_v1.output_to_leaf_tuple( output_data.pubkey, rct::rct2pk(output_data.commitment)); } @@ -6792,9 +6779,9 @@ void BlockchainLMDB::migrate_5_6() // Get the block in which the output will unlock const uint64_t unlock_height = cryptonote::get_unlock_height(output_data.unlock_time, output_data.height); - // Now add the leaf tuple to the locked outputs table + // Now add the leaf tuple to the locked leaves table MDB_val_set(k_height, unlock_height); - MDB_val_set(v_tuple, leaf_tuple); + MDB_val_set(v_tuple, tuple_context); // MDB_NODUPDATA because no benefit to having duplicate outputs in the tree, only 1 can be spent // Can't use MDB_APPENDDUP because outputs aren't inserted in order sorted by unlock height diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 194477962..ab2d37e28 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -278,7 +278,6 @@ public: virtual uint64_t get_tx_block_height(const crypto::hash& h) const; virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual uint64_t get_num_global_outputs() const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const; virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) const; @@ -370,7 +369,7 @@ public: // make private virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, - const std::vector &new_leaves); + const std::vector &new_leaves); virtual void trim_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, const uint64_t trim_n_leaf_tuples); @@ -391,7 +390,7 @@ private: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& block_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ); virtual void remove_block(); @@ -400,7 +399,7 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual uint64_t add_output(const crypto::hash& tx_hash, + virtual output_indexes_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, @@ -451,7 +450,7 @@ private: const uint64_t child_chunk_idx, const uint64_t chunk_width) const; - std::vector get_locked_leaf_tuples_at_height(const uint64_t height); + std::vector get_locked_leaf_tuples_at_height(const uint64_t height); uint64_t num_outputs() const; @@ -547,6 +546,8 @@ private: mdb_txn_cursors m_wcursors; mutable boost::thread_specific_ptr m_tinfo; + // TODO: m_curve_trees + #if defined(__arm__) // force a value so it can compile with 32-bit ARM constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 5ca53b8c0..6925141cb 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -100,7 +100,6 @@ public: virtual std::vector get_tx_list(const std::vector& hlist) const override { return std::vector(); } virtual uint64_t get_tx_block_height(const crypto::hash& h) const override { return 0; } virtual uint64_t get_num_outputs(const uint64_t& amount) const override { return 1; } - virtual uint64_t get_num_global_outputs() const override { return 1; } virtual uint64_t get_indexing_base() const override { return 0; } virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override { return cryptonote::output_data_t(); } virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const override { return cryptonote::tx_out_index(); } @@ -113,12 +112,12 @@ public: virtual void remove_block() override { } virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {} - virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;} + virtual output_indexes_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return {0, 0};} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) override {} virtual void add_spent_key(const crypto::key_image& k_image) override {} virtual void remove_spent_key(const crypto::key_image& k_image) override {} virtual void grow_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, - const std::vector &new_leaves) override {}; + const std::vector &new_leaves) override {}; virtual void trim_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, const uint64_t trim_n_leaf_tuples) override {}; virtual bool audit_tree(const fcmp::curve_trees::CurveTreesV1 &curve_trees, const uint64_t expected_n_leaf_tuples) const override { return false; }; @@ -149,7 +148,7 @@ public: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) override { } virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); } virtual void set_hard_fork_version(uint64_t height, uint8_t version) override {} diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index cb400c358..03f77c051 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1673,8 +1673,10 @@ namespace cryptonote { const auto seconds_since_unlock = hf_v15_time - unlock_time; const auto blocks_since_unlock = seconds_since_unlock / DIFFICULTY_TARGET_V2; - CHECK_AND_ASSERT_THROW_MES(hf_v15_height > blocks_since_unlock, "unexpected blocks since unlock"); - unlock_height = hf_v15_height - blocks_since_unlock; + + unlock_height = hf_v15_height >= blocks_since_unlock + ? (hf_v15_height - blocks_since_unlock) + : default_unlock_height; } else { diff --git a/src/fcmp/curve_trees.cpp b/src/fcmp/curve_trees.cpp index c797f4f7c..13c265c9f 100644 --- a/src/fcmp/curve_trees.cpp +++ b/src/fcmp/curve_trees.cpp @@ -630,25 +630,27 @@ CurveTrees::LeafTuple CurveTrees::output_to_leaf const crypto::public_key &output_pubkey, const crypto::public_key &commitment) const { - CHECK_AND_ASSERT_THROW_MES(crypto::check_key(output_pubkey), "invalid output pub key"); + if (!crypto::check_key(output_pubkey)) + throw std::runtime_error("invalid output pub key"); - const auto clear_torsion = [](const crypto::public_key &key) + const auto clear_torsion = [](const crypto::public_key &key, const std::string &s) { // TODO: don't need to decompress and recompress points, can be optimized rct::key torsion_cleared_key = rct::scalarmultKey(rct::pk2rct(key), rct::INV_EIGHT); torsion_cleared_key = rct::scalarmult8(torsion_cleared_key); - CHECK_AND_ASSERT_THROW_MES(torsion_cleared_key != rct::I, "cannot equal identity"); + if (torsion_cleared_key == rct::I) + throw std::runtime_error(s + " cannot equal identity"); return torsion_cleared_key; }; // Torsion clear the output pub key and commitment - const rct::key rct_O = clear_torsion(output_pubkey); - const rct::key rct_C = clear_torsion(commitment); + const rct::key rct_O = clear_torsion(output_pubkey, "output pub key"); + const rct::key rct_C = clear_torsion(commitment, "commitment"); - const crypto::public_key O = rct::rct2pk(rct_O); - const crypto::public_key C = rct::rct2pk(rct_C); + const crypto::public_key &O = rct::rct2pk(rct_O); + const crypto::public_key &C = rct::rct2pk(rct_C); crypto::ec_point I; crypto::derive_key_image_generator(O, I); @@ -679,12 +681,15 @@ std::vector CurveTrees::flatten_leaves(const std::v //---------------------------------------------------------------------------------------------------------------------- template <> void CurveTrees::tx_outs_to_leaf_tuples(const cryptonote::transaction &tx, + const std::vector &output_ids, const uint64_t tx_height, const bool miner_tx, - std::multimap::LeafTuple> &leaf_tuples_by_unlock_height_inout) const + std::multimap::LeafTupleContext> &leaf_tuples_by_unlock_height_inout) const { const uint64_t unlock_height = cryptonote::get_unlock_height(tx.unlock_time, tx_height); + CHECK_AND_ASSERT_THROW_MES(tx.vout.size() == output_ids.size(), "unexpected size of output ids"); + for (std::size_t i = 0; i < tx.vout.size(); ++i) { const auto &out = tx.vout[i]; @@ -693,7 +698,13 @@ void CurveTrees::tx_outs_to_leaf_tuples(const cryptonote::transa if (!cryptonote::get_output_public_key(out, output_public_key)) throw std::runtime_error("Could not get an output public key from a tx output."); - const rct::key commitment = (miner_tx || tx.version < 2) + static_assert(CURRENT_TRANSACTION_VERSION == 2, "This section of code was written with 2 tx versions in mind. " + "Revisit this section and update for the new tx version."); + + if (!miner_tx && tx.version == 2) + CHECK_AND_ASSERT_THROW_MES(tx.rct_signatures.outPk.size() > i, "unexpected size of outPk"); + + const rct::key commitment = (miner_tx || tx.version != 2) ? rct::zeroCommit(out.amount) : tx.rct_signatures.outPk[i].mask; @@ -704,7 +715,12 @@ void CurveTrees::tx_outs_to_leaf_tuples(const cryptonote::transa output_public_key, rct::rct2pk(commitment)); - leaf_tuples_by_unlock_height_inout.emplace(unlock_height, std::move(leaf_tuple)); + auto tuple_context = CurveTrees::LeafTupleContext{ + .output_id = output_ids[i], + .leaf_tuple = std::move(leaf_tuple), + }; + + leaf_tuples_by_unlock_height_inout.emplace(unlock_height, std::move(tuple_context)); } catch (...) { /*continue*/ }; @@ -715,7 +731,7 @@ template typename CurveTrees::TreeExtension CurveTrees::get_tree_extension( const uint64_t old_n_leaf_tuples, const LastHashes &existing_last_hashes, - const std::vector &new_leaf_tuples) const + const std::vector &new_leaf_tuples) const { TreeExtension tree_extension; @@ -730,15 +746,21 @@ typename CurveTrees::TreeExtension CurveTrees::get_tree_extensio tree_extension.leaves.start_leaf_tuple_idx = grow_layer_instructions.old_total_children / LEAF_TUPLE_SIZE; - // Copy the leaves + // Sort the leaves by order they appear in the chain + // TODO: don't copy here + std::vector sorted_leaf_tuples = new_leaf_tuples; + const auto sort_fn = [](const LeafTupleContext &a, const LeafTupleContext &b) { return a.output_id < b.output_id; }; + std::sort(sorted_leaf_tuples.begin(), sorted_leaf_tuples.end(), sort_fn); + + // Copy the sorted leaves into the tree extension struct // TODO: don't copy here tree_extension.leaves.tuples.reserve(new_leaf_tuples.size()); - for (const auto &leaf : new_leaf_tuples) + for (const auto &leaf : sorted_leaf_tuples) { tree_extension.leaves.tuples.emplace_back(LeafTuple{ - .O_x = leaf.O_x, - .I_x = leaf.I_x, - .C_x = leaf.C_x + .O_x = leaf.leaf_tuple.O_x, + .I_x = leaf.leaf_tuple.I_x, + .C_x = leaf.leaf_tuple.C_x }); } @@ -751,7 +773,7 @@ typename CurveTrees::TreeExtension CurveTrees::get_tree_extensio grow_layer_instructions.need_old_last_parent ? &existing_last_hashes.c2_last_hashes[0] : nullptr, grow_layer_instructions.start_offset, grow_layer_instructions.next_parent_start_index, - this->flatten_leaves(new_leaf_tuples), + this->flatten_leaves(tree_extension.leaves.tuples), m_leaf_layer_chunk_width ); diff --git a/src/fcmp/curve_trees.h b/src/fcmp/curve_trees.h index 6bc6b599f..05802c68c 100644 --- a/src/fcmp/curve_trees.h +++ b/src/fcmp/curve_trees.h @@ -164,6 +164,14 @@ public: static const uint64_t LEAF_TUPLE_SIZE = 3; static_assert(sizeof(LeafTuple) == (sizeof(typename C2::Scalar) * LEAF_TUPLE_SIZE), "unexpected LeafTuple size"); + // Contextual wrapper for leaf tuple + struct LeafTupleContext final + { + // Global output ID useful to order the leaf tuple for insertion into the tree + uint64_t output_id; + LeafTuple leaf_tuple; + }; + // Contiguous leaves in the tree, starting a specified start_idx in the leaf layer struct Leaves final { @@ -221,15 +229,16 @@ public: // Convert cryptonote tx outs to leaf tuples, grouped by the leaf tuple unlock height void tx_outs_to_leaf_tuples(const cryptonote::transaction &tx, + const std::vector &output_ids, const uint64_t tx_height, const bool miner_tx, - std::multimap &leaf_tuples_by_unlock_height_inout) const; + std::multimap &leaf_tuples_by_unlock_height_inout) const; // Take in the existing number of leaf tuples and the existing last hashes of each layer in the tree, as well as new // leaves to add to the tree, and return a tree extension struct that can be used to extend a tree TreeExtension get_tree_extension(const uint64_t old_n_leaf_tuples, const LastHashes &existing_last_hashes, - const std::vector &new_leaf_tuples) const; + const std::vector &new_leaf_tuples) const; // Get instructions useful for trimming all existing layers in the tree std::vector get_trim_instructions( diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp index 26b0f1ab5..281bdb17c 100644 --- a/tests/block_weight/block_weight.cpp +++ b/tests/block_weight/block_weight.cpp @@ -65,7 +65,7 @@ public: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) override { blocks.push_back({block_weight, long_term_block_weight}); } diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index b0f77a024..c1876dbc1 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -88,7 +88,7 @@ namespace , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) override { blocks.push_back({blk, blk_hash}); diff --git a/tests/unit_tests/curve_trees.cpp b/tests/unit_tests/curve_trees.cpp index 2dc98d0ad..7acac6c4c 100644 --- a/tests/unit_tests/curve_trees.cpp +++ b/tests/unit_tests/curve_trees.cpp @@ -730,13 +730,14 @@ void CurveTreesGlobalTree::log_tree() //---------------------------------------------------------------------------------------------------------------------- // Test helpers //---------------------------------------------------------------------------------------------------------------------- -static const std::vector generate_random_leaves(const CurveTreesV1 &curve_trees, - const std::size_t num_leaves) +static const std::vector generate_random_leaves(const CurveTreesV1 &curve_trees, + const std::size_t old_n_leaf_tuples, + const std::size_t new_n_leaf_tuples) { - std::vector tuples; - tuples.reserve(num_leaves); + std::vector tuples; + tuples.reserve(new_n_leaf_tuples); - for (std::size_t i = 0; i < num_leaves; ++i) + for (std::size_t i = 0; i < new_n_leaf_tuples; ++i) { // Generate random output tuple crypto::secret_key o,c; @@ -746,7 +747,10 @@ static const std::vector generate_random_leaves(const C auto leaf_tuple = curve_trees.output_to_leaf_tuple(O, C); - tuples.emplace_back(std::move(leaf_tuple)); + tuples.emplace_back(fcmp::curve_trees::CurveTreesV1::LeafTupleContext{ + .output_id = old_n_leaf_tuples + i, + .leaf_tuple = std::move(leaf_tuple), + }); } return tuples; @@ -775,7 +779,7 @@ static bool grow_tree(CurveTreesV1 &curve_trees, // - The tree extension includes all elements we'll need to add to the existing tree when adding the new leaves const auto tree_extension = curve_trees.get_tree_extension(old_n_leaf_tuples, last_hashes, - generate_random_leaves(curve_trees, new_n_leaf_tuples)); + generate_random_leaves(curve_trees, old_n_leaf_tuples, new_n_leaf_tuples)); global_tree.log_tree_extension(tree_extension); @@ -852,14 +856,14 @@ static bool grow_tree_db(const std::size_t init_leaves, LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then extending by " << ext_leaves << " leaves"); - test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, init_leaves)); + test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, 0, init_leaves)); CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(curve_trees, init_leaves), false, "failed to add initial leaves to db"); MDEBUG("Successfully added initial " << init_leaves << " leaves to db, extending by " << ext_leaves << " leaves"); - test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, ext_leaves)); + test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, init_leaves, ext_leaves)); CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(curve_trees, init_leaves + ext_leaves), false, "failed to extend tree in db"); @@ -881,7 +885,7 @@ static bool trim_tree_db(const std::size_t init_leaves, LOG_PRINT_L1("Adding " << init_leaves << " leaves to db, then trimming by " << trim_leaves << " leaves"); - test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, init_leaves)); + test_db.m_db->grow_tree(curve_trees, generate_random_leaves(curve_trees, 0, init_leaves)); CHECK_AND_ASSERT_MES(test_db.m_db->audit_tree(curve_trees, init_leaves), false, "failed to add initial leaves to db"); diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 6c7a221d0..d5103809c 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -54,7 +54,7 @@ public: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) override { blocks.push_back(blk); } diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index 6e76de218..683527759 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -58,7 +58,7 @@ public: , const uint64_t& coins_generated , uint64_t num_rct_outs , const crypto::hash& blk_hash - , const std::multimap& leaf_tuples_by_unlock_height + , const std::multimap& leaf_tuples_by_unlock_height ) override { blocks.push_back({block_weight, long_term_block_weight}); }