Store {O,C} for each leaf tuple instead of {O.x,I.x,C.x}

- Can derive {O.x,I.x,C.x} from {O,C}
- Note: this slows down tests since they do the derivation both
on insertion into the tree, and when auditing the tree
- At the hard fork, we don't need to store {O,C} in the
output_amounts table anymore since that table will no longer be
useful
This commit is contained in:
j-berman 2024-08-02 10:28:13 -07:00
parent 34eafa85f3
commit b90cee8bab
12 changed files with 144 additions and 85 deletions

View File

@ -231,7 +231,7 @@ std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash
{ {
// miner v2 txes have their coinbase output in one single out to save space, // miner v2 txes have their coinbase output in one single out to save space,
// and we store them as rct outputs with an identity mask // and we store them as rct outputs with an identity mask
// note: tx_outs_to_leaf_tuples in curve_trees.cpp mirrors this logic // note: tx_outs_to_leaf_tuple_contexts in curve_trees.cpp mirrors this logic
if (miner_tx && tx.version == 2) if (miner_tx && tx.version == 2)
{ {
cryptonote::tx_out vout = tx.vout[i]; cryptonote::tx_out vout = tx.vout[i];
@ -314,11 +314,11 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
// When adding a block, we also need to add all the leaf tuples included in // When adding a block, we also need to add all the leaf tuples included in
// the block to a table keeping track of locked leaf tuples. Once those leaf // the block to a table keeping track of locked leaf tuples. Once those leaf
// tuples unlock, we use them to grow the tree. // tuples unlock, we use them to grow the tree.
std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext> leaf_tuples_by_unlock_block; std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext> leaf_tuples_by_unlock_block;
// Get miner tx's leaf tuples // Get miner tx's leaf tuples
CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set"); CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set");
m_curve_trees->tx_outs_to_leaf_tuples( m_curve_trees->tx_outs_to_leaf_tuple_contexts(
blk.miner_tx, blk.miner_tx,
miner_output_ids, miner_output_ids,
prev_height, prev_height,
@ -328,7 +328,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
// Get all other txs' leaf tuples // Get all other txs' leaf tuples
for (std::size_t i = 0; i < txs.size(); ++i) for (std::size_t i = 0; i < txs.size(); ++i)
{ {
m_curve_trees->tx_outs_to_leaf_tuples( m_curve_trees->tx_outs_to_leaf_tuple_contexts(
txs[i].first, txs[i].first,
output_ids[i], output_ids[i],
prev_height, prev_height,

View File

@ -417,7 +417,7 @@ private:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) = 0; ) = 0;
/** /**
@ -1782,7 +1782,7 @@ public:
virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0; virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
// TODO: description and make private // TODO: description and make private
virtual void grow_tree(std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> &&new_leaves) = 0; virtual void grow_tree(std::vector<fcmp::curve_trees::LeafTupleContext> &&new_leaves) = 0;
virtual void trim_tree(const uint64_t trim_n_leaf_tuples) = 0; virtual void trim_tree(const uint64_t trim_n_leaf_tuples) = 0;

View File

@ -216,7 +216,6 @@ namespace
* *
* spent_keys input hash - * spent_keys input hash -
* *
* TODO: don't store leaf tuples, store reference to outputs
* locked_leaves block ID [{output ID, leaf tuple}...] * locked_leaves block ID [{output ID, leaf tuple}...]
* leaves leaf_idx {leaf tuple} * leaves leaf_idx {leaf tuple}
* layers layer_idx [{child_chunk_idx, child_chunk_hash}...] * layers layer_idx [{child_chunk_idx, child_chunk_hash}...]
@ -816,7 +815,7 @@ estim:
} }
void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
uint64_t num_rct_outs, const crypto::hash& blk_hash, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext> &leaf_tuples_by_unlock_block) uint64_t num_rct_outs, const crypto::hash& blk_hash, const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext> &leaf_tuples_by_unlock_block)
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open(); check_open();
@ -845,8 +844,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
} }
// Grow the tree with outputs that unlock at this block height // Grow the tree with outputs that unlock at this block height
auto unlocked_leaf_tuples = this->get_leaf_tuples_at_unlock_block_id(m_height); auto unlocked_leaves = this->get_leaf_tuples_at_unlock_block_id(m_height);
this->grow_tree(std::move(unlocked_leaf_tuples)); this->grow_tree(std::move(unlocked_leaves));
// Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked leaves table // Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked leaves table
this->del_locked_leaf_tuples_at_block_id(m_height); this->del_locked_leaf_tuples_at_block_id(m_height);
@ -1362,7 +1361,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
} }
} }
void BlockchainLMDB::grow_tree(std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> &&new_leaves) void BlockchainLMDB::grow_tree(std::vector<fcmp::curve_trees::LeafTupleContext> &&new_leaves)
{ {
if (new_leaves.empty()) if (new_leaves.empty())
return; return;
@ -1815,6 +1814,8 @@ fcmp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim BlockchainLMDB::get_las
if (trim_leaf_layer_instructions.end_trim_idx > trim_leaf_layer_instructions.start_trim_idx) if (trim_leaf_layer_instructions.end_trim_idx > trim_leaf_layer_instructions.start_trim_idx)
{ {
leaves_to_trim.reserve(trim_leaf_layer_instructions.end_trim_idx - trim_leaf_layer_instructions.start_trim_idx);
uint64_t idx = trim_leaf_layer_instructions.start_trim_idx; uint64_t idx = trim_leaf_layer_instructions.start_trim_idx;
CHECK_AND_ASSERT_THROW_MES(idx % fcmp::curve_trees::CurveTreesV1::LEAF_TUPLE_SIZE == 0, CHECK_AND_ASSERT_THROW_MES(idx % fcmp::curve_trees::CurveTreesV1::LEAF_TUPLE_SIZE == 0,
"expected divisble by leaf tuple size"); "expected divisble by leaf tuple size");
@ -1833,11 +1834,14 @@ fcmp::curve_trees::CurveTreesV1::LastChunkChildrenToTrim BlockchainLMDB::get_las
if (result != MDB_SUCCESS) if (result != MDB_SUCCESS)
throw0(DB_ERROR(lmdb_error("Failed to get leaf: ", result).c_str())); throw0(DB_ERROR(lmdb_error("Failed to get leaf: ", result).c_str()));
const auto leaf = *(fcmp::curve_trees::CurveTreesV1::LeafTuple *)v.mv_data; const auto preprocessed_leaf_tuple = *(fcmp::curve_trees::PreprocessedLeafTuple *)v.mv_data;
leaves_to_trim.push_back(leaf.O_x); // TODO: parallelize calls to this function
leaves_to_trim.push_back(leaf.I_x); auto leaf = m_curve_trees->leaf_tuple(preprocessed_leaf_tuple);
leaves_to_trim.push_back(leaf.C_x);
leaves_to_trim.emplace_back(std::move(leaf.O_x));
leaves_to_trim.emplace_back(std::move(leaf.I_x));
leaves_to_trim.emplace_back(std::move(leaf.C_x));
idx += fcmp::curve_trees::CurveTreesV1::LEAF_TUPLE_SIZE; idx += fcmp::curve_trees::CurveTreesV1::LEAF_TUPLE_SIZE;
} }
@ -2002,8 +2006,10 @@ bool BlockchainLMDB::audit_tree(const uint64_t expected_n_leaf_tuples) const
if (result != MDB_SUCCESS) if (result != MDB_SUCCESS)
throw0(DB_ERROR(lmdb_error("Failed to add leaf: ", result).c_str())); throw0(DB_ERROR(lmdb_error("Failed to add leaf: ", result).c_str()));
const auto leaf = *(fcmp::curve_trees::CurveTreesV1::LeafTuple *)v.mv_data; const auto preprocessed_leaf_tuple = *(fcmp::curve_trees::PreprocessedLeafTuple *)v.mv_data;
leaf_tuples_chunk.push_back(leaf); auto leaf = m_curve_trees->leaf_tuple(preprocessed_leaf_tuple);
leaf_tuples_chunk.emplace_back(std::move(leaf));
if (leaf_tuples_chunk.size() == m_curve_trees->m_c2_width) if (leaf_tuples_chunk.size() == m_curve_trees->m_c2_width)
break; break;
@ -2205,7 +2211,7 @@ bool BlockchainLMDB::audit_layer(const C_CHILD &c_child,
chunk_width); chunk_width);
} }
std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> BlockchainLMDB::get_leaf_tuples_at_unlock_block_id( std::vector<fcmp::curve_trees::LeafTupleContext> BlockchainLMDB::get_leaf_tuples_at_unlock_block_id(
uint64_t block_id) uint64_t block_id)
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -2218,7 +2224,7 @@ std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> BlockchainLMDB::g
MDB_val v_tuple; MDB_val v_tuple;
// Get all the locked outputs at the provided block id // Get all the locked outputs at the provided block id
std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> leaf_tuples; std::vector<fcmp::curve_trees::LeafTupleContext> leaf_tuples;
MDB_cursor_op op = MDB_SET; MDB_cursor_op op = MDB_SET;
while (1) while (1)
@ -2234,8 +2240,8 @@ std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> BlockchainLMDB::g
if (blk_id != block_id) if (blk_id != block_id)
throw0(DB_ERROR(("Blk id " + std::to_string(blk_id) + " not the expected" + std::to_string(block_id)).c_str())); throw0(DB_ERROR(("Blk id " + std::to_string(blk_id) + " not the expected" + std::to_string(block_id)).c_str()));
const auto range_begin = ((const fcmp::curve_trees::CurveTreesV1::LeafTupleContext*)v_tuple.mv_data); const auto range_begin = ((const fcmp::curve_trees::LeafTupleContext*)v_tuple.mv_data);
const auto range_end = range_begin + v_tuple.mv_size / sizeof(fcmp::curve_trees::CurveTreesV1::LeafTupleContext); const auto range_end = range_begin + v_tuple.mv_size / sizeof(fcmp::curve_trees::LeafTupleContext);
auto it = range_begin; auto it = range_begin;
@ -6772,25 +6778,27 @@ void BlockchainLMDB::migrate_5_6()
// Read the output data // Read the output data
uint64_t amount = *(const uint64_t*)k.mv_data; uint64_t amount = *(const uint64_t*)k.mv_data;
output_data_t output_data; output_data_t output_data;
fcmp::curve_trees::CurveTreesV1::LeafTupleContext tuple_context; uint64_t output_id;
if (amount == 0) if (amount == 0)
{ {
const outkey *okp = (const outkey *)v.mv_data; const outkey *okp = (const outkey *)v.mv_data;
output_data = okp->data; output_data = okp->data;
tuple_context.output_id = okp->output_id; output_id = okp->output_id;
} }
else else
{ {
const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
memcpy(&output_data, &okp->data, sizeof(pre_rct_output_data_t)); memcpy(&output_data, &okp->data, sizeof(pre_rct_output_data_t));
output_data.commitment = rct::zeroCommit(amount); output_data.commitment = rct::zeroCommit(amount);
tuple_context.output_id = okp->output_id; output_id = okp->output_id;
} }
// Convert the output into a leaf tuple // Convert the output into a leaf tuple context
fcmp::curve_trees::LeafTupleContext tuple_context;
try try
{ {
tuple_context.leaf_tuple = m_curve_trees->output_to_leaf_tuple( tuple_context = m_curve_trees->output_to_leaf_context(
output_id,
output_data.pubkey, output_data.pubkey,
rct::rct2pk(output_data.commitment)); rct::rct2pk(output_data.commitment));
} }
@ -6890,8 +6898,8 @@ void BlockchainLMDB::migrate_5_6()
} }
// Get the leaf tuples that unlock at the given block // Get the leaf tuples that unlock at the given block
auto unlocked_leaf_tuples = this->get_leaf_tuples_at_unlock_block_id(i); auto unlocked_leaves = this->get_leaf_tuples_at_unlock_block_id(i);
this->grow_tree(std::move(unlocked_leaf_tuples)); this->grow_tree(std::move(unlocked_leaves));
// Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked leaves table // Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked leaves table
this->del_locked_leaf_tuples_at_block_id(i); this->del_locked_leaf_tuples_at_block_id(i);

View File

@ -368,7 +368,7 @@ public:
static int compare_string(const MDB_val *a, const MDB_val *b); static int compare_string(const MDB_val *a, const MDB_val *b);
// make private // make private
virtual void grow_tree(std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> &&new_leaves); virtual void grow_tree(std::vector<fcmp::curve_trees::LeafTupleContext> &&new_leaves);
virtual void trim_tree(const uint64_t trim_n_leaf_tuples); virtual void trim_tree(const uint64_t trim_n_leaf_tuples);
@ -388,7 +388,7 @@ private:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& block_hash , const crypto::hash& block_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
); );
virtual void remove_block(); virtual void remove_block();
@ -449,7 +449,7 @@ private:
const uint64_t child_chunk_idx, const uint64_t child_chunk_idx,
const uint64_t chunk_width) const; const uint64_t chunk_width) const;
std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> get_leaf_tuples_at_unlock_block_id(uint64_t block_id); std::vector<fcmp::curve_trees::LeafTupleContext> get_leaf_tuples_at_unlock_block_id(uint64_t block_id);
void del_locked_leaf_tuples_at_block_id(uint64_t block_id); void del_locked_leaf_tuples_at_block_id(uint64_t block_id);

View File

@ -116,7 +116,7 @@ public:
virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
virtual void add_spent_key(const crypto::key_image& k_image) override {} virtual void add_spent_key(const crypto::key_image& k_image) override {}
virtual void remove_spent_key(const crypto::key_image& k_image) override {} virtual void remove_spent_key(const crypto::key_image& k_image) override {}
virtual void grow_tree(std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> &&new_leaves) override {}; virtual void grow_tree(std::vector<fcmp::curve_trees::LeafTupleContext> &&new_leaves) override {};
virtual void trim_tree(const uint64_t trim_n_leaf_tuples) override {}; virtual void trim_tree(const uint64_t trim_n_leaf_tuples) override {};
virtual bool audit_tree(const uint64_t expected_n_leaf_tuples) const override { return false; }; virtual bool audit_tree(const uint64_t expected_n_leaf_tuples) const override { return false; };
@ -147,7 +147,7 @@ public:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) override { } ) override { }
virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); } virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); }
virtual void set_hard_fork_version(uint64_t height, uint8_t version) override {} virtual void set_hard_fork_version(uint64_t height, uint8_t version) override {}

View File

@ -153,6 +153,7 @@ static LayerExtension<C> hash_children_chunks(const C &curve,
std::size_t chunk_start_idx = chunk_size; std::size_t chunk_start_idx = chunk_size;
while (chunk_start_idx < new_child_scalars.size()) while (chunk_start_idx < new_child_scalars.size())
{ {
// TODO: this loop can be parallelized
chunk_size = std::min(chunk_width, new_child_scalars.size() - chunk_start_idx); chunk_size = std::min(chunk_width, new_child_scalars.size() - chunk_start_idx);
const auto chunk_start = new_child_scalars.data() + chunk_start_idx; const auto chunk_start = new_child_scalars.data() + chunk_start_idx;
@ -626,7 +627,8 @@ static typename fcmp::curve_trees::LayerReduction<C_PARENT> get_next_layer_reduc
// CurveTrees public member functions // CurveTrees public member functions
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
template<> template<>
CurveTrees<Helios, Selene>::LeafTuple CurveTrees<Helios, Selene>::output_to_leaf_tuple( LeafTupleContext CurveTrees<Helios, Selene>::output_to_leaf_context(
const std::uint64_t output_id,
const crypto::public_key &output_pubkey, const crypto::public_key &output_pubkey,
const crypto::public_key &commitment) const const crypto::public_key &commitment) const
{ {
@ -646,11 +648,26 @@ CurveTrees<Helios, Selene>::LeafTuple CurveTrees<Helios, Selene>::output_to_leaf
}; };
// Torsion clear the output pub key and commitment // Torsion clear the output pub key and commitment
const rct::key rct_O = clear_torsion(output_pubkey, "output pub key"); rct::key O = clear_torsion(output_pubkey, "output pub key");
const rct::key rct_C = clear_torsion(commitment, "commitment"); rct::key C = clear_torsion(commitment, "commitment");
const crypto::public_key &O = rct::rct2pk(rct_O); PreprocessedLeafTuple o_c{
const crypto::public_key &C = rct::rct2pk(rct_C); .O = std::move(O),
.C = std::move(C)
};
return LeafTupleContext{
.output_id = output_id,
.preprocessed_leaf_tuple = std::move(o_c)
};
};
//----------------------------------------------------------------------------------------------------------------------
template<>
CurveTrees<Helios, Selene>::LeafTuple CurveTrees<Helios, Selene>::leaf_tuple(
const PreprocessedLeafTuple &preprocessed_leaf_tuple) const
{
const crypto::public_key &O = rct::rct2pk(preprocessed_leaf_tuple.O);
const crypto::public_key &C = rct::rct2pk(preprocessed_leaf_tuple.C);
crypto::ec_point I; crypto::ec_point I;
crypto::derive_key_image_generator(O, I); crypto::derive_key_image_generator(O, I);
@ -680,11 +697,11 @@ std::vector<typename C2::Scalar> CurveTrees<C1, C2>::flatten_leaves(const std::v
}; };
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
template <> template <>
void CurveTrees<Helios, Selene>::tx_outs_to_leaf_tuples(const cryptonote::transaction &tx, void CurveTrees<Helios, Selene>::tx_outs_to_leaf_tuple_contexts(const cryptonote::transaction &tx,
const std::vector<uint64_t> &output_ids, const std::vector<uint64_t> &output_ids,
const uint64_t tx_height, const uint64_t tx_height,
const bool miner_tx, const bool miner_tx,
std::multimap<uint64_t, CurveTrees<Helios, Selene>::LeafTupleContext> &leaf_tuples_by_unlock_block_inout) const std::multimap<uint64_t, LeafTupleContext> &leaf_tuples_by_unlock_block_inout) const
{ {
const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, tx_height); const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, tx_height);
@ -692,6 +709,7 @@ void CurveTrees<Helios, Selene>::tx_outs_to_leaf_tuples(const cryptonote::transa
for (std::size_t i = 0; i < tx.vout.size(); ++i) for (std::size_t i = 0; i < tx.vout.size(); ++i)
{ {
// TODO: this loop can be parallelized
const auto &out = tx.vout[i]; const auto &out = tx.vout[i];
crypto::public_key output_public_key; crypto::public_key output_public_key;
@ -708,13 +726,11 @@ void CurveTrees<Helios, Selene>::tx_outs_to_leaf_tuples(const cryptonote::transa
? rct::zeroCommit(out.amount) ? rct::zeroCommit(out.amount)
: tx.rct_signatures.outPk[i].mask; : tx.rct_signatures.outPk[i].mask;
CurveTrees<Helios, Selene>::LeafTupleContext tuple_context; LeafTupleContext leaf_tuple_context;
tuple_context.output_id = output_ids[i];
try try
{ {
// Convert output to leaf tuple; throws if output is invalid // Convert output to leaf tuple context; throws if output is invalid
tuple_context.leaf_tuple = output_to_leaf_tuple( leaf_tuple_context = output_to_leaf_context(output_ids[i],
output_public_key, output_public_key,
rct::rct2pk(commitment)); rct::rct2pk(commitment));
} }
@ -724,7 +740,7 @@ void CurveTrees<Helios, Selene>::tx_outs_to_leaf_tuples(const cryptonote::transa
continue; continue;
}; };
leaf_tuples_by_unlock_block_inout.emplace(unlock_block, std::move(tuple_context)); leaf_tuples_by_unlock_block_inout.emplace(unlock_block, std::move(leaf_tuple_context));
} }
} }
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
@ -751,16 +767,25 @@ typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extensio
const auto sort_fn = [](const LeafTupleContext &a, const LeafTupleContext &b) { return a.output_id < b.output_id; }; const auto sort_fn = [](const LeafTupleContext &a, const LeafTupleContext &b) { return a.output_id < b.output_id; };
std::sort(new_leaf_tuples.begin(), new_leaf_tuples.end(), sort_fn); std::sort(new_leaf_tuples.begin(), new_leaf_tuples.end(), sort_fn);
// Copy the sorted leaves into the tree extension struct // Convert sorted pre-processed tuples into leaf tuples, place each element of each leaf tuple in a flat vector to
// TODO: don't copy here // be hashed, and place the pre-processed tuples in tree extension struct for insertion into the db
std::vector<typename C2::Scalar> flattened_leaves;
flattened_leaves.reserve(new_leaf_tuples.size() * LEAF_TUPLE_SIZE);
tree_extension.leaves.tuples.reserve(new_leaf_tuples.size()); tree_extension.leaves.tuples.reserve(new_leaf_tuples.size());
for (const auto &leaf : new_leaf_tuples) for (auto &l : new_leaf_tuples)
{ {
tree_extension.leaves.tuples.emplace_back(LeafTuple{ // TODO: this loop can be parallelized
.O_x = leaf.leaf_tuple.O_x, auto leaf = leaf_tuple(l.preprocessed_leaf_tuple);
.I_x = leaf.leaf_tuple.I_x,
.C_x = leaf.leaf_tuple.C_x flattened_leaves.emplace_back(std::move(leaf.O_x));
}); flattened_leaves.emplace_back(std::move(leaf.I_x));
flattened_leaves.emplace_back(std::move(leaf.C_x));
// We only need to store O and C in the db, the leaf tuple can be derived from O and C
tree_extension.leaves.tuples.emplace_back(PreprocessedLeafTuple{
.O = std::move(l.preprocessed_leaf_tuple.O),
.C = std::move(l.preprocessed_leaf_tuple.C)
});
} }
if (grow_layer_instructions.need_old_last_parent) if (grow_layer_instructions.need_old_last_parent)
@ -772,7 +797,7 @@ typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extensio
grow_layer_instructions.need_old_last_parent ? &existing_last_hashes.c2_last_hashes[0] : nullptr, grow_layer_instructions.need_old_last_parent ? &existing_last_hashes.c2_last_hashes[0] : nullptr,
grow_layer_instructions.start_offset, grow_layer_instructions.start_offset,
grow_layer_instructions.next_parent_start_index, grow_layer_instructions.next_parent_start_index,
this->flatten_leaves(tree_extension.leaves.tuples), flattened_leaves,
m_leaf_layer_chunk_width m_leaf_layer_chunk_width
); );
@ -865,6 +890,7 @@ typename CurveTrees<C1, C2>::TreeReduction CurveTrees<C1, C2>::get_tree_reductio
{ {
TreeReduction tree_reduction_out; TreeReduction tree_reduction_out;
CHECK_AND_ASSERT_THROW_MES(!trim_instructions.empty(), "missing trim instructions");
CHECK_AND_ASSERT_THROW_MES((trim_instructions[0].new_total_children % LEAF_TUPLE_SIZE) == 0, CHECK_AND_ASSERT_THROW_MES((trim_instructions[0].new_total_children % LEAF_TUPLE_SIZE) == 0,
"unexpected new total leaves"); "unexpected new total leaves");
const uint64_t new_total_leaf_tuples = trim_instructions[0].new_total_children / LEAF_TUPLE_SIZE; const uint64_t new_total_leaf_tuples = trim_instructions[0].new_total_children / LEAF_TUPLE_SIZE;

View File

@ -128,9 +128,27 @@ struct TrimLayerInstructions final
uint64_t end_trim_idx; uint64_t end_trim_idx;
}; };
// Output pub key and commitment, ready to be converted into a leaf tuple (from {O,C} -> {O.x, I.x, C.x})
struct PreprocessedLeafTuple final
{
// Output pubkey that has been checked valid and torsion cleared
rct::key O;
// Commitment that has been torsion cleared
rct::key C;
};
static_assert(sizeof(PreprocessedLeafTuple) == (32+32), "db expects 64 bytes for pre-processed leaf tuples");
// Contextual wrapper for a pre-processed leaf tuple
struct LeafTupleContext final
{
// Global output ID useful to order the leaf tuple for insertion into the tree
uint64_t output_id;
PreprocessedLeafTuple preprocessed_leaf_tuple;
};
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
// This class is useful help update the curve trees tree without needing to keep the entire tree in memory // This class is useful to help update the curve trees merkle tree without needing to keep the entire tree in memory
// - It requires instantiation with the C1 and C2 curve classes and widths, hardening the tree structure // - It requires instantiation with the C1 and C2 curve classes and widths, hardening the tree structure
// - It ties the C2 curve in the tree to the leaf layer // - It ties the C2 curve in the tree to the leaf layer
template<typename C1, typename C2> template<typename C1, typename C2>
@ -163,21 +181,13 @@ public:
static const std::size_t LEAF_TUPLE_SIZE = 3; static const std::size_t LEAF_TUPLE_SIZE = 3;
static_assert(sizeof(LeafTuple) == (sizeof(typename C2::Scalar) * LEAF_TUPLE_SIZE), "unexpected LeafTuple size"); static_assert(sizeof(LeafTuple) == (sizeof(typename C2::Scalar) * LEAF_TUPLE_SIZE), "unexpected LeafTuple size");
// Contextual wrapper for leaf tuple
struct LeafTupleContext final
{
// Global output ID useful to order the leaf tuple for insertion into the tree
uint64_t output_id;
LeafTuple leaf_tuple;
};
// Contiguous leaves in the tree, starting a specified start_idx in the leaf layer // Contiguous leaves in the tree, starting a specified start_idx in the leaf layer
struct Leaves final struct Leaves final
{ {
// Starting leaf tuple index in the leaf layer // Starting leaf tuple index in the leaf layer
uint64_t start_leaf_tuple_idx{0}; uint64_t start_leaf_tuple_idx{0};
// Contiguous leaves in a tree that start at the start_idx // Contiguous leaves in a tree that start at the start_idx
std::vector<LeafTuple> tuples; std::vector<PreprocessedLeafTuple> tuples;
}; };
// A struct useful to extend an existing tree // A struct useful to extend an existing tree
@ -221,13 +231,17 @@ public:
//member functions //member functions
public: public:
// Convert cryptonote output pub key and commitment to a leaf tuple for the curve trees tree // Convert cryptonote output pub key and commitment to a leaf tuple for the curve trees tree
LeafTuple output_to_leaf_tuple(const crypto::public_key &output_pubkey, const crypto::public_key &C) const; LeafTupleContext output_to_leaf_context(const std::uint64_t output_id,
const crypto::public_key &output_pubkey,
const crypto::public_key &C) const;
LeafTuple leaf_tuple(const PreprocessedLeafTuple &preprocessed_leaf_tuple) const;
// Flatten leaves [(O.x, I.x, C.x),(O.x, I.x, C.x),...] -> [scalar,scalar,scalar,scalar,scalar,scalar,...] // Flatten leaves [(O.x, I.x, C.x),(O.x, I.x, C.x),...] -> [scalar,scalar,scalar,scalar,scalar,scalar,...]
std::vector<typename C2::Scalar> flatten_leaves(const std::vector<LeafTuple> &leaves) const; std::vector<typename C2::Scalar> flatten_leaves(const std::vector<LeafTuple> &leaves) const;
// Convert cryptonote tx outs to leaf tuples, grouped by the leaf tuple unlock height // Convert cryptonote tx outs to contexts ready to be converted to leaf tuples, grouped by unlock height
void tx_outs_to_leaf_tuples(const cryptonote::transaction &tx, void tx_outs_to_leaf_tuple_contexts(const cryptonote::transaction &tx,
const std::vector<uint64_t> &output_ids, const std::vector<uint64_t> &output_ids,
const uint64_t tx_height, const uint64_t tx_height,
const bool miner_tx, const bool miner_tx,

View File

@ -65,7 +65,7 @@ public:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) override { ) override {
blocks.push_back({block_weight, long_term_block_weight}); blocks.push_back({block_weight, long_term_block_weight});
} }

View File

@ -88,7 +88,7 @@ namespace
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) override ) override
{ {
blocks.push_back({blk, blk_hash}); blocks.push_back({blk, blk_hash});

View File

@ -167,12 +167,14 @@ void CurveTreesGlobalTree::extend_tree(const CurveTreesV1::TreeExtension &tree_e
"unexpected leaf start idx"); "unexpected leaf start idx");
m_tree.leaves.reserve(m_tree.leaves.size() + tree_extension.leaves.tuples.size()); m_tree.leaves.reserve(m_tree.leaves.size() + tree_extension.leaves.tuples.size());
for (const auto &leaf : tree_extension.leaves.tuples) for (const auto &preprocessed_leaf_tuple : tree_extension.leaves.tuples)
{ {
auto leaf = m_curve_trees.leaf_tuple(preprocessed_leaf_tuple);
m_tree.leaves.emplace_back(CurveTreesV1::LeafTuple{ m_tree.leaves.emplace_back(CurveTreesV1::LeafTuple{
.O_x = leaf.O_x, .O_x = std::move(leaf.O_x),
.I_x = leaf.I_x, .I_x = std::move(leaf.I_x),
.C_x = leaf.C_x .C_x = std::move(leaf.C_x)
}); });
} }
@ -587,6 +589,9 @@ bool CurveTreesGlobalTree::audit_tree(const std::size_t expected_n_leaf_tuples)
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
void CurveTreesGlobalTree::log_last_hashes(const CurveTreesV1::LastHashes &last_hashes) void CurveTreesGlobalTree::log_last_hashes(const CurveTreesV1::LastHashes &last_hashes)
{ {
if (!el::Loggers::allowed(el::Level::Debug, "serialization"))
return;
const auto &c1_last_hashes = last_hashes.c1_last_hashes; const auto &c1_last_hashes = last_hashes.c1_last_hashes;
const auto &c2_last_hashes = last_hashes.c2_last_hashes; const auto &c2_last_hashes = last_hashes.c2_last_hashes;
@ -622,6 +627,9 @@ void CurveTreesGlobalTree::log_last_hashes(const CurveTreesV1::LastHashes &last_
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
void CurveTreesGlobalTree::log_tree_extension(const CurveTreesV1::TreeExtension &tree_extension) void CurveTreesGlobalTree::log_tree_extension(const CurveTreesV1::TreeExtension &tree_extension)
{ {
if (!el::Loggers::allowed(el::Level::Debug, "serialization"))
return;
const auto &c1_extensions = tree_extension.c1_layer_extensions; const auto &c1_extensions = tree_extension.c1_layer_extensions;
const auto &c2_extensions = tree_extension.c2_layer_extensions; const auto &c2_extensions = tree_extension.c2_layer_extensions;
@ -631,7 +639,8 @@ void CurveTreesGlobalTree::log_tree_extension(const CurveTreesV1::TreeExtension
MDEBUG("Leaf start idx: " << tree_extension.leaves.start_leaf_tuple_idx); MDEBUG("Leaf start idx: " << tree_extension.leaves.start_leaf_tuple_idx);
for (std::size_t i = 0; i < tree_extension.leaves.tuples.size(); ++i) for (std::size_t i = 0; i < tree_extension.leaves.tuples.size(); ++i)
{ {
const auto &leaf = tree_extension.leaves.tuples[i]; const auto &preprocessed_leaf_tuple = tree_extension.leaves.tuples[i];
const auto leaf = m_curve_trees.leaf_tuple(preprocessed_leaf_tuple);
const auto O_x = m_curve_trees.m_c2.to_string(leaf.O_x); const auto O_x = m_curve_trees.m_c2.to_string(leaf.O_x);
const auto I_x = m_curve_trees.m_c2.to_string(leaf.I_x); const auto I_x = m_curve_trees.m_c2.to_string(leaf.I_x);
@ -679,6 +688,9 @@ void CurveTreesGlobalTree::log_tree_extension(const CurveTreesV1::TreeExtension
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
void CurveTreesGlobalTree::log_tree() void CurveTreesGlobalTree::log_tree()
{ {
if (!el::Loggers::allowed(el::Level::Debug, "serialization"))
return;
MDEBUG("Tree has " << m_tree.leaves.size() << " leaves, " MDEBUG("Tree has " << m_tree.leaves.size() << " leaves, "
<< m_tree.c1_layers.size() << " helios layers, " << m_tree.c2_layers.size() << " selene layers"); << m_tree.c1_layers.size() << " helios layers, " << m_tree.c2_layers.size() << " selene layers");
@ -730,27 +742,26 @@ void CurveTreesGlobalTree::log_tree()
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
// Test helpers // Test helpers
//---------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------
static const std::vector<fcmp::curve_trees::CurveTreesV1::LeafTupleContext> generate_random_leaves(const CurveTreesV1 &curve_trees, static const std::vector<fcmp::curve_trees::LeafTupleContext> generate_random_leaves(const CurveTreesV1 &curve_trees,
const std::size_t old_n_leaf_tuples, const std::size_t old_n_leaf_tuples,
const std::size_t new_n_leaf_tuples) const std::size_t new_n_leaf_tuples)
{ {
std::vector<CurveTreesV1::LeafTupleContext> tuples; std::vector<fcmp::curve_trees::LeafTupleContext> tuples;
tuples.reserve(new_n_leaf_tuples); tuples.reserve(new_n_leaf_tuples);
for (std::size_t i = 0; i < new_n_leaf_tuples; ++i) for (std::size_t i = 0; i < new_n_leaf_tuples; ++i)
{ {
const std::uint64_t output_id = old_n_leaf_tuples + i;
// Generate random output tuple // Generate random output tuple
crypto::secret_key o,c; crypto::secret_key o,c;
crypto::public_key O,C; crypto::public_key O,C;
crypto::generate_keys(O, o, o, false); crypto::generate_keys(O, o, o, false);
crypto::generate_keys(C, c, c, false); crypto::generate_keys(C, c, c, false);
auto leaf_tuple = curve_trees.output_to_leaf_tuple(O, C); auto tuple_context = curve_trees.output_to_leaf_context(output_id, O, C);
tuples.emplace_back(fcmp::curve_trees::CurveTreesV1::LeafTupleContext{ tuples.emplace_back(std::move(tuple_context));
.output_id = old_n_leaf_tuples + i,
.leaf_tuple = std::move(leaf_tuple),
});
} }
return tuples; return tuples;

View File

@ -54,7 +54,7 @@ public:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) override { ) override {
blocks.push_back(blk); blocks.push_back(blk);
} }

View File

@ -58,7 +58,7 @@ public:
, const uint64_t& coins_generated , const uint64_t& coins_generated
, uint64_t num_rct_outs , uint64_t num_rct_outs
, const crypto::hash& blk_hash , const crypto::hash& blk_hash
, const std::multimap<uint64_t, fcmp::curve_trees::CurveTreesV1::LeafTupleContext>& leaf_tuples_by_unlock_block , const std::multimap<uint64_t, fcmp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
) override { ) override {
blocks.push_back({block_weight, long_term_block_weight}); blocks.push_back({block_weight, long_term_block_weight});
} }