From a0af217d9ad42ab21d8e79987d8a07eeb457045d Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 23 Oct 2014 19:47:36 -0400 Subject: [PATCH] Adding block data to LMDB BlockchainDB coded Still needs testing (and need to write a few more unit tests), but everything should be there. Lots of unfortunate duplication, but...well, I can't see a way around it using LMDB. A couple of other minor changes in this commit, only slightly relevant. --- .../BlockchainDB_impl/db_lmdb.cpp | 218 +++++++++++++++++- .../BlockchainDB_impl/db_lmdb.h | 12 +- src/cryptonote_core/blockchain_db.h | 36 ++- 3 files changed, 258 insertions(+), 8 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 7330ad867..23c5fd4fc 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -42,6 +42,14 @@ const char* LMDB_BLOCK_SIZES = "block_sizes"; const char* LMDB_BLOCK_DIFFS = "block_diffs"; const char* LMDB_BLOCK_COINS = "block_coins"; const char* LMDB_TXS = "txs"; +const char* LMDB_TX_HEIGHTS = "tx_heights"; +const char* LMDB_TX_OUTPUTS = "tx_outputs"; +const char* LMDB_OUTPUT_TXS = "output_txs"; +const char* LMDB_OUTPUT_INDICES = "output_indices"; +const char* LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* LMDB_OUTPUTS = "outputs"; +const char* LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* LMDB_SPENT_KEYS = "spent_keys"; void BlockchainLMDB::add_block( const block& blk , const size_t& block_size @@ -130,6 +138,42 @@ void BlockchainLMDB::remove_block() void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) { check_open(); + + crypto::hash h = get_transaction_hash(tx); + MDB_val val_h; + val_h.mv_size = sizeof(crypto::hash); + val_h.mv_data = &h; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + { + LOG_PRINT_L1("Attempting to add transaction that's already in the db"); + throw TX_EXISTS("Attempting to add transaction that's already in the db"); + } + + auto bd = tx_to_blob(tx); + + // const-correctness be trolling, yo + std::unique_ptr bd_cpy(new char[bd.size()]); + memcpy(bd_cpy.get(), bd.data(), bd.size()); + + MDB_val blob; + blob.mv_size = bd.size(); + blob.mv_data = bd_cpy.get(); + if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) + { + LOG_PRINT_L0("Failed to add tx blob to db transaction"); + throw DB_ERROR("Failed to add tx blob to db transaction"); + } + + MDB_val height; + height.mv_size = sizeof(uint64_t); + height.mv_data = &m_height; + if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) + { + LOG_PRINT_L0("Failed to add tx block height to db transaction"); + throw DB_ERROR("Failed to add tx block height to db transaction"); + } } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) @@ -140,6 +184,66 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { check_open(); + + MDB_val k; + MDB_val v; + + k.mv_size = sizeof(uint64_t); + k.mv_data = &m_num_outputs; + crypto::hash h_cpy = tx_hash; + v.mv_size = sizeof(crypto::hash); + v.mv_data = &h_cpy; + if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add output tx hash to db transaction"); + throw DB_ERROR("Failed to add output tx hash to db transaction"); + } + if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add tx output index to db transaction"); + throw DB_ERROR("Failed to add tx output index to db transaction"); + } + + uint64_t index_cpy = local_index; + v.mv_size = sizeof(uint64_t); + v.mv_data = &index_cpy; + if (mdb_put(*m_write_txn, m_output_indices, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add tx output index to db transaction"); + throw DB_ERROR("Failed to add tx output index to db transaction"); + } + + uint64_t amount = tx_output.amount; + v.mv_size = sizeof(uint64_t); + v.mv_data = &amount; + if (mdb_put(*m_write_txn, m_output_amounts, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add output amount to db transaction"); + throw DB_ERROR("Failed to add output amount to db transaction"); + } + + blobdata b; + t_serializable_object_to_blob(tx_output, b); + /* + * use this later to deserialize + std::stringstream ss; + ss << tx_blob; + binary_archive ba(ss); + bool r = ::serialization::serialize(ba, tx); + */ + v.mv_size = b.size(); + v.mv_data = &b; + if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add output to db transaction"); + throw DB_ERROR("Failed to add output to db transaction"); + } + if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add output global index to db transaction"); + throw DB_ERROR("Failed to add output global index to db transaction"); + } + } void BlockchainLMDB::remove_output(const tx_out& tx_output) @@ -150,6 +254,27 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { check_open(); + + crypto::key_image key = k_image; + MDB_val val_key; + val_key.mv_size = sizeof(crypto::key_image); + val_key.mv_data = &key; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + { + LOG_PRINT_L1("Attempting to add spent key image that's already in the db"); + throw KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"); + } + + char anything = '\0'; + unused.mv_size = sizeof(char); + unused.mv_data = &anything; + if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + { + LOG_PRINT_L1("Error adding spent key image to db transaction"); + throw DB_ERROR("Error adding spent key image to db transaction"); + } } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -260,11 +385,54 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to open db handle for m_block_coins"); throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_coins" ); } + if (mdb_dbi_open(txn, LMDB_TXS, MDB_CREATE, &m_txs)) { LOG_PRINT_L0("Failed to open db handle for m_txs"); throw DB_OPEN_FAILURE( "Failed to open db handle for m_txs" ); } + if (mdb_dbi_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, &m_tx_heights)) + { + LOG_PRINT_L0("Failed to open db handle for m_tx_heights"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_heights" ); + } + if (mdb_dbi_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, &m_tx_outputs)) + { + LOG_PRINT_L0("Failed to open db handle for m_tx_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_outputs" ); + } + + if (mdb_dbi_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, &m_output_txs)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_txs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_txs" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, &m_output_indices)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_indices"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_indices" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, &m_output_gindices)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_gindices"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_gindices" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, &m_output_amounts)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_amounts"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_amounts" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, &m_outputs)) + { + LOG_PRINT_L0("Failed to open db handle for m_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); + } + + if (mdb_dbi_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, &m_spent_keys)) + { + LOG_PRINT_L0("Failed to open db handle for m_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); + } // get and keep current height MDB_stat db_stats; @@ -275,6 +443,14 @@ void BlockchainLMDB::open(const std::string& filename) } m_height = db_stats.ms_entries; + // get and keep current number of outputs + if (mdb_stat(txn, m_outputs, &db_stats)) + { + LOG_PRINT_L0("Failed to query m_outputs"); + throw DB_ERROR("Failed to query m_outputs"); + } + m_num_outputs = db_stats.ms_entries; + // commit the transaction txn.commit(); @@ -347,6 +523,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) auto get_result = mdb_get(txn, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { + txn.commit(); LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); return false; } @@ -356,6 +533,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) throw DB_ERROR("DB error attempting to fetch block index from hash"); } + txn.commit(); return true; } @@ -525,7 +703,8 @@ block BlockchainLMDB::get_top_block() uint64_t BlockchainLMDB::height() { check_open(); - return 0; + + return m_height; } @@ -610,6 +789,27 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& bool BlockchainLMDB::has_key_image(const crypto::key_image& img) { check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::key_image key = img; + MDB_val val_key; + val_key.mv_size = sizeof(crypto::key_image); + val_key.mv_data = &key; + + MDB_val unused; + if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); return false; } @@ -629,12 +829,20 @@ uint64_t BlockchainLMDB::add_block( const block& blk } m_write_txn = &txn; - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - txn.commit(); + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + throw; + } - m_height++; - return m_height - 1; + return ++m_height; } } // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index b2cf5f30a..7b012e4f6 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -194,12 +194,20 @@ private: MDB_dbi m_block_coins; MDB_dbi m_txs; + MDB_dbi m_tx_heights; + MDB_dbi m_tx_outputs; - MDB_dbi m_spent; - MDB_dbi m_utxo; + MDB_dbi m_output_txs; + MDB_dbi m_output_indices; + MDB_dbi m_output_gindices; + MDB_dbi m_output_amounts; + MDB_dbi m_outputs; + + MDB_dbi m_spent_keys; bool m_open; uint64_t m_height; + uint64_t m_num_outputs; std::string m_folder; txn_safe* m_write_txn; }; diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index b3c73a0d8..f59678869 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -126,6 +126,8 @@ * TX_DNE * TX_EXISTS * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS */ namespace cryptonote @@ -302,7 +304,7 @@ class OUTPUT_DNE : public std::exception private: std::string m; public: - OUTPUT_DNE() : m("The transaction requested does not exist") { } + OUTPUT_DNE() : m("The output requested does not exist!") { } OUTPUT_DNE(const char* s) : m(s) { } virtual ~OUTPUT_DNE() { } @@ -313,6 +315,38 @@ class OUTPUT_DNE : public std::exception } }; +class OUTPUT_EXISTS : public std::exception +{ + private: + std::string m; + public: + OUTPUT_EXISTS() : m("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : m(s) { } + + virtual ~OUTPUT_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class KEY_IMAGE_EXISTS : public std::exception +{ + private: + std::string m; + public: + KEY_IMAGE_EXISTS() : m("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : m(s) { } + + virtual ~KEY_IMAGE_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + /*********************************** * End of Exception Definitions ***********************************/