Merge pull request #1748

bed2d9f2 Get rid of directory lock (Howard Chu) 2e913676 Handle map resizes from other processes (Howard Chu) bf1348b7 Can't cache num_txs or num_outputs either (Howard Chu) dc53e9ee Add a few read txns to streamline (Howard Chu)
2024-10-01 11:49:47 -04:00 · 2017-02-21 00:51:13 +02:00 · 2017-02-21 00:51:13 +02:00 · 675434ffab
commit 675434ffab
parent fb9e4e15ed bed2d9f231
5 changed files with 97 additions and 104 deletions
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@ -377,7 +377,50 @@ void mdb_txn_safe::allow_new_txns()
  creation_gate.clear();
 }

+void lmdb_resized(MDB_env *env)
+{
+  mdb_txn_safe::prevent_new_txns();

+  MGINFO("LMDB map resize detected.");
+
+  MDB_envinfo mei;
+
+  mdb_env_info(env, &mei);
+  uint64_t old = mei.me_mapsize;
+
+  mdb_txn_safe::wait_no_active_txns();
+
+  int result = mdb_env_set_mapsize(env, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
+
+  mdb_env_info(env, &mei);
+  uint64_t new_mapsize = mei.me_mapsize;
+
+  MGINFO("LMDB Mapsize increased." << "  Old: " << old / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
+
+  mdb_txn_safe::allow_new_txns();
+}
+
+inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
+{
+  int res = mdb_txn_begin(env, parent, flags, txn);
+  if (res == MDB_MAP_RESIZED) {
+    lmdb_resized(env);
+    res = mdb_txn_begin(env, parent, flags, txn);
+  }
+  return res;
+}
+
+inline int lmdb_txn_renew(MDB_txn *txn)
+{
+  int res = mdb_txn_renew(txn);
+  if (res == MDB_MAP_RESIZED) {
+    lmdb_resized(mdb_txn_env(txn));
+    res = mdb_txn_renew(txn);
+  }
+  return res;
+}

 void BlockchainLMDB::do_resize(uint64_t increase_size)
 {
@ -561,7 +604,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
  {
    LOG_PRINT_L1("No existing blocks to check for average block size");
  }
-  else if (m_cum_count)
+  else if (m_cum_count >= num_prev_blocks)
  {
    avg_block_size = m_cum_size / m_cum_count;
    LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
@ -570,6 +613,9 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
  }
  else
  {
+    MDB_txn *rtxn;
+    mdb_txn_cursors *rcurs;
+    block_rtxn_start(&rtxn, &rcurs);
    for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
    {
      uint32_t block_size = get_block_size(block_num);
@ -578,6 +624,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
      // some blocks were to be skipped for being outliers.
      ++num_blocks_used;
    }
+    block_rtxn_stop();
    avg_block_size = total_block_size / num_blocks_used;
    LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
  }
@ -702,7 +749,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
  uint64_t m_height = height();

  int result;
-  uint64_t tx_id = m_num_txs;
+  uint64_t tx_id = num_txs();

  CURSOR(txs)
  CURSOR(tx_indices)
@ -735,7 +782,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
  if (result)
    throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str()));

-  m_num_txs++;
  return tx_id;
 }

@ -783,8 +829,6 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
  // Don't delete the tx_indices entry until the end, after we're done with val_tx_id
  if (mdb_cursor_del(m_cur_tx_indices, 0))
      throw1(DB_ERROR("Failed to add removal of tx index to db transaction"));
-
-  m_num_txs--;
 }

 uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
@ -797,6 +841,7 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
  check_open();
  mdb_txn_cursors *m_cursors = &m_wcursors;
  uint64_t m_height = height();
+  uint64_t m_num_outputs = num_outputs();

  int result = 0;

@ -849,7 +894,6 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
  if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP)))
      throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));

-  m_num_outputs++;
  return ok.amount_index;
 }

@ -934,8 +978,6 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in
  result = mdb_cursor_del(m_cur_output_amounts, 0);
  if (result)
    throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
-
-  m_num_outputs--;
 }

 void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
@ -1000,7 +1042,7 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const

 void BlockchainLMDB::check_open() const
 {
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+//  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
  if (!m_open)
    throw0(DB_ERROR("DB operation attempted on a not-open DB instance"));
 }
@ -1155,16 +1197,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
  LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
  uint64_t m_height = db_stats.ms_entries;

-  // get and keep current number of txs
-  if ((result = mdb_stat(txn, m_txs, &db_stats)))
-    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
-  m_num_txs = db_stats.ms_entries;
-
-  // get and keep current number of outputs
-  if ((result = mdb_stat(txn, m_output_txs, &db_stats)))
-    throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
-  m_num_outputs = db_stats.ms_entries;
-
  bool compatible = true;

  MDB_val_copy<const char*> k("version");
@ -1270,7 +1302,7 @@ void BlockchainLMDB::reset()
  check_open();

  mdb_txn_safe txn;
-  if (auto result = mdb_txn_begin(m_env, NULL, 0, txn))
+  if (auto result = lmdb_txn_begin(m_env, NULL, 0, txn))
    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));

  if (auto result = mdb_drop(txn, m_blocks, 0))
@ -1304,7 +1336,6 @@ void BlockchainLMDB::reset()
    throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str()));

  txn.commit();
-  m_num_outputs = 0;
  m_cum_size = 0;
  m_cum_count = 0;
 }
@ -1354,7 +1385,7 @@ void BlockchainLMDB::unlock()
    txn_ptr = m_write_txn; \
  else \
  { \
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
      throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
  } \

@ -1388,7 +1419,7 @@ void BlockchainLMDB::unlock()
    txn_ptr = m_write_txn; \
  else \
  { \
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
      throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
  } \

@ -1714,6 +1745,34 @@ uint64_t BlockchainLMDB::height() const
  return db_stats.ms_entries;
 }

+uint64_t BlockchainLMDB::num_txs() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  // get current height
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_txs, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
+  return db_stats.ms_entries;
+}
+
+uint64_t BlockchainLMDB::num_outputs() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  // get current height
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_output_txs, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
+  return db_stats.ms_entries;
+}
+
 bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
 {
  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -2279,7 +2338,7 @@ bool BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
  m_write_batch_txn = new mdb_txn_safe();

  // NOTE: need to make sure it's destroyed properly when done
-  if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
+  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
  {
    delete m_write_batch_txn;
    m_write_batch_txn = nullptr;
@ -2398,12 +2457,12 @@ bool BlockchainLMDB::block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) co
    m_tinfo.reset(new mdb_threadinfo);
    memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors));
    memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
      throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
    ret = true;
  } else if (!m_tinfo->m_ti_rflags.m_rf_txn)
  {
-    if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn))
+    if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn))
      throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
    ret = true;
  }
@ -2436,12 +2495,12 @@ void BlockchainLMDB::block_txn_start(bool readonly)
      m_tinfo.reset(new mdb_threadinfo);
      memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors));
      memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
-      if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
+      if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
        throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
      didit = true;
    } else if (!m_tinfo->m_ti_rflags.m_rf_txn)
    {
-      if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn))
+      if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn))
        throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
      didit = true;
    }
@ -2467,7 +2526,7 @@ void BlockchainLMDB::block_txn_start(bool readonly)
  {
    m_writer = boost::this_thread::get_id();
    m_write_txn = new mdb_txn_safe();
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn))
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_txn))
    {
      delete m_write_txn;
      m_write_txn = nullptr;
@ -2545,8 +2604,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
    }
  }

-  uint64_t num_txs = m_num_txs;
-  uint64_t num_outputs = m_num_outputs;
  try
  {
    BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
@ -2557,8 +2614,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
  }
  catch (...)
  {
-    m_num_txs = num_txs;
-    m_num_outputs = num_outputs;
    block_txn_abort();
    throw;
  }
@ -2573,8 +2628,6 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)

  block_txn_start(false);

-  uint64_t num_txs = m_num_txs;
-  uint64_t num_outputs = m_num_outputs;
  try
  {
    BlockchainDB::pop_block(blk, txs);
@ -2582,8 +2635,6 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
  }
  catch (...)
  {
-    m_num_txs = num_txs;
-    m_num_outputs = num_outputs;
 	block_txn_abort();
    throw;
  }
@ -3257,8 +3308,6 @@ void BlockchainLMDB::migrate_0_1()
    lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
    mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
    txn.commit();
-    m_num_txs = 0;
-    m_num_outputs = 0;
  } while(0);

  do {
@ -3329,12 +3378,9 @@ void BlockchainLMDB::migrate_0_1()
          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str()));
        if (!i) {
          MDB_stat ms;
-          mdb_stat(txn, m_output_txs, &ms);
-          m_num_outputs = ms.ms_entries;
          mdb_stat(txn, m_txs, &ms);
-          m_num_txs = i = ms.ms_entries;
+          i = ms.ms_entries;
          if (i) {
-            m_num_txs = i;
            MDB_val_set(pk, "txblk");
            result = mdb_cursor_get(c_props, &pk, &k, MDB_SET);
            if (result)
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@ -312,6 +312,9 @@ private:

  virtual void remove_spent_key(const crypto::key_image& k_image);

+  uint64_t num_txs() const;
+  uint64_t num_outputs() const;
+
  // Hard fork
  virtual void set_hard_fork_version(uint64_t height, uint8_t version);
  virtual uint8_t get_hard_fork_version(uint64_t height) const;
@ -369,10 +372,8 @@ private:

  MDB_dbi m_properties;

-  uint64_t m_num_txs;
-  uint64_t m_num_outputs;
  mutable uint64_t m_cum_size;	// used in batch size estimation
-  mutable int m_cum_count;
+  mutable unsigned int m_cum_count;
  std::string m_folder;
  mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
  mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@ -1986,12 +1986,14 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
    return false;
  }

+  m_db->block_txn_start(true);
  resp.total_height = get_current_blockchain_height();
  size_t count = 0;
  for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++)
  {
    resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i));
  }
+  m_db->block_txn_stop();
  return true;
 }
 //------------------------------------------------------------------
@ -2022,6 +2024,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
    }
  }

+  m_db->block_txn_start(true);
  total_height = get_current_blockchain_height();
  size_t count = 0;
  for(size_t i = start_height; i < total_height && count < max_count; i++, count++)
@ -2032,6 +2035,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
    get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis);
    CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found");
  }
+  m_db->block_txn_stop();
  return true;
 }
 //------------------------------------------------------------------
--- a/src/cryptonote_core/cryptonote_core.cpp
+++ b/src/cryptonote_core/cryptonote_core.cpp
@ -219,40 +219,6 @@ namespace cryptonote
    return m_blockchain_storage.get_alternative_blocks_count();
  }
  //-----------------------------------------------------------------------------------------------
-  bool core::lock_db_directory(const boost::filesystem::path &path)
-  {
-    // boost doesn't like locking directories...
-    const boost::filesystem::path lock_path = path / ".daemon_lock";
-
-    try
-    {
-      // ensure the file exists
-      std::ofstream(lock_path.string(), std::ios::out).close();
-
-      db_lock = boost::interprocess::file_lock(lock_path.string().c_str());
-      LOG_PRINT_L1("Locking " << lock_path.string());
-      if (!db_lock.try_lock())
-      {
-        LOG_ERROR("Failed to lock " << lock_path.string());
-        return false;
-      }
-      return true;
-    }
-    catch (const std::exception &e)
-    {
-      LOG_ERROR("Error trying to lock " << lock_path.string() << ": " << e.what());
-      return false;
-    }
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::unlock_db_directory()
-  {
-    db_lock.unlock();
-    db_lock = boost::interprocess::file_lock();
-    LOG_PRINT_L1("Blockchain directory successfully unlocked");
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
  bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options)
  {
    start_time = std::time(nullptr);
@ -284,11 +250,6 @@ namespace cryptonote
    // make sure the data directory exists, and try to lock it
    CHECK_AND_ASSERT_MES (boost::filesystem::exists(folder) || boost::filesystem::create_directories(folder), false,
      std::string("Failed to create directory ").append(folder.string()).c_str());
-    if (!lock_db_directory (folder))
-    {
-      LOG_ERROR ("Failed to lock " << folder);
-      return false;
-    }

    // check for blockchain.bin
    try
@ -440,7 +401,6 @@ namespace cryptonote
    m_miner.stop();
    m_mempool.deinit();
    m_blockchain_storage.deinit();
-    unlock_db_directory();
    return true;
  }
  //-----------------------------------------------------------------------------------------------
--- a/src/cryptonote_core/cryptonote_core.h
+++ b/src/cryptonote_core/cryptonote_core.h
@ -763,24 +763,6 @@ namespace cryptonote
      */
     bool relay_txpool_transactions();

-     /**
-      * @brief locks a file in the BlockchainDB directory
-      *
-      * @param path the directory in which to place the file
-      *
-      * @return true if lock acquired successfully, otherwise false
-      */
-     bool lock_db_directory(const boost::filesystem::path &path);
-
-     /**
-      * @brief unlocks the db directory
-      *
-      * @note see lock_db_directory()
-      *
-      * @return true
-      */
-     bool unlock_db_directory();
-
     bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing)

     uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so