Pruning

The blockchain prunes seven eighths of prunable tx data.
This saves about two thirds of the blockchain size, while
keeping the node useful as a sync source for an eighth
of the blockchain.

No other data is currently pruned.

There are three ways to prune a blockchain:

- run monerod with --prune-blockchain
- run "prune_blockchain" in the monerod console
- run the monero-blockchain-prune utility

The first two will prune in place. Due to how LMDB works, this
will not reduce the blockchain size on disk. Instead, it will
mark parts of the file as free, so that future data will use
that free space, causing the file to not grow until free space
grows scarce.

The third way will create a second database, a pruned copy of
the original one. Since this is a new file, this one will be
smaller than the original one.

Once the database is pruned, it will stay pruned as it syncs.
That is, there is no need to use --prune-blockchain again, etc.

src/cryptonote_protocol/block_queue.cpp

@@ -34,6 +34,7 @@
 #include <boost/uuid/uuid_io.hpp>
 #include "string_tools.h"
 #include "cryptonote_protocol_defs.h"
+#include "common/pruning.h"
 #include "block_queue.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -60,7 +61,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp
   if (has_hashes)
   {
     for (const crypto::hash &h: hashes)
+    {
       requested_hashes.insert(h);
+      have_blocks.insert(h);
+    }
     set_span_hashes(height, connection_id, hashes);
   }
 }
@@ -90,7 +94,10 @@ void block_queue::erase_block(block_map::iterator j)
 {
   CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator");
   for (const crypto::hash &h: j->hashes)
+  {
     requested_hashes.erase(h);
+    have_blocks.erase(h);
+  }
   blocks.erase(j);
 }
 
@@ -98,12 +105,10 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   block_map::iterator i = blocks.begin();
-  if (i != blocks.end() && is_blockchain_placeholder(*i))
-    ++i;
   while (i != blocks.end())
   {
     block_map::iterator j = i++;
-    if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0)
+    if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end())
     {
       erase_block(j);
     }
@@ -152,23 +157,56 @@ uint64_t block_queue::get_max_block_height() const
   return height;
 }
 
+uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  if (blocks.empty())
+    return blockchain_height;
+  uint64_t last_needed_height = blockchain_height;
+  bool first = true;
+  for (const auto &span: blocks)
+  {
+    if (span.start_block_height + span.nblocks - 1 < blockchain_height)
+      continue;
+    if (span.start_block_height != last_needed_height || (first && span.blocks.empty()))
+      return last_needed_height;
+    last_needed_height = span.start_block_height + span.nblocks;
+    first = false;
+  }
+  return last_needed_height;
+}
+
 void block_queue::print() const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   MDEBUG("Block queue has " << blocks.size() << " spans");
   for (const auto &span: blocks)
-    MDEBUG("  " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled    ") << "  " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)");
+    MDEBUG("  " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled    ") << "  " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)");
 }
 
-std::string block_queue::get_overview() const
+std::string block_queue::get_overview(uint64_t blockchain_height) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   if (blocks.empty())
     return "[]";
   block_map::const_iterator i = blocks.begin();
-  std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":";
-  while (++i != blocks.end())
-    s += i->blocks.empty() ? "." : "o";
+  std::string s = std::string("[");
+  uint64_t expected = blockchain_height;
+  while (i != blocks.end())
+  {
+    if (expected > i->start_block_height)
+    {
+      s += "<";
+    }
+    else
+    {
+      if (expected < i->start_block_height)
+        s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_');
+      s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o";
+      expected = i->start_block_height + i->nblocks;
+    }
+    ++i;
+  }
   s += "]";
   return s;
 }
@@ -184,16 +222,31 @@ bool block_queue::requested(const crypto::hash &hash) const
   return requested_internal(hash);
 }
 
-std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time)
+bool block_queue::have(const crypto::hash &hash) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  return have_blocks.find(hash) != have_blocks.end();
+}
+
+std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
 
+  MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height
+      << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " <<
+      blockchain_height << ", block hashes size " << block_hashes.size());
   if (last_block_height < first_block_height || max_blocks == 0)
   {
     MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks);
     return std::make_pair(0, 0);
   }
+  if (block_hashes.size() >= last_block_height)
+  {
+    MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height);
+    return std::make_pair(0, 0);
+  }
 
+  // skip everything we've already requested
   uint64_t span_start_height = last_block_height - block_hashes.size() + 1;
   std::vector<crypto::hash>::const_iterator i = block_hashes.begin();
   while (i != block_hashes.end() && requested_internal(*i))
@@ -201,55 +254,57 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
     ++i;
     ++span_start_height;
   }
+
+  // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there
+  const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed);
+  MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed "
+      << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE);
+  if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE)
+  {
+    MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height <<
+        "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed);
+    span_start_height = next_unpruned_height;
+  }
+  MDEBUG("span_start_height: " <<span_start_height);
+  const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1;
+  if (span_start_height >= block_hashes.size() + block_hashes_start_height)
+  {
+    MDEBUG("Out of hashes, cannot reserve");
+    return std::make_pair(0, 0);
+  }
+
+  i = block_hashes.begin() + span_start_height - block_hashes_start_height;
+  while (i != block_hashes.end() && requested_internal(*i))
+  {
+    ++i;
+    ++span_start_height;
+  }
+
   uint64_t span_length = 0;
   std::vector<crypto::hash> hashes;
-  while (i != block_hashes.end() && span_length < max_blocks)
+  while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed))
   {
     hashes.push_back(*i);
     ++i;
     ++span_length;
   }
   if (span_length == 0)
+  {
+    MDEBUG("span_length 0, cannot reserve");
     return std::make_pair(0, 0);
+  }
   MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id);
   add_blocks(span_start_height, span_length, connection_id, time);
   set_span_hashes(span_start_height, connection_id, hashes);
   return std::make_pair(span_start_height, span_length);
 }
 
-bool block_queue::is_blockchain_placeholder(const span &span) const
-{
-  return span.connection_id == boost::uuids::nil_uuid();
-}
-
-std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const
-{
-  boost::unique_lock<boost::recursive_mutex> lock(mutex);
-  if (blocks.empty())
-    return std::make_pair(0, 0);
-  block_map::const_iterator i = blocks.begin();
-  if (!is_blockchain_placeholder(*i))
-    return std::make_pair(0, 0);
-  uint64_t current_height = i->start_block_height + i->nblocks - 1;
-  ++i;
-  if (i == blocks.end())
-    return std::make_pair(0, 0);
-  uint64_t first_span_height = i->start_block_height;
-  if (first_span_height <= current_height + 1)
-    return std::make_pair(0, 0);
-  MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height);
-  print();
-  return std::make_pair(current_height + 1, first_span_height - current_height - 1);
-}
-
 std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   if (blocks.empty())
     return std::make_pair(0, 0);
   block_map::const_iterator i = blocks.begin();
-  if (is_blockchain_placeholder(*i))
-    ++i;
   if (i == blocks.end())
     return std::make_pair(0, 0);
   if (!i->blocks.empty())
@@ -260,6 +315,16 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vecto
   return std::make_pair(i->start_block_height, i->nblocks);
 }
 
+void block_queue::reset_next_span_time(boost::posix_time::ptime t)
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time");
+  block_map::iterator i = blocks.begin();
+  CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time");
+  CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty");
+  (boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting
+}
+
 void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
@@ -284,8 +349,6 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_
   if (blocks.empty())
     return false;
   block_map::const_iterator i = blocks.begin();
-  if (is_blockchain_placeholder(*i))
-    ++i;
   for (; i != blocks.end(); ++i)
   {
     if (!filled || !i->blocks.empty())
@@ -299,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_
   return false;
 }
 
-bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const
+bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   if (blocks.empty())
     return false;
   block_map::const_iterator i = blocks.begin();
-  if (is_blockchain_placeholder(*i))
-    ++i;
   if (i == blocks.end())
     return false;
   if (i->connection_id != connection_id)
     return false;
   filled = !i->blocks.empty();
+  time = i->time;
+  return true;
+}
+
+bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  if (blocks.empty())
+    return false;
+  block_map::const_iterator i = blocks.begin();
+  if (i == blocks.end())
+    return false;
+  if (i->start_block_height > height)
+    return false;
+  filled = !i->blocks.empty();
+  time = i->time;
+  connection_id = i->connection_id;
   return true;
 }
 
@@ -331,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const
   if (blocks.empty())
     return 0;
   block_map::const_iterator i = blocks.begin();
-  if (is_blockchain_placeholder(*i))
-    ++i;
   size_t size = 0;
   while (i != blocks.end() && !i->blocks.empty())
   {
@@ -417,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const
   return speed;
 }
 
-bool block_queue::foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder) const
+float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  float conn_rate = -1.f;
+  for (const auto &span: blocks)
+  {
+    if (span.blocks.empty())
+      continue;
+    if (span.connection_id != connection_id)
+      continue;
+    // note that the average below does not average over the whole set, but over the
+    // previous pseudo average and the latest rate: this gives much more importance
+    // to the latest measurements, which is fine here
+    if (conn_rate < 0.f)
+      conn_rate = span.rate;
+    else
+      conn_rate = (conn_rate + span.rate) / 2;
+  }
+
+  if (conn_rate < 0)
+    conn_rate = 0.0f;
+  MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s");
+  return conn_rate;
+}
+
+bool block_queue::foreach(std::function<bool(const span&)> f) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   block_map::const_iterator i = blocks.begin();
-  if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i))
-    ++i;
   while (i != blocks.end())
     if (!f(*i++))
       return false;

src/cryptonote_protocol/block_queue.h

@@ -76,22 +76,26 @@ namespace cryptonote
     void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height);
     uint64_t get_max_block_height() const;
     void print() const;
-    std::string get_overview() const;
-    std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
-    bool is_blockchain_placeholder(const span &span) const;
-    std::pair<uint64_t, uint64_t> get_start_gap_span() const;
+    std::string get_overview(uint64_t blockchain_height) const;
+    bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const;
+    std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
+    uint64_t get_next_needed_height(uint64_t blockchain_height) const;
     std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const;
+    void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time());
     void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes);
     bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const;
-    bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const;
+    bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const;
+    bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const;
     size_t get_data_size() const;
     size_t get_num_filled_spans_prefix() const;
     size_t get_num_filled_spans() const;
     crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const;
     bool has_spans(const boost::uuids::uuid &connection_id) const;
     float get_speed(const boost::uuids::uuid &connection_id) const;
-    bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const;
+    float get_download_rate(const boost::uuids::uuid &connection_id) const;
+    bool foreach(std::function<bool(const span&)> f) const;
     bool requested(const crypto::hash &hash) const;
+    bool have(const crypto::hash &hash) const;
 
   private:
     void erase_block(block_map::iterator j);
@@ -101,5 +105,6 @@ namespace cryptonote
     block_map blocks;
     mutable boost::recursive_mutex mutex;
     std::unordered_set<crypto::hash> requested_hashes;
+    std::unordered_set<crypto::hash> have_blocks;
   };
 }

src/cryptonote_protocol/cryptonote_protocol_defs.h

@@ -78,6 +78,8 @@ namespace cryptonote
 
     uint64_t height;
 
+    uint32_t pruning_seed;
+
     BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE(incoming)
       KV_SERIALIZE(localhost)
@@ -100,6 +102,7 @@ namespace cryptonote
       KV_SERIALIZE(support_flags)
       KV_SERIALIZE(connection_id)
       KV_SERIALIZE(height)
+      KV_SERIALIZE(pruning_seed)
     END_KV_SERIALIZE_MAP()
   };
 
@@ -200,12 +203,14 @@ namespace cryptonote
     uint64_t cumulative_difficulty;
     crypto::hash  top_id;
     uint8_t top_version;
+    uint32_t pruning_seed;
 
     BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE(current_height)
       KV_SERIALIZE(cumulative_difficulty)
       KV_SERIALIZE_VAL_POD_AS_BLOB(top_id)
       KV_SERIALIZE_OPT(top_version, (uint8_t)0)
+      KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0)
     END_KV_SERIALIZE_MAP()
   };
 

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -43,6 +43,7 @@
 #include "cryptonote_protocol_defs.h"
 #include "cryptonote_protocol_handler_common.h"
 #include "block_queue.h"
+#include "common/perf_timer.h"
 #include "cryptonote_basic/connection_context.h"
 #include "cryptonote_basic/cryptonote_stat_info.h"
 #include <boost/circular_buffer.hpp>
@@ -109,6 +110,10 @@ namespace cryptonote
     const block_queue &get_block_queue() const { return m_block_queue; }
     void stop();
     void on_connection_close(cryptonote_connection_context &context);
+    void set_max_out_peers(unsigned int max) { m_max_out_peers = max; }
+    std::string get_peers_overview() const;
+    std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const;
+    bool needs_new_sync_connections() const;
   private:
     //----------------- commands handlers ----------------------------------------------
     int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context);
@@ -125,14 +130,17 @@ namespace cryptonote
     virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context);
     //----------------------------------------------------------------------------------
     //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context);
+    bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe);
     bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false);
     size_t get_synchronizing_connections_count();
     bool on_connection_synchronized();
-    bool should_download_next_span(cryptonote_connection_context& context) const;
+    bool should_download_next_span(cryptonote_connection_context& context, bool standby);
     void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
     bool kick_idle_peers();
     bool check_standby_peers();
     int try_add_next_blocks(cryptonote_connection_context &context);
+    void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe);
+    void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const;
 
     t_core& m_core;
 
@@ -145,6 +153,12 @@ namespace cryptonote
     block_queue m_block_queue;
     epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker;
     epee::math_helper::once_a_time_milliseconds<100> m_standby_checker;
+    std::atomic<unsigned int> m_max_out_peers;
+    tools::PerformanceTimer m_sync_timer, m_add_timer;
+    uint64_t m_last_add_end_time;
+    uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded;
+    uint64_t m_sync_download_chain_size, m_sync_download_objects_size;
+    size_t m_block_download_max_size;
 
     boost::mutex m_buffer_mutex;
     double get_avg_block_size();