Merge upstream into blockchain

external/CMakeLists.txt

@@ -35,7 +35,7 @@
 # ...except for FreeBSD, because FreeBSD is a special case that doesn't play well with
 # others.
 
-find_package(MiniUpnpc QUIET)
+find_package(Miniupnpc QUIET)
 
 # FreeBSD doesn't play well with the local copy, so default to using shared
 set(USE_SHARED_MINIUPNPC false)

src/common/dns_utils.cpp

@@ -34,7 +34,65 @@
 
 #include <stdlib.h>
 #include "include_base_utils.h"
+#include <boost/filesystem/fstream.hpp>
 using namespace epee;
+namespace bf = boost::filesystem;
+
+namespace
+{
+
+/*
+ * The following two functions were taken from unbound-anchor.c, from
+ * the unbound library packaged with this source.  The license and source
+ * can be found in $PROJECT_ROOT/external/unbound
+ */
+
+/* Cert builtin commented out until it's used, as the compiler complains
+
+// return the built in root update certificate
+static const char*
+get_builtin_cert(void)
+{
+	return
+// The ICANN CA fetched at 24 Sep 2010.  Valid to 2028
+"-----BEGIN CERTIFICATE-----\n"
+"MIIDdzCCAl+gAwIBAgIBATANBgkqhkiG9w0BAQsFADBdMQ4wDAYDVQQKEwVJQ0FO\n"
+"TjEmMCQGA1UECxMdSUNBTk4gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxFjAUBgNV\n"
+"BAMTDUlDQU5OIFJvb3QgQ0ExCzAJBgNVBAYTAlVTMB4XDTA5MTIyMzA0MTkxMloX\n"
+"DTI5MTIxODA0MTkxMlowXTEOMAwGA1UEChMFSUNBTk4xJjAkBgNVBAsTHUlDQU5O\n"
+"IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRYwFAYDVQQDEw1JQ0FOTiBSb290IENB\n"
+"MQswCQYDVQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKDb\n"
+"cLhPNNqc1NB+u+oVvOnJESofYS9qub0/PXagmgr37pNublVThIzyLPGCJ8gPms9S\n"
+"G1TaKNIsMI7d+5IgMy3WyPEOECGIcfqEIktdR1YWfJufXcMReZwU4v/AdKzdOdfg\n"
+"ONiwc6r70duEr1IiqPbVm5T05l1e6D+HkAvHGnf1LtOPGs4CHQdpIUcy2kauAEy2\n"
+"paKcOcHASvbTHK7TbbvHGPB+7faAztABLoneErruEcumetcNfPMIjXKdv1V1E3C7\n"
+"MSJKy+jAqqQJqjZoQGB0necZgUMiUv7JK1IPQRM2CXJllcyJrm9WFxY0c1KjBO29\n"
+"iIKK69fcglKcBuFShUECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8B\n"
+"Af8EBAMCAf4wHQYDVR0OBBYEFLpS6UmDJIZSL8eZzfyNa2kITcBQMA0GCSqGSIb3\n"
+"DQEBCwUAA4IBAQAP8emCogqHny2UYFqywEuhLys7R9UKmYY4suzGO4nkbgfPFMfH\n"
+"6M+Zj6owwxlwueZt1j/IaCayoKU3QsrYYoDRolpILh+FPwx7wseUEV8ZKpWsoDoD\n"
+"2JFbLg2cfB8u/OlE4RYmcxxFSmXBg0yQ8/IoQt/bxOcEEhhiQ168H2yE5rxJMt9h\n"
+"15nu5JBSewrCkYqYYmaxyOC3WrVGfHZxVI7MpIFcGdvSb2a1uyuua8l0BKgk3ujF\n"
+"0/wsHNeP22qNyVO+XVBzrM8fk8BSUFuiT/6tZTYXRtEt5aKQZgXbKU5dUF3jT9qg\n"
+"j/Br5BZw3X/zd325TvnswzMC1+ljLzHnQGGk\n"
+"-----END CERTIFICATE-----\n"
+		;
+}
+*/
+
+/** return the built in root DS trust anchor */
+static const char*
+get_builtin_ds(void)
+{
+	return
+". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n";
+}
+
+/************************************************************
+ ************************************************************
+ ***********************************************************/
+
+} // anonymous namespace
 
 namespace tools
 {
@@ -109,6 +167,8 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData())
   // look for "/etc/resolv.conf" and "/etc/hosts" or platform equivalent
   ub_ctx_resolvconf(m_data->m_ub_context, &empty_string);
   ub_ctx_hosts(m_data->m_ub_context, &empty_string);
+
+  ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds());
 }
 
 DNSResolver::~DNSResolver()
@@ -143,6 +203,8 @@ std::vector<std::string> DNSResolver::get_ipv4(const std::string& url, bool& dns
   // call DNS resolver, blocking.  if return value not zero, something went wrong
   if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_A, DNS_CLASS_IN, &(result.ptr)))
   {
+    dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus));
+    dnssec_valid = !result.ptr->bogus;
     if (result.ptr->havedata)
     {
       for (size_t i=0; result.ptr->data[i] != NULL; i++)
@@ -175,6 +237,8 @@ std::vector<std::string> DNSResolver::get_ipv6(const std::string& url, bool& dns
   // call DNS resolver, blocking.  if return value not zero, something went wrong
   if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_AAAA, DNS_CLASS_IN, &(result.ptr)))
   {
+    dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus));
+    dnssec_valid = !result.ptr->bogus;
     if (result.ptr->havedata)
     {
       for (size_t i=0; result.ptr->data[i] != NULL; i++)
@@ -207,6 +271,8 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo
   // call DNS resolver, blocking.  if return value not zero, something went wrong
   if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_TXT, DNS_CLASS_IN, &(result.ptr)))
   {
+    dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus));
+    dnssec_valid = !result.ptr->bogus;
     if (result.ptr->havedata)
     {
       for (size_t i=0; result.ptr->data[i] != NULL; i++)

src/cryptonote_core/blockchain_storage.cpp

@@ -1329,7 +1329,7 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx
   return true;
 }
 //------------------------------------------------------------------
-bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height)
+bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size)
 {
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
   struct add_transaction_input_visitor: public boost::static_visitor<bool>
@@ -1368,6 +1368,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const
   }
   transaction_chain_entry ch_e;
   ch_e.m_keeper_block_height = bl_height;
+  ch_e.m_blob_size = blob_size;
   ch_e.tx = tx;
   auto i_r = m_transactions.insert(std::pair<crypto::hash, transaction_chain_entry>(tx_id, ch_e));
   if(!i_r.second)
@@ -1633,10 +1634,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt
     bvc.m_verifivation_failed = true;
     return false;
   }
-  size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx);
+  crypto::hash coinbase_hash = null_hash;
+  size_t coinbase_blob_size = 0;
+  get_transaction_hash(bl.miner_tx, coinbase_hash, coinbase_blob_size);
   size_t cumulative_block_size = coinbase_blob_size;
   //process transactions
-  if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height()))
+  if(!add_transaction_from_block(bl.miner_tx, coinbase_hash, id, get_current_blockchain_height(), coinbase_blob_size))
   {
     LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage");
     bvc.m_verifivation_failed = true;
@@ -1670,7 +1673,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt
       return false;
     }
 
-    if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height()))
+    if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height(), blob_size))
     {
        LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage");
        cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);

src/cryptonote_core/blockchain_storage.h

@@ -239,7 +239,7 @@ namespace cryptonote
     bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const;
     bool validate_transaction(const block& b, uint64_t height, const transaction& tx) const;
     bool rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height);
-    bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height);
+    bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size);
     bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes);
     bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id);
     bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const;

src/cryptonote_core/checkpoints_create.cpp

@@ -138,21 +138,34 @@ bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testne
   size_t cur_index = first_index;
   do
   {
+    std::string url;
     if (testnet)
     {
-      records = tools::DNSResolver::instance().get_txt_record(testnet_dns_urls[cur_index], avail, valid);
+      url = testnet_dns_urls[cur_index];
     }
     else
     {
-      records = tools::DNSResolver::instance().get_txt_record(dns_urls[cur_index], avail, valid);
+      url = dns_urls[cur_index];
     }
-    if (records.size() == 0 || (avail && !valid))
+
+    records = tools::DNSResolver::instance().get_txt_record(url, avail, valid);
+    if (!avail)
+    {
+      LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping.");
+    }
+    if (!valid)
+    {
+      LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping.");
+    }
+
+    if (records.size() == 0 || !avail || !valid)
     {
       cur_index++;
       if (cur_index == dns_urls.size())
       {
 	cur_index = 0;
       }
+      records.clear();
       continue;
     }
     break;
@@ -160,13 +173,7 @@ bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testne
 
   if (records.size() == 0)
   {
-    LOG_PRINT_L1("Fetching MoneroPulse checkpoints failed, no TXT records available.");
+    LOG_PRINT_L0("WARNING: All MoneroPulse checkpoint URLs failed DNSSEC validation and/or returned no records");
-    return true;
-  }
-
-  if (avail && !valid)
-  {
-    LOG_PRINT_L0("WARNING: MoneroPulse failed DNSSEC validation and/or returned no records");
     return true;
   }
 

src/simplewallet/simplewallet.cpp

@@ -67,6 +67,7 @@ namespace po = boost::program_options;
 
 #define EXTENDED_LOGS_FILE "wallet_details.log"
 
+#define DEFAULT_MIX 3
 
 namespace
 {
@@ -280,7 +281,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), "incoming_transfers [available|unavailable] - Show incoming transfers - all of them or filter them by availability");
   m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), "payments <payment_id_1> [<payment_id_2> ... <payment_id_N>] - Show payments <payment_id_1>, ... <payment_id_N>");
   m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), "Show blockchain height");
-  m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer <mixin_count> <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)");
+  m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)");
   m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4");
   m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), "Show current wallet public address");
   m_cmd_binder.set_handler("save", boost::bind(&simple_wallet::save, this, _1), "Save wallet synchronized data");
@@ -1067,19 +1068,24 @@ bool simple_wallet::transfer(const std::vector<std::string> &args_)
     return true;
 
   std::vector<std::string> local_args = args_;
-  if(local_args.size() < 3)
-  {
-    fail_msg_writer() << "wrong number of arguments, expected at least 3, got " << local_args.size();
-    return true;
-  }
 
   size_t fake_outs_count;
-  if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0]))
+  if(local_args.size() > 0) {
-  {
+    if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0]))
-    fail_msg_writer() << "mixin_count should be non-negative integer, got " << local_args[0];
+    {
-    return true;
+      fake_outs_count = DEFAULT_MIX;
+    }
+    else
+    {
+      local_args.erase(local_args.begin());
+    }
+  }
+
+  if(local_args.size() < 2)
+  {
+     fail_msg_writer() << "wrong number of arguments";
+     return true;
   }
-  local_args.erase(local_args.begin());
 
   std::vector<uint8_t> extra;
   if (1 == local_args.size() % 2)

tests/unit_tests/dns_resolver.cpp

@@ -43,13 +43,13 @@ TEST(DNSResolver, IPv4Success)
 
   ASSERT_EQ(1, ips.size());
 
-  ASSERT_STREQ("93.184.216.119", ips[0].c_str());
+  //ASSERT_STREQ("93.184.216.119", ips[0].c_str());
 
   ips = tools::DNSResolver::instance().get_ipv4("example.com", avail, valid);
 
   ASSERT_EQ(1, ips.size());
 
-  ASSERT_STREQ("93.184.216.119", ips[0].c_str());
+  //ASSERT_STREQ("93.184.216.119", ips[0].c_str());
 }
 
 TEST(DNSResolver, IPv4Failure)
@@ -68,6 +68,38 @@ TEST(DNSResolver, IPv4Failure)
   ASSERT_EQ(0, ips.size());
 }
 
+TEST(DNSResolver, DNSSECSuccess)
+{
+  tools::DNSResolver resolver;
+
+  bool avail, valid;
+
+  auto ips = resolver.get_ipv4("example.com", avail, valid);
+
+  ASSERT_EQ(1, ips.size());
+
+  //ASSERT_STREQ("93.184.216.119", ips[0].c_str());
+
+  ASSERT_TRUE(avail);
+  ASSERT_TRUE(valid);
+}
+
+TEST(DNSResolver, DNSSECFailure)
+{
+  tools::DNSResolver resolver;
+
+  bool avail, valid;
+
+  auto ips = resolver.get_ipv4("dnssec-failed.org", avail, valid);
+
+  ASSERT_EQ(1, ips.size());
+
+  //ASSERT_STREQ("93.184.216.119", ips[0].c_str());
+
+  ASSERT_TRUE(avail);
+  ASSERT_FALSE(valid);
+}
+
 // It would be great to include an IPv6 test and assume it'll pass, but not every ISP / resolver plays nicely with IPv6;)
 /*TEST(DNSResolver, IPv6Success)
 {