Merge remote-tracking branch 'upstream/master'

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -490,7 +490,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 			sleep_before_packet(cb, 1, 1);
 		}
 
-    epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical ***
+    m_send_que_lock.lock(); // *** critical ***
+    epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_send_que_lock.unlock();});
+
     long int retry=0;
     const long int retry_limit = 5*4;
     while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
@@ -504,14 +506,15 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 
         long int ms = 250 + (rand()%50);
         _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
+        m_send_que_lock.unlock();
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
+        m_send_que_lock.lock();
         _dbg1("sleep for queue: " << ms);
 
         if (retry > retry_limit) {
-            send_guard.unlock();
             _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
             //	_dbg1_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
-            close();
+            shutdown();
             return false;
         }
     }

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -135,7 +135,7 @@ const unsigned int DB_BUFFER_LENGTH = 32 * MB;
 const unsigned int DB_DEF_CACHESIZE = 256 * MB;
 
 #if defined(BDB_BULK_CAN_THREAD)
-const unsigned int DB_BUFFER_COUNT = boost::thread::hardware_concurrency();
+const unsigned int DB_BUFFER_COUNT = tools::get_max_concurrency();
 #else
 const unsigned int DB_BUFFER_COUNT = 1;
 #endif

src/common/util.cpp

@@ -422,4 +422,27 @@ std::string get_nix_version_display_string()
     umask(mode);
 #endif
   }
+
+  namespace
+  {
+    boost::mutex max_concurrency_lock;
+    unsigned max_concurrency = boost::thread::hardware_concurrency();
+  }
+
+  void set_max_concurrency(unsigned n)
+  {
+    if (n < 1)
+      n = boost::thread::hardware_concurrency();
+    unsigned hwc = boost::thread::hardware_concurrency();
+    if (n > hwc)
+      n = hwc;
+    boost::lock_guard<boost::mutex> lock(max_concurrency_lock);
+    max_concurrency = n;
+  }
+
+  unsigned get_max_concurrency()
+  {
+    boost::lock_guard<boost::mutex> lock(max_concurrency_lock);
+    return max_concurrency;
+  }
 }

src/common/util.h

@@ -160,4 +160,7 @@ namespace tools
   };
 
   void set_strict_default_file_permissions(bool strict);
+
+  void set_max_concurrency(unsigned n);
+  unsigned get_max_concurrency();
 }

src/crypto/random.c

@@ -45,7 +45,7 @@ static void generate_system_random_bytes(size_t n, void *result);
 
 static void generate_system_random_bytes(size_t n, void *result) {
   HCRYPTPROV prov;
-#define must_succeed(x) do if (!(x)) assert(0); while (0)
+#define must_succeed(x) do if (!(x)) abort(); while (0)
   must_succeed(CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT));
   must_succeed(CryptGenRandom(prov, (DWORD)n, result));
   must_succeed(CryptReleaseContext(prov, 0));

src/cryptonote_core/blockchain.cpp

@@ -498,6 +498,7 @@ block Blockchain::pop_block_from_blockchain()
       }
     }
   }
+  update_next_cumulative_size_limit();
   m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id());
 
   return popped_block;
@@ -2123,7 +2124,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, tx_verification_context
   std::vector < uint64_t > results;
   results.resize(tx.vin.size(), 0);
 
-  int threads = boost::thread::hardware_concurrency();
+  int threads = tools::get_max_concurrency();
 
   boost::asio::io_service ioservice;
   boost::thread_group threadpool;
@@ -2712,7 +2713,11 @@ leave:
   // populate various metadata about the block to be stored alongside it.
   block_size = cumulative_block_size;
   cumulative_difficulty = current_diffic;
-  already_generated_coins = already_generated_coins + base_reward;
+  // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of
+  // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins
+  // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a
+  // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state.
+  already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY;
   if(m_db->height())
     cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1);
 
@@ -2996,7 +3001,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e
     return true;
 
   bool blocks_exist = false;
-  uint64_t threads = boost::thread::hardware_concurrency();
+  uint64_t threads = tools::get_max_concurrency();
 
   if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1)
   {
@@ -3195,7 +3200,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e
   // [output] stores all transactions for each tx_out_index::hash found
   std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size());
 
-  threads = boost::thread::hardware_concurrency();
+  threads = tools::get_max_concurrency();
   if (!m_db->can_thread_bulk_indices())
     threads = 1;
 

src/daemon/command_line_args.h

@@ -59,6 +59,11 @@ namespace daemon_args
     "os-version"
   , "OS for which this executable was compiled"
   };
+  const command_line::arg_descriptor<unsigned> arg_max_concurrency = {
+    "max-concurrency"
+  , "Max number of threads to use for a parallel job"
+  , 0
+  };
 }  // namespace daemon_args
 
 #endif // DAEMON_COMMAND_LINE_ARGS_H

src/daemon/main.cpp

@@ -81,6 +81,7 @@ int main(int argc, char const * argv[])
       bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log");
       command_line::add_arg(core_settings, daemon_args::arg_log_file, default_log.string());
       command_line::add_arg(core_settings, daemon_args::arg_log_level);
+      command_line::add_arg(core_settings, daemon_args::arg_max_concurrency);
 
       daemonizer::init_options(hidden_options, visible_options);
       daemonize::t_executor::init_options(core_settings);
@@ -260,6 +261,9 @@ int main(int argc, char const * argv[])
         );
     }
 
+    if (command_line::has_arg(vm, daemon_args::arg_max_concurrency))
+      tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency));
+
     _note_c("dbg/main", "Moving from main() into the daemonize now.");
 
     return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm);

src/p2p/net_node.inl

@@ -1319,6 +1319,7 @@ namespace nodetool
         if(rsp.status != PING_OK_RESPONSE_STATUS_TEXT || pr != rsp.peer_id)
         {
           LOG_PRINT_CC_L2(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << ip << ":" << port << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << rsp.peer_id);
+          m_net_server.get_config_object().close(ping_context.m_connection_id);
           return;
         }
         m_net_server.get_config_object().close(ping_context.m_connection_id);

src/rpc/core_rpc_server.cpp

@@ -166,6 +166,25 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res)
+  {
+    CHECK_CORE_BUSY();
+    NOTIFY_RESPONSE_CHAIN_ENTRY::request resp;
+
+    resp.start_height = req.start_height;
+    if(!m_core.find_blockchain_supplement(req.block_ids, resp))
+    {
+      res.status = "Failed";
+      return false;
+    }
+    res.current_height = resp.total_height;
+    res.start_height = resp.start_height;
+    res.m_block_ids = std::move(resp.m_block_ids);
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res)
   {
     CHECK_CORE_BUSY();

src/rpc/core_rpc_server.h

@@ -75,6 +75,7 @@ namespace cryptonote
     BEGIN_URI_MAP2()
       MAP_URI_AUTO_JON2("/getheight", on_get_height, COMMAND_RPC_GET_HEIGHT)
       MAP_URI_AUTO_BIN2("/getblocks.bin", on_get_blocks, COMMAND_RPC_GET_BLOCKS_FAST)
+      MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST)
       MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES)      
       MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS)      
       MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS)
@@ -115,6 +116,7 @@ namespace cryptonote
 
     bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res);
     bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res);
+    bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res);
     bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res);
     bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res);
     bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res);

src/rpc/core_rpc_server_commands_defs.h

@@ -89,6 +89,36 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
   };
+
+  struct COMMAND_RPC_GET_HASHES_FAST
+  {
+
+    struct request
+    {
+      std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
+      uint64_t    start_height;
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
+        KV_SERIALIZE(start_height)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::list<crypto::hash> m_block_ids;
+      uint64_t    start_height;
+      uint64_t    current_height;
+      std::string status;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids)
+        KV_SERIALIZE(start_height)
+        KV_SERIALIZE(current_height)
+        KV_SERIALIZE(status)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
   //-----------------------------------------------
   struct COMMAND_RPC_GET_TRANSACTIONS
   {

src/simplewallet/simplewallet.cpp

@@ -77,6 +77,23 @@ typedef cryptonote::simple_wallet sw;
 
 #define DEFAULT_MIX 4
 
+#define LOCK_REFRESH_THREAD_SCOPE() \
+  bool auto_refresh_run = m_auto_refresh_run.load(std::memory_order_relaxed); \
+  m_auto_refresh_run.store(false, std::memory_order_relaxed); \
+  /* stop any background refresh, and take over */ \
+  m_wallet->stop(); \
+  m_auto_refresh_mutex.lock(); \
+  m_auto_refresh_cond.notify_one(); \
+  m_auto_refresh_mutex.unlock(); \
+  if (auto_refresh_run) \
+    m_auto_refresh_thread.join(); \
+  boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); \
+  epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \
+    m_auto_refresh_run.store(auto_refresh_run, std::memory_order_relaxed); \
+    if (auto_refresh_run) \
+      m_auto_refresh_thread = boost::thread([&]{wallet_refresh_thread();}); \
+  })
+
 namespace
 {
   const command_line::arg_descriptor<std::string> arg_wallet_file = {"wallet-file", sw::tr("Use wallet <arg>"), ""};
@@ -93,10 +110,12 @@ namespace
   const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Create non-deterministic view and spend keys"), false};
   const command_line::arg_descriptor<int> arg_daemon_port = {"daemon-port", sw::tr("Use daemon instance at port <arg> instead of 18081"), 0};
   const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0};
+  const command_line::arg_descriptor<uint32_t> arg_max_concurrency = {"max-concurrency", "Max number of threads to use for a parallel job", 0};
   const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", sw::tr("Specify log file"), ""};
   const command_line::arg_descriptor<bool> arg_testnet = {"testnet", sw::tr("For testnet. Daemon must also be launched with --testnet flag"), false};
   const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", sw::tr("Restricts RPC to view-only commands"), false};
   const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false};
+  const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0};
 
   const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""};
 
@@ -213,6 +232,44 @@ namespace
       return true;
     return false;
   }
+
+  const struct
+  {
+    const char *name;
+    tools::wallet2::RefreshType refresh_type;
+  } refresh_type_names[] =
+  {
+    { "full", tools::wallet2::RefreshFull },
+    { "optimize-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
+    { "optimized-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
+    { "no-coinbase", tools::wallet2::RefreshNoCoinbase },
+    { "default", tools::wallet2::RefreshDefault },
+  };
+
+  bool parse_refresh_type(const std::string &s, tools::wallet2::RefreshType &refresh_type)
+  {
+    for (size_t n = 0; n < sizeof(refresh_type_names) / sizeof(refresh_type_names[0]); ++n)
+    {
+      if (s == refresh_type_names[n].name)
+      {
+        refresh_type = refresh_type_names[n].refresh_type;
+        return true;
+      }
+    }
+    fail_msg_writer() << tr("failed to parse refresh type");
+    return false;
+  }
+
+  std::string get_refresh_type_name(tools::wallet2::RefreshType type)
+  {
+    for (size_t n = 0; n < sizeof(refresh_type_names) / sizeof(refresh_type_names[0]); ++n)
+    {
+      if (type == refresh_type_names[n].refresh_type)
+        return refresh_type_names[n].name;
+    }
+    return "invalid";
+  }
+
 }
 
 
@@ -465,32 +522,6 @@ bool simple_wallet::set_auto_refresh(const std::vector<std::string> &args/* = st
   return true;
 }
 
-static bool parse_refresh_type(const std::string &s, tools::wallet2::RefreshType &refresh_type)
-{
-  static const struct
-  {
-    const char *name;
-    tools::wallet2::RefreshType refresh_type;
-  } names[] =
-  {
-    { "full", tools::wallet2::RefreshFull },
-    { "optimize-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
-    { "optimized-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
-    { "no-coinbase", tools::wallet2::RefreshNoCoinbase },
-    { "default", tools::wallet2::RefreshDefault },
-  };
-  for (size_t n = 0; n < sizeof(names) / sizeof(names[0]); ++n)
-  {
-    if (s == names[n].name)
-    {
-      refresh_type = names[n].refresh_type;
-      return true;
-    }
-  }
-  fail_msg_writer() << tr("failed to parse refresh type");
-  return false;
-}
-
 bool simple_wallet::set_refresh_type(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   bool success = false;
@@ -547,6 +578,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), tr("transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of extra inputs to include for untraceability (from 0 to maximum available)"));
   m_cmd_binder.set_handler("transfer_new", boost::bind(&simple_wallet::transfer_new, this, _1), tr("Same as transfer, but using a new transaction building algorithm"));
   m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with mixin 0"));
+  m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr("Send all unlocked balance an address"));
   m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), tr("set_log <level> - Change current log detail level, <0-4>"));
   m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), tr("Show current wallet public address"));
   m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), tr("integrated_address [PID] - Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID"));
@@ -561,6 +593,8 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>"));
   m_cmd_binder.set_handler("show_transfers", boost::bind(&simple_wallet::show_transfers, this, _1), tr("show_transfers [in|out] [<min_height> [<max_height>]] - Show incoming/outgoing transfers within an optional height range"));
   m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), tr("Rescan blockchain from scratch"));
+  m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr("Set an arbitrary string note for a txid"));
+  m_cmd_binder.set_handler("get_tx_note", boost::bind(&simple_wallet::get_tx_note, this, _1), tr("Get a string note for a txid"));
   m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("Show this help"));
 }
 //----------------------------------------------------------------------------------------------------
@@ -568,7 +602,12 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
 {
   if (args.empty())
   {
-    fail_msg_writer() << tr("set: needs an argument. available options: seed, always-confirm-transfers, default-mixin, auto-refresh, refresh-type");
+    success_msg_writer() << "seed = " << m_wallet->get_seed_language();
+    success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers();
+    success_msg_writer() << "store-tx-info = " << m_wallet->store_tx_info();
+    success_msg_writer() << "default-mixin = " << m_wallet->default_mixin();
+    success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh();
+    success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type());
     return true;
   }
   else
@@ -1136,6 +1175,22 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
         return false;
       }
     }
+    if (!m_restore_height)
+    {
+      std::string heightstr = command_line::input_line("Restore from specific blockchain height (optional, default 0): ");
+      if (std::cin.eof())
+        return false;
+      if (heightstr.size())
+      {
+        try {
+          m_restore_height = boost::lexical_cast<uint64_t>(heightstr);
+        }
+        catch (boost::bad_lexical_cast &) {
+          fail_msg_writer() << tr("bad m_restore_height parameter:") << " " << heightstr;
+          return false;
+        }
+      }
+    }
     if (!m_generate_from_view_key.empty())
     {
       // parse address
@@ -1306,6 +1361,7 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_
   m_restore_deterministic_wallet  = command_line::get_arg(vm, arg_restore_deterministic_wallet);
   m_non_deterministic             = command_line::get_arg(vm, arg_non_deterministic);
   m_trusted_daemon                = command_line::get_arg(vm, arg_trusted_daemon);
+  m_restore_height                = command_line::get_arg(vm, arg_restore_height);
 
   return true;
 }
@@ -1410,6 +1466,15 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
+  // for a totally new account, we don't care about older blocks.
+  if (!m_restore_deterministic_wallet)
+  {
+    std::string err;
+    m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err));
+  } else if (m_restore_height)
+  {
+    m_wallet->set_refresh_from_block_height(m_restore_height);
+  }
 
   // convert rng value to electrum-style word list
   std::string electrum_words;
@@ -1457,6 +1522,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
+  m_wallet->set_refresh_from_block_height(m_restore_height);
 
   return true;
 }
@@ -1482,6 +1548,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   }
 
   m_wallet->init(m_daemon_address);
+  m_wallet->set_refresh_from_block_height(m_restore_height);
 
   return true;
 }
@@ -1584,6 +1651,7 @@ bool simple_wallet::save(const std::vector<std::string> &args)
 {
   try
   {
+    LOCK_REFRESH_THREAD_SCOPE();
     m_wallet->store();
     success_msg_writer() << tr("Wallet data saved");
   }
@@ -1639,7 +1707,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args)
   req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->testnet());
 
   bool ok = true;
-  size_t max_mining_threads_count = (std::max)(boost::thread::hardware_concurrency(), static_cast<unsigned>(2));
+  size_t max_mining_threads_count = (std::max)(tools::get_max_concurrency(), static_cast<unsigned>(2));
   if (0 == args.size())
   {
     req.threads_count = 1;
@@ -1752,12 +1820,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset)
   if (!try_connect_to_daemon())
     return true;
 
-  bool auto_refresh_run = m_auto_refresh_run.load(std::memory_order_relaxed);
-  m_auto_refresh_run.store(false, std::memory_order_relaxed);
-  // stop any background refresh, and take over
-  m_wallet->stop();
-  boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex);
-  m_auto_refresh_cond.notify_one();
+  LOCK_REFRESH_THREAD_SCOPE();
 
   if (reset)
     m_wallet->rescan_blockchain(false);
@@ -1770,13 +1833,13 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset)
   try
   {
     m_in_manual_refresh.store(true, std::memory_order_relaxed);
+    epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);});
     m_wallet->refresh(start_height, fetched_blocks);
-    m_in_manual_refresh.store(false, std::memory_order_relaxed);
     ok = true;
     // Clear line "Height xxx of xxx"
     std::cout << "\r                                                                \r";
     success_msg_writer(true) << tr("Refresh done, blocks received: ") << fetched_blocks;
-    show_balance();
+    show_balance_unlocked();
   }
   catch (const tools::error::daemon_busy&)
   {
@@ -1817,8 +1880,6 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset)
     fail_msg_writer() << tr("refresh failed: ") << ss.str() << ". " << tr("Blocks received: ") << fetched_blocks;
   }
 
-  m_in_manual_refresh.store(false, std::memory_order_relaxed);
-  m_auto_refresh_run.store(auto_refresh_run, std::memory_order_relaxed);
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -1838,15 +1899,24 @@ bool simple_wallet::refresh(const std::vector<std::string>& args)
   return refresh_main(start_height, false);
 }
 //----------------------------------------------------------------------------------------------------
-bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/)
+bool simple_wallet::show_balance_unlocked()
 {
   success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance()) << ", "
     << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance());
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/)
+{
+  LOCK_REFRESH_THREAD_SCOPE();
+  show_balance_unlocked();
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args)
 {
+  LOCK_REFRESH_THREAD_SCOPE();
+
   bool filter = false;
   bool available = false;
   if (!args.empty())
@@ -1912,6 +1982,8 @@ bool simple_wallet::show_payments(const std::vector<std::string> &args)
     return true;
   }
 
+  LOCK_REFRESH_THREAD_SCOPE();
+
   message_writer() << boost::format("%68s%68s%12s%21s%16s") %
     tr("payment") % tr("transaction") % tr("height") % tr("amount") % tr("unlock time");
 
@@ -1989,6 +2061,7 @@ bool simple_wallet::rescan_spent(const std::vector<std::string> &args)
 
   try
   {
+    LOCK_REFRESH_THREAD_SCOPE();
     m_wallet->rescan_spent();
   }
   catch (const tools::error::daemon_busy&)
@@ -2022,11 +2095,83 @@ bool simple_wallet::rescan_spent(const std::vector<std::string> &args)
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool simple_wallet::get_address_from_str(const std::string &str, cryptonote::account_public_address &address, bool &has_payment_id, crypto::hash8 &payment_id)
+{
+    if(!get_account_integrated_address_from_str(address, has_payment_id, payment_id, m_wallet->testnet(), str))
+    {
+      // if treating as an address fails, try as url
+      bool dnssec_ok = false;
+      std::string url = str;
+
+      // attempt to get address from dns query
+      auto addresses_from_dns = tools::wallet2::addresses_from_url(url, dnssec_ok);
+
+      // for now, move on only if one address found
+      if (addresses_from_dns.size() == 1)
+      {
+        if (get_account_integrated_address_from_str(address, has_payment_id, payment_id, m_wallet->testnet(), addresses_from_dns[0]))
+        {
+          // if it was an address, prompt user for confirmation.
+          // inform user of DNSSEC validation status as well.
+
+          std::string dnssec_str;
+          if (dnssec_ok)
+          {
+            dnssec_str = tr("DNSSEC validation passed");
+          }
+          else
+          {
+            dnssec_str = tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!");
+          }
+          std::stringstream prompt;
+          prompt << tr("For URL: ") << url
+                 << ", " << dnssec_str << std::endl
+                 << tr(" Monero Address = ") << addresses_from_dns[0]
+                 << std::endl
+                 << tr("Is this OK? (Y/n) ")
+          ;
+
+          // prompt the user for confirmation given the dns query and dnssec status
+          std::string confirm_dns_ok = command_line::input_line(prompt.str());
+          if (std::cin.eof())
+          {
+            return false;
+          }
+          if (confirm_dns_ok != "Y" && confirm_dns_ok != "y" && confirm_dns_ok != "Yes" && confirm_dns_ok != "yes"
+            && confirm_dns_ok != tr("yes") && confirm_dns_ok != tr("no"))
+          {
+            fail_msg_writer() << tr("you have cancelled the transfer request");
+            return false;
+          }
+        }
+        else
+        {
+          fail_msg_writer() << tr("failed to get a Monero address from: ") << url;
+          return false;
+        }
+      }
+      else if (addresses_from_dns.size() > 1)
+      {
+        fail_msg_writer() << tr("not yet supported: Multiple Monero addresses found for given URL: ") << url;
+        return false;
+      }
+      else
+      {
+        fail_msg_writer() << tr("wrong address: ") << url;
+        return false;
+      }
+    }
+
+    return true;
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::string> &args_)
 {
   if (!try_connect_to_daemon())
     return true;
 
+  LOCK_REFRESH_THREAD_SCOPE();
+
   std::vector<std::string> local_args = args_;
 
   size_t fake_outs_count;
@@ -2096,69 +2241,8 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
     cryptonote::tx_destination_entry de;
     bool has_payment_id;
     crypto::hash8 new_payment_id;
-    if(!get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet->testnet(), local_args[i]))
-    {
-      // if treating as an address fails, try as url
-      bool dnssec_ok = false;
-      std::string url = local_args[i];
-
-      // attempt to get address from dns query
-      auto addresses_from_dns = tools::wallet2::addresses_from_url(url, dnssec_ok);
-
-      // for now, move on only if one address found
-      if (addresses_from_dns.size() == 1)
-      {
-        if (get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet->testnet(), addresses_from_dns[0]))
-        {
-          // if it was an address, prompt user for confirmation.
-          // inform user of DNSSEC validation status as well.
-
-          std::string dnssec_str;
-          if (dnssec_ok)
-          {
-            dnssec_str = tr("DNSSEC validation passed");
-          }
-          else
-          {
-            dnssec_str = tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!");
-          }
-          std::stringstream prompt;
-          prompt << tr("For URL: ") << url
-                 << ", " << dnssec_str << std::endl
-                 << tr(" Monero Address = ") << addresses_from_dns[0]
-                 << std::endl
-                 << tr("Is this OK? (Y/n) ")
-          ;
-
-          // prompt the user for confirmation given the dns query and dnssec status
-          std::string confirm_dns_ok = command_line::input_line(prompt.str());
-          if (std::cin.eof())
-          {
+    if (!get_address_from_str(local_args[i], de.addr, has_payment_id, new_payment_id))
       return true;
-          }
-          if (confirm_dns_ok != "Y" && confirm_dns_ok != "y" && confirm_dns_ok != "Yes" && confirm_dns_ok != "yes"
-            && confirm_dns_ok != tr("yes") && confirm_dns_ok != tr("no"))
-          {
-            fail_msg_writer() << tr("you have cancelled the transfer request");
-            return true;
-          }
-        }
-        else
-        {
-          fail_msg_writer() << tr("failed to get a Monero address from: ") << local_args[i];
-          return true;
-        }
-      }
-      else if (addresses_from_dns.size() > 1)
-      {
-        fail_msg_writer() << tr("not yet supported: Multiple Monero addresses found for given URL: ") << url;
-      }
-      else
-      {
-        fail_msg_writer() << tr("wrong address: ") << local_args[i];
-        return true;
-      }
-    }
 
     if (has_payment_id)
     {
@@ -2202,24 +2286,36 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
     if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1)
     {
         uint64_t total_fee = 0;
+        uint64_t dust_not_in_fee = 0;
+        uint64_t dust_in_fee = 0;
         for (size_t n = 0; n < ptx_vector.size(); ++n)
         {
           total_fee += ptx_vector[n].fee;
+
+          if (ptx_vector[n].dust_added_to_fee)
+            dust_in_fee += ptx_vector[n].dust;
+          else
+            dust_not_in_fee += ptx_vector[n].dust;
         }
 
-        std::string prompt_str;
+        std::stringstream prompt;
         if (ptx_vector.size() > 1)
         {
-          prompt_str = (boost::format(tr("Your transaction needs to be split into %llu transactions.  "
-            "This will result in a transaction fee being applied to each transaction, for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
-            ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str();
+          prompt << boost::format(tr("Your transaction needs to be split into %llu transactions.  "
+            "This will result in a transaction fee being applied to each transaction, for a total fee of %s")) %
+            ((unsigned long long)ptx_vector.size()) % print_money(total_fee);
         }
         else
         {
-          prompt_str = (boost::format(tr("The transaction fee is %s.  Is this okay?  (Y/Yes/N/No)")) %
-            print_money(total_fee)).str();
+          prompt << boost::format(tr("The transaction fee is %s")) %
+            print_money(total_fee);
         }
-        std::string accepted = command_line::input_line(prompt_str);
+        if (dust_in_fee != 0) prompt << boost::format(tr(", of which %s is dust from change")) % print_money(dust_in_fee);
+        if (dust_not_in_fee != 0)  prompt << tr(".") << ENDL << boost::format(tr("A total of %s from dust change will be sent to dust address")) 
+                                                   % print_money(dust_not_in_fee);
+        prompt << tr(".") << ENDL << tr("Is this okay?  (Y/Yes/N/No)");
+        
+        std::string accepted = command_line::input_line(prompt.str());
         if (std::cin.eof())
           return true;
         if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes")
@@ -2346,6 +2442,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
      return true;
   }
 
+  LOCK_REFRESH_THREAD_SCOPE();
   try
   {
     // figure out what tx will be necessary
@@ -2486,6 +2583,238 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool simple_wallet::sweep_all(const std::vector<std::string> &args_)
+{
+  if (!try_connect_to_daemon())
+    return true;
+
+  if(m_wallet->watch_only())
+  {
+     fail_msg_writer() << tr("this is a watch only wallet");
+     return true;
+  }
+
+  std::vector<std::string> local_args = args_;
+
+  size_t fake_outs_count;
+  if(local_args.size() > 0) {
+    if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0]))
+    {
+      fake_outs_count = m_wallet->default_mixin();
+      if (fake_outs_count == 0)
+        fake_outs_count = DEFAULT_MIX;
+    }
+    else
+    {
+      local_args.erase(local_args.begin());
+    }
+  }
+
+  std::vector<uint8_t> extra;
+  bool payment_id_seen = false;
+  if (2 == local_args.size())
+  {
+    std::string payment_id_str = local_args.back();
+    local_args.pop_back();
+
+    crypto::hash payment_id;
+    bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id);
+    if(r)
+    {
+      std::string extra_nonce;
+      set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id);
+      r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
+    }
+    else
+    {
+      crypto::hash8 payment_id8;
+      r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8);
+      if(r)
+      {
+        std::string extra_nonce;
+        set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8);
+        r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
+      }
+    }
+
+    if(!r)
+    {
+      fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str;
+      return true;
+    }
+    payment_id_seen = true;
+  }
+
+  if (local_args.size() == 0)
+  {
+    fail_msg_writer() << tr("No address given");
+    return true;
+  }
+
+  bool has_payment_id;
+  crypto::hash8 new_payment_id;
+  cryptonote::account_public_address address;
+  if (!get_address_from_str(local_args[0], address, has_payment_id, new_payment_id))
+    return true;
+
+  if (has_payment_id)
+  {
+    if (payment_id_seen)
+    {
+      fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[0];
+      return true;
+    }
+
+    std::string extra_nonce;
+    set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, new_payment_id);
+    bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
+    if(!r)
+    {
+      fail_msg_writer() << tr("failed to set up payment id, though it was decoded correctly");
+      return true;
+    }
+  }
+
+  try
+  {
+    // figure out what tx will be necessary
+    auto ptx_vector = m_wallet->create_transactions_all(address, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon);
+
+    if (ptx_vector.empty())
+    {
+      fail_msg_writer() << tr("No outputs found");
+      return true;
+    }
+
+    // give user total and fee, and prompt to confirm
+    uint64_t total_fee = 0, total_sent = 0;
+    for (size_t n = 0; n < ptx_vector.size(); ++n)
+    {
+      total_fee += ptx_vector[n].fee;
+      for (const auto &vin: ptx_vector[n].tx.vin)
+      {
+        if (vin.type() == typeid(txin_to_key))
+          total_sent += boost::get<txin_to_key>(vin).amount;
+      }
+    }
+
+    std::string prompt_str;
+    if (ptx_vector.size() > 1) {
+      prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
+        print_money(total_sent) %
+        ((unsigned long long)ptx_vector.size()) %
+        print_money(total_fee)).str();
+    }
+    else {
+      prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
+        print_money(total_sent) %
+        print_money(total_fee)).str();
+    }
+    std::string accepted = command_line::input_line(prompt_str);
+    if (std::cin.eof())
+      return true;
+    if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes")
+    {
+      fail_msg_writer() << tr("transaction cancelled.");
+
+      // would like to return false, because no tx made, but everything else returns true
+      // and I don't know what returning false might adversely affect.  *sigh*
+      return true;
+    }
+
+    // actually commit the transactions
+    while (!ptx_vector.empty())
+    {
+      auto & ptx = ptx_vector.back();
+      m_wallet->commit_tx(ptx);
+      success_msg_writer(true) << tr("Money successfully sent, transaction: ") << get_transaction_hash(ptx.tx);
+
+      // if no exception, remove element from vector
+      ptx_vector.pop_back();
+    }
+  }
+  catch (const tools::error::daemon_busy&)
+  {
+    fail_msg_writer() << tr("daemon is busy. Please try again later.");
+  }
+  catch (const tools::error::no_connection_to_daemon&)
+  {
+    fail_msg_writer() << tr("no connection to daemon. Please make sure daemon is running.");
+  }
+  catch (const tools::error::wallet_rpc_error& e)
+  {
+    LOG_ERROR("RPC error: " << e.to_string());
+    fail_msg_writer() << tr("RPC error: ") << e.what();
+  }
+  catch (const tools::error::get_random_outs_error&)
+  {
+    fail_msg_writer() << tr("failed to get random outputs to mix");
+  }
+  catch (const tools::error::not_enough_money& e)
+  {
+    fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee).\n%s")) %
+      print_money(e.available()) %
+      print_money(e.tx_amount() + e.fee()) %
+      print_money(e.tx_amount()) %
+      print_money(e.fee()) %
+      tr("This is usually due to dust which is so small it cannot pay for itself in fees");
+  }
+  catch (const tools::error::not_enough_outs_to_mix& e)
+  {
+    auto writer = fail_msg_writer();
+    writer << tr("not enough outputs for specified mixin_count") << " = " << e.mixin_count() << ":";
+    for (const cryptonote::COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& outs_for_amount : e.scanty_outs())
+    {
+      writer << "\n" << tr("output amount") << " = " << print_money(outs_for_amount.amount) << ", " << tr("found outputs to mix") << " = " << outs_for_amount.outs.size();
+    }
+  }
+  catch (const tools::error::tx_not_constructed&)
+  {
+    fail_msg_writer() << tr("transaction was not constructed");
+  }
+  catch (const tools::error::tx_rejected& e)
+  {
+    fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status();
+    std::string reason = e.reason();
+    if (!reason.empty())
+      fail_msg_writer() << tr("Reason: ") << reason;
+  }
+  catch (const tools::error::tx_sum_overflow& e)
+  {
+    fail_msg_writer() << e.what();
+  }
+  catch (const tools::error::zero_destination&)
+  {
+    fail_msg_writer() << tr("one of destinations is zero");
+  }
+  catch (const tools::error::tx_too_big& e)
+  {
+    fail_msg_writer() << tr("failed to find a suitable way to split transactions");
+  }
+  catch (const tools::error::transfer_error& e)
+  {
+    LOG_ERROR("unknown transfer error: " << e.to_string());
+    fail_msg_writer() << tr("unknown transfer error: ") << e.what();
+  }
+  catch (const tools::error::wallet_internal_error& e)
+  {
+    LOG_ERROR("internal error: " << e.to_string());
+    fail_msg_writer() << tr("internal error: ") << e.what();
+  }
+  catch (const std::exception& e)
+  {
+    LOG_ERROR("unexpected error: " << e.what());
+    fail_msg_writer() << tr("unexpected error: ") << e.what();
+  }
+  catch (...)
+  {
+    LOG_ERROR("unknown error");
+    fail_msg_writer() << tr("unknown error");
+  }
+
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
 {
   std::vector<std::string> local_args = args_;
@@ -2503,6 +2832,8 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
   }
   crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
 
+  LOCK_REFRESH_THREAD_SCOPE();
+
   crypto::secret_key tx_key;
   bool r = m_wallet->get_tx_key(txid, tx_key);
   if (r)
@@ -2537,6 +2868,8 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_)
   }
   crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
 
+  LOCK_REFRESH_THREAD_SCOPE();
+
   cryptonote::blobdata tx_key_data;
   if(!epee::string_tools::parse_hexstr_to_binbuff(local_args[1], tx_key_data))
   {
@@ -2626,6 +2959,23 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_)
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+static std::string get_human_readable_timestamp(uint64_t ts)
+{
+  char buffer[64];
+  if (ts < 1234567890)
+    return "<unknown>";
+  time_t tt = ts;
+  struct tm tm;
+  gmtime_r(&tt, &tm);
+  uint64_t now = time(NULL);
+  uint64_t diff = ts > now ? ts - now : now - ts;
+  if (diff > 24*3600)
+    strftime(buffer, sizeof(buffer), "%F", &tm);
+  else
+    strftime(buffer, sizeof(buffer), "%r", &tm);
+  return std::string(buffer);
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
 {
   std::vector<std::string> local_args = args_;
@@ -2641,6 +2991,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
     return true;
   }
 
+  LOCK_REFRESH_THREAD_SCOPE();
+
   // optional in/out selector
   if (local_args.size() > 0) {
     if (local_args[0] == "in" || local_args[0] == "incoming") {
@@ -2698,7 +3050,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
       std::string payment_id = string_tools::pod_to_hex(i->first);
       if (payment_id.substr(16).find_first_not_of('0') == std::string::npos)
         payment_id = payment_id.substr(0,16);
-      output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%20.20s %s %s %s") % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % "-").str())));
+      std::string note = m_wallet->get_tx_note(pd.m_tx_hash);
+      output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%16.16s %20.20s %s %s %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % "-" % note).str())));
     }
   }
 
@@ -2718,7 +3071,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
       std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id);
       if (payment_id.substr(16).find_first_not_of('0') == std::string::npos)
         payment_id = payment_id.substr(0,16);
-      output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%20.20s %s %s %14.14s %s") % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests).str())));
+      std::string note = m_wallet->get_tx_note(i->first);
+      output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%16.16s %20.20s %s %s %14.14s %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % note).str())));
     }
   }
 
@@ -2741,9 +3095,10 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
       std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id);
       if (payment_id.substr(16).find_first_not_of('0') == std::string::npos)
         payment_id = payment_id.substr(0,16);
+      std::string note = m_wallet->get_tx_note(i->first);
       bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed;
       if ((failed && is_failed) || (!is_failed && pending)) {
-        message_writer() << (boost::format("%8.8s %6.6s %20.20s %s %s %14.14s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % print_money(amount - pd.m_change) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee)).str();
+        message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %14.14s %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % note).str();
       }
     }
   }
@@ -2795,7 +3150,12 @@ bool simple_wallet::run()
 void simple_wallet::stop()
 {
   m_cmd_binder.stop_handling();
+
+  m_auto_refresh_run.store(false, std::memory_order_relaxed);
   m_wallet->stop();
+  // make the background refresh thread quit
+  boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex);
+  m_auto_refresh_cond.notify_one();
 }
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
@@ -2846,6 +3206,59 @@ bool simple_wallet::print_integrated_address(const std::vector<std::string> &arg
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool simple_wallet::set_tx_note(const std::vector<std::string> &args)
+{
+  if (args.size() == 0)
+  {
+    fail_msg_writer() << tr("usage: set_tx_note [txid] free text note");
+    return true;
+  }
+
+  cryptonote::blobdata txid_data;
+  if(!epee::string_tools::parse_hexstr_to_binbuff(args.front(), txid_data))
+  {
+    fail_msg_writer() << tr("failed to parse txid");
+    return false;
+  }
+  crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
+
+  std::string note = "";
+  for (size_t n = 1; n < args.size(); ++n)
+  {
+    if (n > 1)
+      note += " ";
+    note += args[n];
+  }
+  m_wallet->set_tx_note(txid, note);
+
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
+bool simple_wallet::get_tx_note(const std::vector<std::string> &args)
+{
+  if (args.size() != 1)
+  {
+    fail_msg_writer() << tr("usage: get_tx_note [txid]");
+    return true;
+  }
+
+  cryptonote::blobdata txid_data;
+  if(!epee::string_tools::parse_hexstr_to_binbuff(args.front(), txid_data))
+  {
+    fail_msg_writer() << tr("failed to parse txid");
+    return false;
+  }
+  crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
+
+  std::string note = m_wallet->get_tx_note(txid);
+  if (note.empty())
+    success_msg_writer() << "no note found";
+  else
+    success_msg_writer() << "note found: " << note;
+
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::process_command(const std::vector<std::string> &args)
 {
   return m_cmd_binder.process_command_vec(args);
@@ -2894,6 +3307,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_params, arg_daemon_port);
   command_line::add_arg(desc_params, arg_command);
   command_line::add_arg(desc_params, arg_log_level);
+  command_line::add_arg(desc_params, arg_max_concurrency);
 
   bf::path default_log {log_space::log_singletone::get_default_log_folder()};
   std::string log_file_name = log_space::log_singletone::get_default_log_file();
@@ -2921,6 +3335,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_params, arg_testnet);
   command_line::add_arg(desc_params, arg_restricted);
   command_line::add_arg(desc_params, arg_trusted_daemon);
+  command_line::add_arg(desc_params, arg_restore_height);
   tools::wallet_rpc_server::init_options(desc_params);
 
   po::positional_options_description positional_options;
@@ -2978,6 +3393,9 @@ int main(int argc, char* argv[])
     LOG_LEVEL_4
     );
 
+  if(command_line::has_arg(vm, arg_max_concurrency))
+    tools::set_max_concurrency(command_line::get_arg(vm, arg_max_concurrency));
+
   message_writer(epee::log_space::console_color_white, true) << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")";
 
   if(command_line::has_arg(vm, arg_log_level))

src/simplewallet/simplewallet.h

@@ -114,6 +114,7 @@ namespace cryptonote
     bool stop_mining(const std::vector<std::string> &args);
     bool save_bc(const std::vector<std::string>& args);
     bool refresh(const std::vector<std::string> &args);
+    bool show_balance_unlocked();
     bool show_balance(const std::vector<std::string> &args = std::vector<std::string>());
     bool show_incoming_transfers(const std::vector<std::string> &args);
     bool show_payments(const std::vector<std::string> &args);
@@ -121,6 +122,7 @@ namespace cryptonote
     bool transfer_main(bool new_algorithm, const std::vector<std::string> &args);
     bool transfer(const std::vector<std::string> &args);
     bool transfer_new(const std::vector<std::string> &args);
+    bool sweep_all(const std::vector<std::string> &args);
     bool sweep_unmixable(const std::vector<std::string> &args);
     std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts(
         std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits
@@ -137,10 +139,13 @@ namespace cryptonote
     bool show_transfers(const std::vector<std::string> &args);
     bool rescan_blockchain(const std::vector<std::string> &args);
     bool refresh_main(uint64_t start_height, bool reset = false);
+    bool set_tx_note(const std::vector<std::string> &args);
+    bool get_tx_note(const std::vector<std::string> &args);
 
     uint64_t get_daemon_blockchain_height(std::string& err);
     bool try_connect_to_daemon();
     bool ask_wallet_create_if_needed();
+    bool get_address_from_str(const std::string &str, cryptonote::account_public_address &address, bool &has_payment_id, crypto::hash8 &payment_id);
 
     /*!
      * \brief Prints the seed with a nice message
@@ -231,6 +236,7 @@ namespace cryptonote
     bool m_restore_deterministic_wallet;  // recover flag
     bool m_non_deterministic;  // old 2-random generation
     bool m_trusted_daemon;
+    uint64_t m_restore_height;  // optional
 
     std::string m_daemon_address;
     std::string m_daemon_host;

src/wallet/wallet2.cpp

@@ -184,7 +184,7 @@ void wallet2::check_acc_out(const account_keys &acc, const tx_out &o, const cryp
   error = false;
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx)
+void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, bool miner_tx)
 {
   if (!miner_tx)
     process_unconfirmed(tx, height);
@@ -213,7 +213,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
 
     tx_pub_key = pub_key_field.pub_key;
     bool r = true;
-    int threads = boost::thread::hardware_concurrency();
+    int threads = tools::get_max_concurrency();
     if (miner_tx && m_refresh_type == RefreshNoCoinbase)
     {
       // assume coinbase isn't for us
@@ -409,7 +409,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
 
   if (tx_money_spent_in_ins > 0)
   {
-    process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs);
+    process_outgoing(tx, height, ts, tx_money_spent_in_ins, tx_money_got_in_outs);
   }
 
   uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0;
@@ -459,6 +459,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
     payment.m_amount       = received;
     payment.m_block_height = height;
     payment.m_unlock_time  = tx.unlock_time;
+    payment.m_timestamp    = ts;
     m_payments.emplace(payment_id, payment);
     LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount);
   }
@@ -482,7 +483,7 @@ void wallet2::process_unconfirmed(const cryptonote::transaction& tx, uint64_t he
   }
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t height, uint64_t spent, uint64_t received)
+void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received)
 {
   crypto::hash txid = get_transaction_hash(tx);
   confirmed_transfer_details &ctd = m_confirmed_txs[txid];
@@ -492,6 +493,7 @@ void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t heigh
   ctd.m_amount_out = get_outs_money_amount(tx);
   ctd.m_change = received;
   ctd.m_block_height = height;
+  ctd.m_timestamp = ts;
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height)
@@ -502,7 +504,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry
   if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height)
   {
     TIME_MEASURE_START(miner_tx_handle_time);
-    process_new_transaction(b.miner_tx, height, true);
+    process_new_transaction(b.miner_tx, height, b.timestamp, true);
     TIME_MEASURE_FINISH(miner_tx_handle_time);
 
     TIME_MEASURE_START(txs_handle_time);
@@ -511,7 +513,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry
       cryptonote::transaction tx;
       bool r = parse_and_validate_tx_from_blob(txblob, tx);
       THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob);
-      process_new_transaction(tx, height, false);
+      process_new_transaction(tx, height, b.timestamp, false);
     }
     TIME_MEASURE_FINISH(txs_handle_time);
     LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " <<  miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms");
@@ -578,12 +580,30 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height,
   blocks = res.blocks;
 }
 //----------------------------------------------------------------------------------------------------
+void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes)
+{
+  cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req);
+  cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res);
+  req.block_ids = short_chain_history;
+
+  req.start_height = start_height;
+  m_daemon_rpc_mutex.lock();
+  bool r = net_utils::invoke_http_bin_remote_command2(m_daemon_address + "/gethashes.bin", req, res, m_http_client, WALLET_RCP_CONNECTION_TIMEOUT);
+  m_daemon_rpc_mutex.unlock();
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin");
+  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin");
+  THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status);
+
+  blocks_start_height = res.start_height;
+  hashes = res.m_block_ids;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added)
 {
   size_t current_index = start_height;
   blocks_added = 0;
 
-  int threads = boost::thread::hardware_concurrency();
+  int threads = tools::get_max_concurrency();
   if (threads > 1)
   {
     std::vector<crypto::hash> round_block_hashes(threads);
@@ -771,6 +791,60 @@ void wallet2::check_pending_txes()
     }
   }
 }
+//----------------------------------------------------------------------------------------------------
+void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history)
+{
+  std::list<crypto::hash> hashes;
+  size_t current_index = m_blockchain.size();
+  while(current_index < stop_height)
+  {
+    pull_hashes(0, blocks_start_height, short_chain_history, hashes);
+    if (hashes.size() < 3)
+      return;
+    if (hashes.size() + current_index < stop_height) {
+      std::list<crypto::hash>::iterator right;
+      // drop early 3 off, skipping the genesis block
+      if (short_chain_history.size() > 3) {
+        right = short_chain_history.end();
+        std::advance(right,-1);
+        std::list<crypto::hash>::iterator left = right;
+        std::advance(left, -3);
+        short_chain_history.erase(left, right);
+      }
+      right = hashes.end();
+      // prepend 3 more
+      for (int i = 0; i<3; i++) {
+        right--;
+        short_chain_history.push_front(*right);
+      }
+    }
+    current_index = blocks_start_height;
+    BOOST_FOREACH(auto& bl_id, hashes)
+    {
+      if(current_index >= m_blockchain.size())
+      {
+        LOG_PRINT_L2( "Skipped block by height: " << current_index);
+        m_blockchain.push_back(bl_id);
+        ++m_local_bc_height;
+
+        if (0 != m_callback)
+        { // FIXME: this isn't right, but simplewallet just logs that we got a block.
+          cryptonote::block dummy;
+          m_callback->on_new_block(current_index, dummy);
+        }
+      }
+      else if(bl_id != m_blockchain[current_index])
+      {
+        //split detected here !!!
+        return;
+      }
+      ++current_index;
+      if (current_index >= stop_height)
+        return;
+    }
+  }
+}
+
 //----------------------------------------------------------------------------------------------------
 void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money)
 {
@@ -786,7 +860,22 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re
 
   // pull the first set of blocks
   get_short_chain_history(short_chain_history);
+  if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) {
+    if (!start_height)
+      start_height = m_refresh_from_block_height;
+    // we can shortcut by only pulling hashes up to the start_height
+    fast_refresh(start_height, blocks_start_height, short_chain_history);
+    // regenerate the history now that we've got a full set of hashes
+    short_chain_history.clear();
+    get_short_chain_history(short_chain_history);
+    start_height = 0;
+    // and then fall through to regular refresh processing
+  }
+
   pull_blocks(start_height, blocks_start_height, short_chain_history, blocks);
+  // always reset start_height to 0 to force short_chain_ history to be used on
+  // subsequent pulls in this refresh.
+  start_height = 0;
 
   m_run.store(true, std::memory_order_relaxed);
   while(m_run.load(std::memory_order_relaxed))
@@ -1758,6 +1847,7 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, const std::v
   utd.m_dests = dests;
   utd.m_payment_id = payment_id;
   utd.m_state = wallet2::unconfirmed_transfer_details::pending;
+  utd.m_timestamp = time(NULL);
 }
 
 //----------------------------------------------------------------------------------------------------
@@ -1950,8 +2040,9 @@ void wallet2::commit_tx(pending_tx& ptx)
   BOOST_FOREACH(transfer_container::iterator it, ptx.selected_transfers)
     it->m_spent = true;
 
+  //fee includes dust if dust policy specified it.
   LOG_PRINT_L0("Transaction successfully sent. <" << txid << ">" << ENDL
-            << "Commission: " << print_money(ptx.fee+ptx.dust) << " (dust: " << print_money(ptx.dust) << ")" << ENDL
+            << "Commission: " << print_money(ptx.fee) << " (dust sent to dust addr: " << print_money((ptx.dust_added_to_fee ? 0 : ptx.dust)) << ")" << ENDL
             << "Balance: " << print_money(balance()) << ENDL
             << "Unlocked: " << print_money(unlocked_balance()) << ENDL
             << "Please, wait for confirmation for your balance to be unlocked.");
@@ -2210,9 +2301,16 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent
   });
   THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx);
   
+  
+  bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key
+                                || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key);
+  
+  if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust;
+
   ptx.key_images = key_images;
-  ptx.fee = fee;
-  ptx.dust = dust;
+  ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee);
+  ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0);
+  ptx.dust_added_to_fee = dust_policy.add_to_fee;
   ptx.tx = tx;
   ptx.change_dts = change_dts;
   ptx.selected_transfers = selected_transfers;
@@ -2377,7 +2475,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
         detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx);
       auto txBlob = t_serializable_object_to_blob(test_ptx.tx);
       needed_fee = calculate_fee(txBlob);
-      available_for_fee = test_ptx.fee + test_ptx.change_dts.amount;
+      available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0);
       LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
         print_money(needed_fee) << " needed)");
 
@@ -2459,6 +2557,133 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
   return ptx_vector;
 }
 
+std::vector<wallet2::pending_tx> wallet2::create_transactions_all(const cryptonote::account_public_address &address, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon)
+{
+  std::vector<size_t> unused_transfers_indices;
+  std::vector<size_t> unused_dust_indices;
+  uint64_t accumulated_fee, accumulated_outputs, accumulated_change;
+  struct TX {
+    std::list<transfer_container::iterator> selected_transfers;
+    std::vector<cryptonote::tx_destination_entry> dsts;
+    cryptonote::transaction tx;
+    pending_tx ptx;
+    size_t bytes;
+  };
+  std::vector<TX> txes;
+  uint64_t needed_fee, available_for_fee = 0;
+  uint64_t upper_transaction_size_limit = get_upper_tranaction_size_limit();
+
+  // gather all our dust and non dust outputs
+  for (size_t i = 0; i < m_transfers.size(); ++i)
+  {
+    const transfer_details& td = m_transfers[i];
+    if (!td.m_spent && is_transfer_unlocked(td))
+    {
+      if (is_valid_decomposed_amount(td.amount()))
+        unused_transfers_indices.push_back(i);
+      else
+        unused_dust_indices.push_back(i);
+    }
+  }
+  LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs");
+
+  // start with an empty tx
+  txes.push_back(TX());
+  accumulated_fee = 0;
+  accumulated_outputs = 0;
+  accumulated_change = 0;
+  needed_fee = 0;
+
+  // while we have something to send
+  while (!unused_dust_indices.empty() || !unused_transfers_indices.empty()) {
+    TX &tx = txes.back();
+
+    // get a random unspent output and use it to pay next chunk. We try to alternate
+    // dust and non dust to ensure we never get with only dust, from which we might
+    // get a tx that can't pay for itself
+    size_t idx = unused_transfers_indices.empty() ? pop_random_value(unused_dust_indices) : unused_dust_indices.empty() ? pop_random_value(unused_transfers_indices) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > FEE_PER_KB * (upper_transaction_size_limit + 1023) / 1024) ? pop_random_value(unused_dust_indices) : pop_random_value(unused_transfers_indices);
+
+    const transfer_details &td = m_transfers[idx];
+    LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount()));
+
+    // add this output to the list to spend
+    tx.selected_transfers.push_back(m_transfers.begin() + idx);
+    uint64_t available_amount = td.amount();
+    accumulated_outputs += available_amount;
+
+    // here, check if we need to sent tx and start a new one
+    LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit "
+      << upper_transaction_size_limit);
+    bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || (tx.selected_transfers.size() * (fake_outs_count+1) * APPROXIMATE_INPUT_BYTES >= TX_SIZE_TARGET(upper_transaction_size_limit));
+
+    if (try_tx) {
+      cryptonote::transaction test_tx;
+      pending_tx test_ptx;
+
+      needed_fee = 0;
+
+      tx.dsts.push_back(tx_destination_entry(1, address));
+
+      LOG_PRINT_L2("Trying to create a tx now, with " << tx.dsts.size() << " destinations and " <<
+        tx.selected_transfers.size() << " outputs");
+      transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra,
+        detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx);
+      auto txBlob = t_serializable_object_to_blob(test_ptx.tx);
+      needed_fee = calculate_fee(txBlob);
+      available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount;
+      LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
+        print_money(needed_fee) << " needed)");
+
+      THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself");
+
+      do {
+        LOG_PRINT_L2("We made a tx, adjusting fee and saving it");
+        tx.dsts[0].amount = available_for_fee - needed_fee;
+        transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra,
+          detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx);
+        txBlob = t_serializable_object_to_blob(test_ptx.tx);
+        needed_fee = calculate_fee(txBlob);
+        LOG_PRINT_L2("Made an attempt at a final " << ((txBlob.size() + 1023)/1024) << " kB tx, with " << print_money(test_ptx.fee) <<
+          " fee  and " << print_money(test_ptx.change_dts.amount) << " change");
+      } while (needed_fee > test_ptx.fee);
+
+      LOG_PRINT_L2("Made a final " << ((txBlob.size() + 1023)/1024) << " kB tx, with " << print_money(test_ptx.fee) <<
+        " fee  and " << print_money(test_ptx.change_dts.amount) << " change");
+
+      tx.tx = test_tx;
+      tx.ptx = test_ptx;
+      tx.bytes = txBlob.size();
+      accumulated_fee += test_ptx.fee;
+      accumulated_change += test_ptx.change_dts.amount;
+      if (!unused_transfers_indices.empty() || !unused_dust_indices.empty())
+      {
+        LOG_PRINT_L2("We have more to pay, starting another tx");
+        txes.push_back(TX());
+      }
+    }
+  }
+
+  LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) <<
+    " total fee, " << print_money(accumulated_change) << " total change");
+
+  std::vector<wallet2::pending_tx> ptx_vector;
+  for (std::vector<TX>::iterator i = txes.begin(); i != txes.end(); ++i)
+  {
+    TX &tx = *i;
+    uint64_t tx_money = 0;
+    for (std::list<transfer_container::iterator>::const_iterator mi = tx.selected_transfers.begin(); mi != tx.selected_transfers.end(); ++mi)
+      tx_money += (*mi)->amount();
+    LOG_PRINT_L1("  Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() <<
+      ": " << (tx.bytes+1023)/1024 << " kB, sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() <<
+      " outputs to " << tx.dsts.size() << " destination(s), including " <<
+      print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change");
+    ptx_vector.push_back(tx.ptx);
+  }
+
+  // if we made it this far, we're OK to actually send the transactions
+  return ptx_vector;
+}
+
 uint64_t wallet2::unlocked_dust_balance(const tx_dust_policy &dust_policy) const
 {
   uint64_t money = 0;
@@ -2835,6 +3060,19 @@ std::string wallet2::get_daemon_address() const
   return m_daemon_address;
 }
 
+void wallet2::set_tx_note(const crypto::hash &txid, const std::string &note)
+{
+  m_tx_notes[txid] = note;
+}
+
+std::string wallet2::get_tx_note(const crypto::hash &txid) const
+{
+  std::unordered_map<crypto::hash, std::string>::const_iterator i = m_tx_notes.find(txid);
+  if (i == m_tx_notes.end())
+    return std::string();
+  return i->second;
+}
+
 //----------------------------------------------------------------------------------------------------
 void wallet2::generate_genesis(cryptonote::block& b) {
   if (m_testnet)

src/wallet/wallet2.h

@@ -110,6 +110,7 @@ namespace tools
       uint64_t m_amount;
       uint64_t m_block_height;
       uint64_t m_unlock_time;
+      uint64_t m_timestamp;
     };
 
     struct unconfirmed_transfer_details
@@ -120,6 +121,7 @@ namespace tools
       std::vector<cryptonote::tx_destination_entry> m_dests;
       crypto::hash m_payment_id;
       enum { pending, pending_not_in_pool, failed } m_state;
+      uint64_t m_timestamp;
     };
 
     struct confirmed_transfer_details
@@ -130,10 +132,11 @@ namespace tools
       uint64_t m_block_height;
       std::vector<cryptonote::tx_destination_entry> m_dests;
       crypto::hash m_payment_id;
+      uint64_t m_timestamp;
 
       confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(cryptonote::null_hash) {}
       confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height):
-        m_amount_out(get_outs_money_amount(utd.m_tx)), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id) { get_inputs_money_amount(utd.m_tx, m_amount_in); }
+        m_amount_out(get_outs_money_amount(utd.m_tx)), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp) { get_inputs_money_amount(utd.m_tx, m_amount_in); }
     };
 
     typedef std::vector<transfer_details> transfer_container;
@@ -143,6 +146,7 @@ namespace tools
     {
       cryptonote::transaction tx;
       uint64_t dust, fee;
+      bool dust_added_to_fee;
       cryptonote::tx_destination_entry change_dts;
       std::list<transfer_container::iterator> selected_transfers;
       std::string key_images;
@@ -289,6 +293,7 @@ namespace tools
     void commit_tx(std::vector<pending_tx>& ptx_vector);
     std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra, bool trusted_daemon);
     std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon);
+    std::vector<wallet2::pending_tx> create_transactions_all(const cryptonote::account_public_address &address, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon);
     std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon);
     bool check_connection();
     void get_transfers(wallet2::transfer_container& incoming_transfers) const;
@@ -324,6 +329,9 @@ namespace tools
       if(ver < 11)
         return;
       a & m_refresh_from_block_height;
+      if(ver < 12)
+        return;
+      a & m_tx_notes;
     }
 
     /*!
@@ -365,6 +373,15 @@ namespace tools
     std::string get_wallet_file() const;
     std::string get_keys_file() const;
     std::string get_daemon_address() const;
+
+    std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon);
+    std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f);
+    std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon);
+    std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon);
+
+    void set_tx_note(const crypto::hash &txid, const std::string &note);
+    std::string get_tx_note(const crypto::hash &txid) const;
+
   private:
     /*!
      * \brief  Stores wallet information to wallet file.
@@ -380,7 +397,7 @@ namespace tools
      * \param password       Password of wallet file
      */
     bool load_keys(const std::string& keys_file_name, const std::string& password);
-    void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx);
+    void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, bool miner_tx);
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height);
     void detach_blockchain(uint64_t height);
     void get_short_chain_history(std::list<crypto::hash>& ids) const;
@@ -388,12 +405,14 @@ namespace tools
     bool is_transfer_unlocked(const transfer_details& td) const;
     bool clear();
     void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks);
+    void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes);
+    void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history);
     void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, bool &error);
     void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added);
     uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon);
     bool prepare_file_names(const std::string& file_path);
     void process_unconfirmed(const cryptonote::transaction& tx, uint64_t height);
-    void process_outgoing(const cryptonote::transaction& tx, uint64_t height, uint64_t spent, uint64_t received);
+    void process_outgoing(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received);
     void add_unconfirmed_tx(const cryptonote::transaction& tx, const std::vector<cryptonote::tx_destination_entry> &dests, const crypto::hash &payment_id, uint64_t change_amount);
     void generate_genesis(cryptonote::block& b);
     void check_genesis(const crypto::hash& genesis_hash) const; //throws
@@ -404,10 +423,6 @@ namespace tools
     uint64_t get_upper_tranaction_size_limit();
     void check_pending_txes();
     std::vector<uint64_t> get_unspent_amounts_vector();
-    std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon);
-    std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f);
-    std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon);
-    std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon);
 
     cryptonote::account_base m_account;
     std::string m_daemon_address;
@@ -424,6 +439,7 @@ namespace tools
     payment_container m_payments;
     std::unordered_map<crypto::key_image, size_t> m_key_images;
     cryptonote::account_public_address m_account_public_address;
+    std::unordered_map<crypto::hash, std::string> m_tx_notes;
     uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value
 
     std::atomic<bool> m_run;
@@ -444,10 +460,10 @@ namespace tools
     uint64_t m_refresh_from_block_height;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 11)
-BOOST_CLASS_VERSION(tools::wallet2::payment_details, 0)
-BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 2)
-BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 1)
+BOOST_CLASS_VERSION(tools::wallet2, 12)
+BOOST_CLASS_VERSION(tools::wallet2::payment_details, 1)
+BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 3)
+BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 2)
 
 namespace boost
 {
@@ -477,6 +493,9 @@ namespace boost
       if (ver < 2)
         return;
       a & x.m_state;
+      if (ver < 3)
+        return;
+      a & x.m_timestamp;
     }
 
     template <class Archive>
@@ -490,6 +509,9 @@ namespace boost
         return;
       a & x.m_dests;
       a & x.m_payment_id;
+      if (ver < 2)
+        return;
+      a & x.m_timestamp;
     }
 
     template <class Archive>
@@ -499,6 +521,9 @@ namespace boost
       a & x.m_amount;
       a & x.m_block_height;
       a & x.m_unlock_time;
+      if (ver < 1)
+        return;
+      a & x.m_timestamp;
     }
 
     template <class Archive>
@@ -713,9 +738,15 @@ namespace tools
     });
     THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx);
     
+    bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key
+                                || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key);      
+
+    if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust;
+
     ptx.key_images = key_images;
-    ptx.fee = fee;
-    ptx.dust = dust;
+    ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee);
+    ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0);
+    ptx.dust_added_to_fee = dust_policy.add_to_fee;
     ptx.tx = tx;
     ptx.change_dts = change_dts;
     ptx.selected_transfers = selected_transfers;

src/wallet/wallet_errors.h

@@ -60,6 +60,7 @@ namespace tools
     //         acc_outs_lookup_error
     //         block_parse_error
     //         get_blocks_error
+    //         get_hashes_error
     //         get_out_indexes_error
     //         tx_parse_error
     //         get_tx_pool_error
@@ -107,12 +108,14 @@ namespace tools
     //----------------------------------------------------------------------------------------------------
     const char* const failed_rpc_request_messages[] = {
       "failed to get blocks",
+      "failed to get hashes",
       "failed to get out indices",
       "failed to get random outs"
     };
     enum failed_rpc_request_message_indices
     {
       get_blocks_error_message_index,
+      get_hashes_error_message_index,
       get_out_indices_error_message_index,
       get_random_outs_error_message_index
     };
@@ -291,6 +294,8 @@ namespace tools
     //----------------------------------------------------------------------------------------------------
     typedef failed_rpc_request<refresh_error, get_blocks_error_message_index> get_blocks_error;
     //----------------------------------------------------------------------------------------------------
+    typedef failed_rpc_request<refresh_error, get_hashes_error_message_index> get_hashes_error;
+    //----------------------------------------------------------------------------------------------------
     typedef failed_rpc_request<refresh_error, get_out_indices_error_message_index> get_out_indices_error;
     //----------------------------------------------------------------------------------------------------
     struct tx_parse_error : public refresh_error

src/wallet/wallet_rpc_server.cpp

@@ -382,6 +382,65 @@ namespace tools
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er)
+  {
+    std::vector<cryptonote::tx_destination_entry> dsts;
+    std::vector<uint8_t> extra;
+
+    if (m_wallet.restricted())
+    {
+      er.code = WALLET_RPC_ERROR_CODE_DENIED;
+      er.message = "Command unavailable in restricted mode.";
+      return false;
+    }
+
+    // validate the transfer requested and populate dsts & extra
+    std::list<wallet_rpc::transfer_destination> destination;
+    destination.push_back(wallet_rpc::transfer_destination());
+    destination.back().amount = 0;
+    destination.back().address = req.address;
+    if (!validate_transfer(destination, req.payment_id, dsts, extra, er))
+    {
+      return false;
+    }
+
+    try
+    {
+      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions_all(dsts[0].addr, req.mixin, req.unlock_time, req.fee, extra, req.trusted_daemon);
+
+      m_wallet.commit_tx(ptx_vector);
+
+      // populate response with tx hashes
+      for (auto & ptx : ptx_vector)
+      {
+        res.tx_hash_list.push_back(boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(ptx.tx)));
+        if (req.get_tx_keys)
+          res.tx_key_list.push_back(boost::lexical_cast<std::string>(ptx.tx_key));
+      }
+
+      return true;
+    }
+    catch (const tools::error::daemon_busy& e)
+    {
+      er.code = WALLET_RPC_ERROR_CODE_DAEMON_IS_BUSY;
+      er.message = e.what();
+      return false;
+    }
+    catch (const std::exception& e)
+    {
+      er.code = WALLET_RPC_ERROR_CODE_GENERIC_TRANSFER_ERROR;
+      er.message = e.what();
+      return false;
+    }
+    catch (...)
+    {
+      er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+      er.message = "WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR";
+      return false;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool wallet_rpc_server::on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er)
   {
     try
@@ -711,5 +770,164 @@ namespace tools
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er)
+  {
+    if (req.txids.size() != req.notes.size())
+    {
+      er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+      er.message = "Different amount of txids and notes";
+      return false;
+    }
+
+    std::list<crypto::hash> txids;
+    std::list<std::string>::const_iterator i = req.txids.begin();
+    while (i != req.txids.end())
+    {
+      cryptonote::blobdata txid_blob;
+      if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob))
+      {
+        er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID;
+        er.message = "TX ID has invalid format";
+        return false;
+      }
+
+      crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data());
+      txids.push_back(txid);
+    }
+
+    std::list<crypto::hash>::const_iterator il = txids.begin();
+    std::list<std::string>::const_iterator in = req.notes.begin();
+    while (il != txids.end())
+    {
+      m_wallet.set_tx_note(*il++, *in++);
+    }
+
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er)
+  {
+    res.notes.clear();
+
+    std::list<crypto::hash> txids;
+    std::list<std::string>::const_iterator i = req.txids.begin();
+    while (i != req.txids.end())
+    {
+      cryptonote::blobdata txid_blob;
+      if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob))
+      {
+        er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID;
+        er.message = "TX ID has invalid format";
+        return false;
+      }
+
+      crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data());
+      txids.push_back(txid);
+    }
+
+    std::list<crypto::hash>::const_iterator il = txids.begin();
+    while (il != txids.end())
+    {
+      res.notes.push_back(m_wallet.get_tx_note(*il++));
+    }
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+  bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er)
+  {
+    if (m_wallet.restricted())
+    {
+      er.code = WALLET_RPC_ERROR_CODE_DENIED;
+      er.message = "Command unavailable in restricted mode.";
+      return false;
+    }
+
+    uint64_t min_height = 0, max_height = (uint64_t)-1;
+    if (req.filter_by_height)
+    {
+      min_height = req.min_height;
+      max_height = req.max_height;
+    }
+
+    if (req.in)
+    {
+      std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments;
+      m_wallet.get_payments(payments, min_height, max_height);
+      for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) {
+        res.in.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry());
+        wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = res.in.back();
+        const tools::wallet2::payment_details &pd = i->second;
+
+        entry.txid = string_tools::pod_to_hex(pd.m_tx_hash);
+        entry.payment_id = string_tools::pod_to_hex(i->first);
+        if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos)
+          entry.payment_id = entry.payment_id.substr(0,16);
+        entry.height = pd.m_block_height;
+        entry.timestamp = pd.m_timestamp;
+        entry.amount = pd.m_amount;
+        entry.fee = 0; // TODO
+        entry.note = m_wallet.get_tx_note(pd.m_tx_hash);
+      }
+    }
+
+    if (req.out)
+    {
+      std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> payments;
+      m_wallet.get_payments_out(payments, min_height, max_height);
+      for (std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) {
+        res.in.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry());
+        wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = res.in.back();
+        const tools::wallet2::confirmed_transfer_details &pd = i->second;
+
+        entry.txid = string_tools::pod_to_hex(i->first);
+        entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id);
+        if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos)
+          entry.payment_id = entry.payment_id.substr(0,16);
+        entry.height = pd.m_block_height;
+        entry.timestamp = pd.m_timestamp;
+        entry.fee = pd.m_amount_in - pd.m_amount_out;
+        uint64_t change = pd.m_change == (uint64_t)-1 ? 0 : pd.m_change; // change may not be known
+        entry.amount = pd.m_amount_in - change - entry.fee;
+        entry.note = m_wallet.get_tx_note(i->first);
+
+        for (const auto &d: pd.m_dests) {
+          entry.destinations.push_back(wallet_rpc::transfer_destination());
+          wallet_rpc::transfer_destination &td = entry.destinations.back();
+          td.amount = d.amount;
+          td.address = get_account_address_as_str(m_wallet.testnet(), d.addr);
+        }
+      }
+    }
+
+    if (req.pending || req.failed) {
+      std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>> upayments;
+      m_wallet.get_unconfirmed_payments_out(upayments);
+      for (std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>>::const_iterator i = upayments.begin(); i != upayments.end(); ++i) {
+        const tools::wallet2::unconfirmed_transfer_details &pd = i->second;
+        bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed;
+        if (!((req.failed && is_failed) || (!is_failed && req.pending)))
+          continue;
+        std::list<wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry> &entries = is_failed ? res.failed : res.pending;
+        entries.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry());
+        wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = entries.back();
+
+        entry.txid = string_tools::pod_to_hex(i->first);
+        entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id);
+        entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id);
+        if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos)
+          entry.payment_id = entry.payment_id.substr(0,16);
+        entry.height = 0;
+        entry.timestamp = pd.m_timestamp;
+        uint64_t amount = 0;
+        cryptonote::get_inputs_money_amount(pd.m_tx, amount);
+        entry.fee = amount - get_outs_money_amount(pd.m_tx);
+        entry.amount = amount - pd.m_change - entry.fee;
+        entry.note = m_wallet.get_tx_note(i->first);
+      }
+    }
+
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
 }
 

src/wallet/wallet_rpc_server.h

@@ -67,6 +67,7 @@ namespace tools
         MAP_JON_RPC_WE("transfer",           on_transfer,           wallet_rpc::COMMAND_RPC_TRANSFER)
         MAP_JON_RPC_WE("transfer_split",     on_transfer_split,     wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT)
         MAP_JON_RPC_WE("sweep_dust",         on_sweep_dust,         wallet_rpc::COMMAND_RPC_SWEEP_DUST)
+        MAP_JON_RPC_WE("sweep_all",          on_sweep_all,          wallet_rpc::COMMAND_RPC_SWEEP_ALL)
         MAP_JON_RPC_WE("store",              on_store,              wallet_rpc::COMMAND_RPC_STORE)
         MAP_JON_RPC_WE("get_payments",       on_get_payments,       wallet_rpc::COMMAND_RPC_GET_PAYMENTS)
         MAP_JON_RPC_WE("get_bulk_payments",  on_get_bulk_payments,  wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS)
@@ -76,6 +77,9 @@ namespace tools
         MAP_JON_RPC_WE("split_integrated_address", on_split_integrated_address, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS)
         MAP_JON_RPC_WE("stop_wallet",        on_stop_wallet,        wallet_rpc::COMMAND_RPC_STOP_WALLET)
         MAP_JON_RPC_WE("rescan_blockchain",  on_rescan_blockchain,  wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN)
+        MAP_JON_RPC_WE("set_tx_notes",       on_set_tx_notes,       wallet_rpc::COMMAND_RPC_SET_TX_NOTES)
+        MAP_JON_RPC_WE("get_tx_notes",       on_get_tx_notes,       wallet_rpc::COMMAND_RPC_GET_TX_NOTES)
+        MAP_JON_RPC_WE("get_transfers",      on_get_transfers,      wallet_rpc::COMMAND_RPC_GET_TRANSFERS)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -87,6 +91,7 @@ namespace tools
       bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er);
       bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er);
       bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er);
+      bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er);
       bool on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er);
       bool on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er);
       bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er);
@@ -95,6 +100,9 @@ namespace tools
       bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er);
       bool on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er);
       bool on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er);
+      bool on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er);
+      bool on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er);
+      bool on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er);
 
       bool handle_command_line(const boost::program_options::variables_map& vm);
 

src/wallet/wallet_rpc_server_commands_defs.h

@@ -202,6 +202,41 @@ namespace wallet_rpc
     };
   };
 
+  struct COMMAND_RPC_SWEEP_ALL
+  {
+    struct request
+    {
+      std::string address;
+      uint64_t fee;
+      uint64_t mixin;
+      uint64_t unlock_time;
+      std::string payment_id;
+      bool get_tx_keys;
+      bool trusted_daemon;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(address)
+        KV_SERIALIZE(fee)
+        KV_SERIALIZE(mixin)
+        KV_SERIALIZE(unlock_time)
+        KV_SERIALIZE(payment_id)
+        KV_SERIALIZE(get_tx_keys)
+        KV_SERIALIZE(trusted_daemon)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::list<std::string> tx_hash_list;
+      std::list<std::string> tx_key_list;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(tx_hash_list)
+        KV_SERIALIZE(tx_key_list)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
   struct COMMAND_RPC_STORE
   {
     struct request
@@ -412,6 +447,110 @@ namespace wallet_rpc
     };
   };
 
+  struct COMMAND_RPC_SET_TX_NOTES
+  {
+    struct request
+    {
+      std::list<std::string> txids;
+      std::list<std::string> notes;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(txids)
+        KV_SERIALIZE(notes)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
+  struct COMMAND_RPC_GET_TX_NOTES
+  {
+    struct request
+    {
+      std::list<std::string> txids;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(txids)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::list<std::string> notes;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(notes)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
+  struct COMMAND_RPC_GET_TRANSFERS
+  {
+    struct request
+    {
+      bool in;
+      bool out;
+      bool pending;
+      bool failed;
+
+      bool filter_by_height;
+      uint64_t min_height;
+      uint64_t max_height;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(in);
+        KV_SERIALIZE(out);
+        KV_SERIALIZE(pending);
+        KV_SERIALIZE(failed);
+        KV_SERIALIZE(filter_by_height);
+        KV_SERIALIZE(min_height);
+        KV_SERIALIZE(max_height);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct entry
+    {
+      std::string txid;
+      std::string payment_id;
+      uint64_t height;
+      uint64_t timestamp;
+      uint64_t amount;
+      uint64_t fee;
+      std::string note;
+      std::list<transfer_destination> destinations;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(txid);
+        KV_SERIALIZE(payment_id);
+        KV_SERIALIZE(height);
+        KV_SERIALIZE(timestamp);
+        KV_SERIALIZE(amount);
+        KV_SERIALIZE(fee);
+        KV_SERIALIZE(note);
+        KV_SERIALIZE(destinations);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::list<entry> in;
+      std::list<entry> out;
+      std::list<entry> pending;
+      std::list<entry> failed;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(in);
+        KV_SERIALIZE(out);
+        KV_SERIALIZE(pending);
+        KV_SERIALIZE(failed);
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
 }
 }
 

src/wallet/wallet_rpc_server_error_codes.h

@@ -38,3 +38,4 @@
 #define WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID        -5
 #define WALLET_RPC_ERROR_CODE_TRANSFER_TYPE           -6
 #define WALLET_RPC_ERROR_CODE_DENIED                  -7
+#define WALLET_RPC_ERROR_CODE_WRONG_TXID              -8

tests/functional_tests/transactions_flow_test.cpp

@@ -85,7 +85,8 @@ bool do_send_money(tools::wallet2& w1, tools::wallet2& w2, size_t mix_in_factor,
   try
   {
     tools::wallet2::pending_tx ptx;
-    w1.transfer(dsts, mix_in_factor, 0, TEST_FEE, std::vector<uint8_t>(), tools::detail::null_split_strategy, tools::tx_dust_policy(TEST_DUST_THRESHOLD), tx, ptx, true);
+    std::vector<size_t> indices = w1.select_available_outputs([](const tools::wallet2::transfer_details&) { return true; });
+    w1.transfer(dsts, mix_in_factor, indices, 0, TEST_FEE, std::vector<uint8_t>(), tools::detail::null_split_strategy, tools::tx_dust_policy(TEST_DUST_THRESHOLD), tx, ptx, true);
     w1.commit_tx(ptx);
     return true;
   }