Add byte_stream for zero-copy serialization, and add support in ZMQ-JSON.

This commit is contained in:
Lee Clagett 2020-03-18 23:22:39 +00:00
parent 8185054db7
commit c26c93019a
14 changed files with 806 additions and 171 deletions

View file

@ -31,9 +31,9 @@
#include <boost/utility/string_ref.hpp>
#include <cstring>
#include <rapidjson/document.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
#include "byte_stream.h"
#include "cryptonote_basic/cryptonote_basic.h"
#include "rpc/message_data_structs.h"
#include "cryptonote_protocol/cryptonote_protocol_defs.h"
@ -123,7 +123,7 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest);
// POD to json key
template <class Type>
inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod)
inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod)
{
const auto hex = epee::to_hex::array(pod);
dest.Key(hex.data(), hex.size());
@ -131,7 +131,7 @@ inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Wri
// POD to json value
template <class Type>
inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod)
inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod)
{
const auto hex = epee::to_hex::array(pod);
dest.String(hex.data(), hex.size());
@ -144,16 +144,16 @@ inline typename std::enable_if<is_to_hex<Type>()>::type fromJsonValue(const rapi
json::read_hex(val, epee::as_mut_byte_span(t));
}
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rapidjson::Value& src);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rapidjson::Value& src);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, boost::string_ref i);
inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const std::string& i)
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, boost::string_ref i);
inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::string& i)
{
toJsonValue(dest, boost::string_ref{i});
}
void fromJsonValue(const rapidjson::Value& val, std::string& str);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, bool i);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i);
void fromJsonValue(const rapidjson::Value& val, bool& b);
// integers overloads for toJsonValue are not needed for standard promotions
@ -168,144 +168,144 @@ void fromJsonValue(const rapidjson::Value& val, unsigned short& i);
void fromJsonValue(const rapidjson::Value& val, short& i);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned i);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned i);
void fromJsonValue(const rapidjson::Value& val, unsigned& i);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const int);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const int);
void fromJsonValue(const rapidjson::Value& val, int& i);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long long i);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long long i);
void fromJsonValue(const rapidjson::Value& val, unsigned long long& i);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long long i);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long long i);
void fromJsonValue(const rapidjson::Value& val, long long& i);
inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long i) {
inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long i) {
toJsonValue(dest, static_cast<unsigned long long>(i));
}
void fromJsonValue(const rapidjson::Value& val, unsigned long& i);
inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long i) {
inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long i) {
toJsonValue(dest, static_cast<long long>(i));
}
void fromJsonValue(const rapidjson::Value& val, long& i);
// end integers
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::transaction& tx);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::transaction& tx);
void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block& b);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b);
void fromJsonValue(const rapidjson::Value& val, cryptonote::block& b);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_v& txin);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_v& txin);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_gen& txin);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_gen& txin);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_gen& txin);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_script& txin);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_script& txin);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_script& txin);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_scripthash& txin);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_scripthash& txin);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_scripthash& txin);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_key& txin);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_key& txin);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_target_v& txout);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_target_v& txout);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_script& txout);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_script& txout);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_script& txout);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_scripthash& txout);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_scripthash& txout);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_scripthash& txout);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_key& txout);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_key& txout);
void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_key& txout);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_out& txout);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_out& txout);
void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::connection_info& info);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::connection_info& info);
void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& info);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_blob_entry& tx);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_blob_entry& tx);
void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block_complete_entry& blk);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block_complete_entry& blk);
void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry& blk);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::block_with_transactions& blk);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::block_with_transactions& blk);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::block_with_transactions& blk);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::transaction_info& tx_info);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::transaction_info& tx_info);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::transaction_info& tx_info);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_and_amount_index& out);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_and_amount_index& out);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_and_amount_index& out);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::amount_with_random_outputs& out);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::amount_with_random_outputs& out);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::amount_with_random_outputs& out);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::peer& peer);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::peer& peer);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::tx_in_pool& tx);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::tx_in_pool& tx);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::hard_fork_info& info);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::hard_fork_info& info);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::hard_fork_info& info);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_count& out);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_count& out);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_count& out);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_and_index& out);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_and_index& out);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_and_index& out);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_mask_unlocked& out);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_mask_unlocked& out);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_mask_unlocked& out);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::error& err);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::error& err);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::error& error);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::BlockHeaderResponse& response);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::BlockHeaderResponse& response);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResponse& response);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rctSig& i);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i);
void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::ecdhTuple& tuple);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple);
void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rangeSig& sig);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rangeSig& sig);
void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::Bulletproof& p);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::Bulletproof& p);
void fromJsonValue(const rapidjson::Value& val, rct::Bulletproof& p);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::boroSig& sig);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::boroSig& sig);
void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::mgSig& sig);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig);
void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::DaemonInfo& info);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info);
void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_distribution& dist);
void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist);
void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist);
template <typename Map>
typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map);
typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map);
template <typename Map>
typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type fromJsonValue(const rapidjson::Value& val, Map& map);
template <typename Vec>
typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec);
typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec);
template <typename Vec>
typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec);
@ -315,7 +315,7 @@ typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJson
// unfortunately because of how templates work they have to be here.
template <typename Map>
inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map)
inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map)
{
using key_type = typename Map::key_type;
static_assert(std::is_same<std::string, key_type>() || is_to_hex<key_type>(), "invalid map key type");
@ -351,7 +351,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from
}
template <typename Vec>
inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec)
inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec)
{
dest.StartArray();
for (const auto& t : vec)