diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index a33c82fe6..2dbea4cc4 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2018 by Retroshare Team * + * Copyright (C) 2018 Retroshare Team * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -58,7 +60,8 @@ RsFileTree::fromBase64(const std::string& base64) if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec); RsGenericSerializer::SerializeContext ctx( - mem.data(), static_cast(mem.size()) ); + mem.data(), static_cast(mem.size()), + SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ) ); std::unique_ptr ft(new RsFileTree); ft->serial_process( RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); @@ -70,6 +73,7 @@ RsFileTree::fromBase64(const std::string& base64) std::string RsFileTree::toBase64() const { RsGenericSerializer::SerializeContext ctx; + ctx.mFlags = SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ); RsFileTree* ncThis = const_cast(this); ncThis->serial_process( RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 7842e03cc..a923475d5 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -481,6 +481,7 @@ HEADERS += util/folderiterator.h \ util/radix32.h \ util/radix64.h \ util/rsbase64.h \ + util/rsendian.h \ util/rsinitedptr.h \ util/rsprint.h \ util/rsstring.h \ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 35d13c729..ae5c00dea 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -309,10 +309,6 @@ public: RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) { - /* TODO: most of the time handles are smaller then 64bit use VLQ for - * binary serialization of those - * https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding - */ RS_SERIAL_PROCESS(mFiles); RS_SERIAL_PROCESS(mDirs); RS_SERIAL_PROCESS(mTotalFiles); diff --git a/libretroshare/src/retroshare/rsflags.h b/libretroshare/src/retroshare/rsflags.h index 4222855fa..37320f08b 100644 --- a/libretroshare/src/retroshare/rsflags.h +++ b/libretroshare/src/retroshare/rsflags.h @@ -24,6 +24,7 @@ #include #include +#include /** Check if given type is a scoped enum */ template @@ -128,13 +129,7 @@ typename std::enable_if::enabled, std::ostream>::type& operator <<(std::ostream& stream, EFT flags) { using u_t = typename std::underlying_type::type; - - for(int i = sizeof(u_t); i>=0; --i) - { - stream << (flags & ( 1 << i ) ? "1" : "0"); - if( i % 8 == 0 ) stream << " "; - } - return stream; + return stream << std::bitset(static_cast(flags)); } #include @@ -170,6 +165,7 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 inline t_RsFlags32() : _bits(0) {} inline explicit t_RsFlags32(uint32_t N) : _bits(N) {} // allows initialization from a set of uint32_t + inline t_RsFlags32 operator| (const t_RsFlags32& f) const { return t_RsFlags32(_bits | f._bits) ; } inline t_RsFlags32 operator^ (const t_RsFlags32& f) const { return t_RsFlags32(_bits ^ f._bits) ; } inline t_RsFlags32 operator* (const t_RsFlags32& f) const { return t_RsFlags32(_bits & f._bits) ; } @@ -187,6 +183,19 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 //inline explicit operator bool() const { return _bits>0; } inline uint32_t toUInt32() const { return _bits ; } + /// Easier porting to new flag system + template inline + typename std::enable_if<(Rs__BitFlagsOps::enabled && + sizeof(EFT) >= sizeof(uint32_t) ), EFT>::type + toEFT() { return static_cast(_bits); } + + /// Easier porting to new flag system + template + static inline typename std::enable_if< + Rs__BitFlagsOps::enabled && + sizeof(EFT) <= sizeof(uint32_t), t_RsFlags32>::type + fromEFT(EFT e) { return t_RsFlags32(static_cast(e)); } + void clear() { _bits = 0 ; } friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32& f) // friendly print with 0 and I @@ -199,8 +208,10 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 } return o ; } - private: - uint32_t _bits ; + +private: + friend struct RsTypeSerializer; + uint32_t _bits; }; #define FLAGS_TAG_TRANSFER_REQS 0x4228af @@ -230,7 +241,7 @@ typedef t_RsFlags32 ServicePermissionFlags ; // typedef t_RsFlags32 ChatLobbyFlags ; -// Flags for serializer -// +/// @deprecated Flags for serializer +RS_DEPRECATED_FOR(RsSerializationFlags) typedef t_RsFlags32 SerializationFlags ; diff --git a/libretroshare/src/serialiser/rsserializer.h b/libretroshare/src/serialiser/rsserializer.h index e9b60a0aa..60f4a21d8 100644 --- a/libretroshare/src/serialiser/rsserializer.h +++ b/libretroshare/src/serialiser/rsserializer.h @@ -192,6 +192,39 @@ class RsRawSerialiser: public RsSerialType virtual RsItem * deserialise(void *data, uint32_t *size); }; +/** These are convenience flags to be used by the items when processing the + * data. The names of the flags are not very important. What matters is that + * the serial_process() method of each item correctly deals with the data + * when it sees the flags, if the serialiser sets them. + * By default the flags are not set and shouldn't be handled. + * When deriving a new serializer, the user can set his own flags, using + * compatible values + */ +enum class RsSerializationFlags +{ + NONE = 0, + CONFIG = 1, + SIGNATURE = 2, + SKIP_HEADER = 4, + + /** Used for JSON deserialization in JSON API, it causes the deserialization + * to continue even if some field is missing (or incorrect), this way the + * API is more user friendly as some methods need just part of the structs + * they take as parameters. */ + YIELDING = 8, + + /** When set integers typer are serialized/deserialized in Variable Length + * Quantity mode + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * This type of encoding is efficent when absoulte value is usually much + * smaller then the maximum representable with the original type. + * This encoding is also capable of representing big values at expences of a + * one more byte used. + */ + INTEGER_VLQ = 16 +}; +RS_REGISTER_ENUM_FLAGS_TYPE(RsSerializationFlags); + /// Top class for all services and config serializers. struct RsGenericSerializer : RsSerialType { @@ -200,7 +233,7 @@ struct RsGenericSerializer : RsSerialType SIZE_ESTIMATE = 0x01, SERIALIZE = 0x02, DESERIALIZE = 0x03, - PRINT = 0x04, + PRINT = 0x04, /// @deprecated use rsdebug.h << operator instead TO_JSON, FROM_JSON } SerializeJob; @@ -227,7 +260,7 @@ struct RsGenericSerializer : RsSerialType SerializationFlags flags, RsJson::AllocatorType* allocator = nullptr) : mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format), - mFlags(flags), mJson(rapidjson::kObjectType, allocator) {} + mFlags{flags}, mJson(rapidjson::kObjectType, allocator) {} unsigned char *mData; uint32_t mSize; diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 1b086576a..d57f3d0d4 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2017 Cyril Soler * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -25,7 +26,7 @@ #include "serialiser/rsbaseserial.h" #include "serialiser/rstlvkeys.h" #include "serialiser/rsserializable.h" -#include "util/radix64.h" +#include "util/rsbase64.h" #include "util/rsprint.h" #include "util/rstime.h" @@ -34,8 +35,6 @@ #include // for typeid #include -static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB. - #ifdef RSSERIAL_DEBUG # define SAFE_GET_JSON_V() \ const char* mName = memberName.c_str(); \ @@ -56,32 +55,10 @@ static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB. rapidjson::Value& v = jDoc[mName] #endif // ifdef RSSERIAL_DEBUG - //============================================================================// // std::string // //============================================================================// -template<> uint32_t RsTypeSerializer::serial_size(const std::string& str) -{ - return getRawStringSize(str); -} -template<> bool RsTypeSerializer::serialize( uint8_t data[], uint32_t size, - uint32_t& offset, - const std::string& str ) -{ - return setRawString(data, size, &offset, str); -} -template<> bool RsTypeSerializer::deserialize( const uint8_t data[], - uint32_t size, uint32_t &offset, - std::string& str ) -{ - return getRawString(data, size, &offset, str); -} -template<> void RsTypeSerializer::print_data( const std::string& n, - const std::string& str ) -{ - std::cerr << " [std::string] " << n << ": " << str << std::endl; -} template<> /*static*/ bool RsTypeSerializer::to_JSON( const std::string& membername, const std::string& member, RsJson& jDoc ) @@ -112,135 +89,11 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return ret; } + //============================================================================// -// Integer types // +// Integral types // //============================================================================// -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const bool& member) -{ - return setRawUInt8(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, const int32_t& /*member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return false; -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint8_t& member) -{ - return setRawUInt8(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint16_t& member) -{ - return setRawUInt16(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint32_t& member) -{ - return setRawUInt32(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint64_t& member) -{ - return setRawUInt64(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const rstime_t& member) -{ - return setRawTimeT(data,size,&offset,member); -} - -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, bool& member) -{ - uint8_t m; - bool ok = getRawUInt8(data,size,&offset,&m); - member = m; - return ok; -} -template<> bool RsTypeSerializer::deserialize(const uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, int32_t& /*member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return false; -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint8_t& member) -{ - return getRawUInt8(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t& member) -{ - return getRawUInt16(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint32_t& member) -{ - return getRawUInt32(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint64_t& member) -{ - return getRawUInt64(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, rstime_t& member) -{ - return getRawTimeT(data,size,&offset,member); -} - -template<> uint32_t RsTypeSerializer::serial_size(const bool& /* member*/) -{ - return 1; -} -template<> uint32_t RsTypeSerializer::serial_size(const int32_t& /* member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return 0; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint8_t& /* member*/) -{ - return 1; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint16_t& /* member*/) -{ - return 2; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint32_t& /* member*/) -{ - return 4; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint64_t& /* member*/) -{ - return 8; -} -template<> uint32_t RsTypeSerializer::serial_size(const rstime_t& /* member*/) -{ - return 8; -} - -template<> void RsTypeSerializer::print_data(const std::string& n, const bool & V) -{ - std::cerr << " [bool ] " << n << ": " << V << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const int32_t& V) -{ - std::cerr << " [int32_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint8_t & V) -{ - std::cerr << " [uint8_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint16_t& V) -{ - std::cerr << " [uint16_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint32_t& V) -{ - std::cerr << " [uint32_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint64_t& V) -{ - std::cerr << " [uint64_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const rstime_t& V) -{ - std::cerr << " [rstime_t ] " << n << ": " << V << " (" << time(NULL)-V << " secs ago)" << std::endl; -} - #define SIMPLE_TO_JSON_DEF(T) \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ const T& member, RsJson& jDoc ) \ @@ -268,28 +121,45 @@ SIMPLE_TO_JSON_DEF(uint32_t) /** Be very careful in changing this constant as it would break 64 bit integers * members JSON string representation retrocompatibility */ -static constexpr char strReprSuffix[] = "_sixtyfour_str"; +static constexpr char strReprKey[] = "xstr64"; -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we add - * to JSON also the string representation for this types as a workaround for the - * sake of JavaScript clients @see https://stackoverflow.com/a/34989371 +/** Be very careful in changing this constant as it would break 64 bit integers + * members JSON string representation retrocompatibility */ +static constexpr char intReprKey[] = "xint64"; + +/** While JSON doesn't have problems representing 64 bits integers JavaScript, + * Dart and other languages represents numbers in a double-like format thus they + * are not capable to handle safely integers outside the range + * [-(2^53 - 1), 2^53 - 1]. + * To overcome this limitation we represent 64 bit integers as an object with + * two keys, one as integer and one as string representation. + * In our case we need to wrap those into an object instead of just adding a key + * with a suffix so support well also containers like std::map or std::vector. + * More discussion on the topic at @see https://stackoverflow.com/a/34989371 */ #define SIXTYFOUR_INTEGERS_TO_JSON_DEF(T) \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ const T& member, RsJson& jDoc ) \ { \ - rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); \ + using namespace rapidjson; \ + Document::AllocatorType& allocator = jDoc.GetAllocator(); \ \ - rapidjson::Value key; \ + Document wrapper(rapidjson::kObjectType, &allocator); \ + \ + Value intKey; \ + intKey.SetString(intReprKey, allocator ); \ + Value intValue(member); \ + wrapper.AddMember(intKey, intValue, allocator); \ + \ + bool ok = to_JSON(strReprKey, std::to_string(member), wrapper); \ + \ + Value key; \ key.SetString( memberName.c_str(), \ static_cast(memberName.length()), \ allocator ); \ - rapidjson::Value value(member); \ - jDoc.AddMember(key, value, allocator); \ + jDoc.AddMember(key, wrapper, allocator); \ \ - return to_JSON(memberName + strReprSuffix, std::to_string(member), jDoc); \ + return ok; \ } SIXTYFOUR_INTEGERS_TO_JSON_DEF(int64_t); @@ -345,100 +215,71 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return ret; } -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look - * for the string representation in the JSON for this types as a workaround for - * the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 - */ -template<> /*static*/ -bool RsTypeSerializer::from_JSON( - const std::string& memberName, int64_t& member, RsJson& jDoc ) -{ - const char* mName = memberName.c_str(); - if(jDoc.HasMember(mName)) - { - rapidjson::Value& v = jDoc[mName]; - if(v.IsInt64()) - { - member = v.GetInt64(); - return true; - } - } - - Dbg4() << __PRETTY_FUNCTION__ << " int64_t " << memberName << " not found " - << "in JSON then attempt to look for string representation" - << std::endl; - - const std::string str_key = memberName + strReprSuffix; - std::string str_value; - if(from_JSON(str_key, str_value, jDoc)) - { - try { member = std::stoll(str_value); } - catch (...) - { - RsErr() << __PRETTY_FUNCTION__ << " cannot convert " - << str_value << " to int64_t" << std::endl; - return false; - } - - return true; - } - - Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its " - << "string representation " << str_key << " has been found " - << "in JSON" << std::endl; - - return false; -} - -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look - * for the string representation in the JSON for this types as a workaround for - * the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 - */ -template<> /*static*/ -bool RsTypeSerializer::from_JSON( - const std::string& memberName, uint64_t& member, RsJson& jDoc ) -{ - const char* mName = memberName.c_str(); - if(jDoc.HasMember(mName)) - { - rapidjson::Value& v = jDoc[mName]; - if(v.IsUint64()) - { - member = v.GetUint64(); - return true; - } - } - - Dbg4() << __PRETTY_FUNCTION__ << " uint64_t " << memberName << " not found " - << "in JSON then attempt to look for string representation" - << std::endl; - - const std::string str_key = memberName + strReprSuffix; - std::string str_value; - if(from_JSON(str_key, str_value, jDoc)) - { - try { member = std::stoull(str_value); } - catch (...) - { - RsErr() << __PRETTY_FUNCTION__ << " cannot convert " - << str_value << " to uint64_t" << std::endl; - return false; - } - - return true; - } - - Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its " - << "string representation " << str_key << " has been found " - << "in JSON" << std::endl; - - return false; +/** inverse of @see SIXTYFOUR_INTEGERS_TO_JSON_DEF */ +#define SIXTYFOUR_INTEGERS_FROM_JSON_DEF(T, PRED, GET, CONV) \ +template<> bool RsTypeSerializer::from_JSON( \ + const std::string& memberName, T& member, RsJson& jDoc ) \ +{ \ + using namespace rapidjson; \ + \ + SAFE_GET_JSON_V(); \ + \ + /* For retro-compatibility take it directly if it is passed as integer */ \ + if(v.PRED()) \ + { \ + member = v.GET(); \ + return true; \ + } \ + \ + ret = ret && v.IsObject(); \ + \ + if(!ret) \ + { \ + Dbg3() << __PRETTY_FUNCTION__ << " " << memberName << " not found" \ + << std::endl; \ + return false; \ + } \ + \ + if(v.HasMember(intReprKey)) \ + { \ + Value& iVal = v[intReprKey]; \ + if(iVal.PRED()) \ + { \ + member = iVal.GET(); \ + return true; \ + } \ + } \ + \ + Dbg4() << __PRETTY_FUNCTION__ << " integer representation of " << memberName \ + << " not found in JSON then attempt to look for string representation" \ + << std::endl; \ + \ + \ + if(v.HasMember(strReprKey)) \ + { \ + Value& sVal = v[strReprKey]; \ + if(sVal.IsString()) \ + { \ + try { member = CONV(sVal.GetString()); } \ + catch (...) \ + { \ + RsErr() << __PRETTY_FUNCTION__ << " cannot convert " \ + << sVal.GetString() << " to integral type" << std::endl; \ + return false; \ + } \ + \ + return true; \ + } \ + } \ + \ + Dbg3() << __PRETTY_FUNCTION__ << " neither integral representation nor " \ + << "string representation found for: " << memberName << std::endl; \ + \ + return false; \ } +SIXTYFOUR_INTEGERS_FROM_JSON_DEF(uint64_t, IsUint64, GetUint64, std::stoull) +SIXTYFOUR_INTEGERS_FROM_JSON_DEF( int64_t, IsInt64, GetInt64, std::stoll) //============================================================================// // Floats // @@ -662,113 +503,170 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/, // Binary blocks // //============================================================================// -template<> uint32_t RsTypeSerializer::serial_size(const RsTypeSerializer::TlvMemBlock_proxy& r) { return 4 + r.second ; } - -template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,RsTypeSerializer::TlvMemBlock_proxy& r) +void RsTypeSerializer::RawMemoryWrapper::serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) { - uint32_t saved_offset = offset ; - - bool ok = deserialize(data,size,offset,r.second) ; - - if(r.second == 0) - { - r.first = NULL ; - - if(!ok) - offset = saved_offset ; - - return ok ; - } - if(r.second > MAX_SERIALIZED_CHUNK_SIZE) - { - std::cerr << "(EE) RsTypeSerializer::deserialize(): data chunk has size larger than safety size (" << MAX_SERIALIZED_CHUNK_SIZE << "). Item will be dropped." << std::endl; - offset = saved_offset ; - return false ; - } - - r.first = (uint8_t*)rs_malloc(r.second) ; - - ok = ok && (NULL != r.first); - - memcpy(r.first,&data[offset],r.second) ; - offset += r.second ; - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const RsTypeSerializer::TlvMemBlock_proxy& r) -{ - uint32_t saved_offset = offset ; - - bool ok = serialize(data,size,offset,r.second) ; - - memcpy(&data[offset],r.first,r.second) ; - offset += r.second ; - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> void RsTypeSerializer::print_data(const std::string& n, const RsTypeSerializer::TlvMemBlock_proxy& s) -{ - std::cerr << " [Binary data] " << n << ", length=" << s.second << " data=" << RsUtil::BinToHex((uint8_t*)s.first,std::min(50u,s.second)) << ((s.second>50)?"...":"") << std::endl; -} - -template<> /*static*/ -bool RsTypeSerializer::to_JSON( - const std::string& memberName, - const RsTypeSerializer::TlvMemBlock_proxy& member, RsJson& jDoc ) -{ - rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); - - rapidjson::Value key; - key.SetString( memberName.c_str(), - static_cast(memberName.length()), - allocator ); - - std::string encodedValue; - Radix64::encode( reinterpret_cast(member.first), - static_cast(member.second), encodedValue ); - - rapidjson::Value value; - value.SetString(encodedValue.data(), allocator); - - jDoc.AddMember(key, value, allocator); - - return true; -} - -template<> /*static*/ -bool RsTypeSerializer::from_JSON( const std::string& memberName, - RsTypeSerializer::TlvMemBlock_proxy& member, - RsJson& jDoc) -{ - SAFE_GET_JSON_V(); - ret = ret && v.IsString(); - if(ret) + switch(j) { - std::string encodedValue = v.GetString(); - std::vector decodedValue = Radix64::decode(encodedValue); - member.second = decodedValue.size(); - - if(member.second == 0) + case RsGenericSerializer::SIZE_ESTIMATE: + RS_SERIAL_PROCESS(second); + ctx.mOffset += second; + break; + case RsGenericSerializer::SERIALIZE: + if(!ctx.mOk) break; + if(second > MAX_SERIALIZED_CHUNK_SIZE) { - member.first = nullptr; - return ret; + RsErr() << __PRETTY_FUNCTION__ + << std::errc::message_size << " " + << second << " > " << MAX_SERIALIZED_CHUNK_SIZE + << std::endl; + print_stacktrace(); + break; + } + RS_SERIAL_PROCESS(second); + if(!ctx.mOk) break; + ctx.mOk = ctx.mSize >= ctx.mOffset + second; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + print_stacktrace(); + break; + } + memcpy(ctx.mData + ctx.mOffset, first, second); + ctx.mOffset += second; + break; + case RsGenericSerializer::DESERIALIZE: + { + uint32_t serialSize = 0; + RS_SERIAL_PROCESS(serialSize); + if(!ctx.mOk) break; + ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ + << std::errc::message_size << " " + << serialSize << " > " << MAX_SERIALIZED_CHUNK_SIZE + << std::endl; + clear(); + break; } - member.first = rs_malloc(member.second); - ret = ret && member.first && - memcpy(member.first, decodedValue.data(), member.second); + if(!serialSize) + { + clear(); + break; + } + + ctx.mOk = ctx.mSize >= ctx.mOffset + serialSize; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + print_stacktrace(); + + clear(); + break; + } + + if(serialSize != second) + { + first = reinterpret_cast(realloc(first, serialSize)); + second = static_cast(serialSize); + } + + memcpy(first, ctx.mData + ctx.mOffset, second); + ctx.mOffset += second; + break; + } + case RsGenericSerializer::PRINT: break; + case RsGenericSerializer::TO_JSON: + { + if(!ctx.mOk) break; + std::string encodedValue; + RsBase64::encode(first, second, encodedValue, true, false); + ctx.mJson.SetString( + encodedValue.data(), + static_cast(encodedValue.length()) ); + break; + } + case RsGenericSerializer::FROM_JSON: + { + const bool yelding = !!( RsSerializationFlags::YIELDING & + ctx.mFlags.toEFT() ); + if(!(ctx.mOk || yelding)) + { + clear(); + break; + } + if(!ctx.mJson.IsString()) + { + RsErr() << __PRETTY_FUNCTION__ << " " + << std::errc::invalid_argument << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + if( ctx.mJson.GetStringLength() > + RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) ) + { + RsErr() << __PRETTY_FUNCTION__ << " " + << std::errc::message_size << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + + std::string encodedValue = ctx.mJson.GetString(); + std::vector decoded; + auto ec = RsBase64::decode(encodedValue, decoded); + if(ec) + { + RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + + const auto decodedSize = decoded.size(); + + if(!decodedSize) + { + clear(); + break; + } + + if(decodedSize != second) + { + first = reinterpret_cast(realloc(first, decodedSize)); + second = static_cast(decodedSize); + } + + memcpy(first, decoded.data(), second); + break; + } + default: RsTypeSerializer::fatalUnknownSerialJob(j); } - return ret; } +void RsTypeSerializer::RawMemoryWrapper::clear() +{ + free(first); + first = nullptr; + second = 0; +} + +//============================================================================// +// std::error_condition // +//============================================================================// + void RsTypeSerializer::ErrConditionWrapper::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) @@ -778,12 +676,12 @@ void RsTypeSerializer::ErrConditionWrapper::serial_process( case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough case RsGenericSerializer::DESERIALIZE: // fallthrough case RsGenericSerializer::SERIALIZE: // fallthrough - case RsGenericSerializer::FROM_JSON: + case RsGenericSerializer::FROM_JSON: // [[fallthrough]] + case RsGenericSerializer::PRINT: RsFatal() << __PRETTY_FUNCTION__ << " SerializeJob: " << j << "is not supported on std::error_condition " << std::endl; print_stacktrace(); exit(-2); - case RsGenericSerializer::PRINT: // fallthrough case RsGenericSerializer::TO_JSON: { constexpr RsGenericSerializer::SerializeJob rj = diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 55568632a..a8d61fdeb 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2017 Cyril Soler * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -24,125 +25,159 @@ #include // for typeid #include -#include +#include #include - +#include +#include #include "serialiser/rsserial.h" #include "serialiser/rstlvbase.h" #include "serialiser/rstlvlist.h" - #include "retroshare/rsflags.h" #include "retroshare/rsids.h" - +#include "util/rsendian.h" #include "serialiser/rsserializer.h" #include "serialiser/rsserializable.h" #include "util/rsjson.h" #include "util/rsdebug.h" -/* INTERNAL ONLY helper to avoid copy paste code for std::{vector,list,set} - * Can't use a template function because T is needed for const_cast */ -#define RsTypeSerializer_PRIVATE_TO_JSON_ARRAY() do \ -{ \ - using namespace rapidjson; \ -\ - Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); \ -\ - Value arrKey; arrKey.SetString(memberName.c_str(), allocator); \ -\ - Value arr(kArrayType); \ -\ - for (auto& el : v) \ - { \ - /* Use same allocator to avoid deep copy */\ - RsGenericSerializer::SerializeContext elCtx(\ - nullptr, 0, ctx.mFlags, &allocator );\ -\ - /* If el is const the default serial_process template is matched */ \ - /* also when specialization is necessary so the compilation break */ \ - serial_process(j, elCtx, const_cast(el), memberName); \ -\ - elCtx.mOk = elCtx.mOk && elCtx.mJson.HasMember(arrKey);\ - if(elCtx.mOk) arr.PushBack(elCtx.mJson[arrKey], allocator);\ - else\ - {\ - ctx.mOk = false;\ - break;\ - }\ - }\ -\ - ctx.mJson.AddMember(arrKey, arr, allocator);\ -} while (false) - -/* INTERNAL ONLY helper to avoid copy paste code for std::{vector,list,set} - * Can't use a template function because std::{vector,list,set} has different - * name for insert/push_back function - */ -#define RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(INSERT_FUN) do\ -{\ - using namespace rapidjson;\ -\ - bool ok = ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING;\ - Document& jDoc(ctx.mJson);\ - Document::AllocatorType& allocator = jDoc.GetAllocator();\ -\ - Value arrKey;\ - arrKey.SetString(memberName.c_str(), memberName.length());\ -\ - ok = ok && jDoc.IsObject();\ - ok = ok && jDoc.HasMember(arrKey);\ -\ - if(ok && jDoc[arrKey].IsArray())\ - {\ - for (auto&& arrEl : jDoc[arrKey].GetArray())\ - {\ - Value arrKeyT;\ - arrKeyT.SetString(memberName.c_str(), memberName.length());\ -\ - RsGenericSerializer::SerializeContext elCtx(\ - nullptr, 0, ctx.mFlags, &allocator );\ - elCtx.mJson.AddMember(arrKeyT, arrEl, allocator);\ -\ - T el;\ - serial_process(j, elCtx, el, memberName); \ - ok = ok && elCtx.mOk;\ - ctx.mOk &= ok;\ - if(ok) v.INSERT_FUN(el);\ - else break;\ - }\ - }\ - else\ - {\ - ctx.mOk = false;\ - }\ -\ -} while(false) - - struct RsTypeSerializer { - /** This type should be used to pass a parameter to drive the serialisation - * if needed */ - struct TlvMemBlock_proxy : std::pair + /** Use this wrapper to serialize raw memory chunks + * RsTypeSerializer::RawMemoryWrapper chunkWrapper(chunk_data, chunk_size); + * RsTypeSerializer::serial_process(j, ctx, chunkWrapper, "chunk_data"); + **/ + struct RawMemoryWrapper: std::pair, RsSerializable { - TlvMemBlock_proxy(void*& p, uint32_t& s) : - std::pair(p,s) {} - TlvMemBlock_proxy(uint8_t*& p,uint32_t& s) : - std::pair(*(void**)&p,s) {} + RawMemoryWrapper(uint8_t*& p,uint32_t& s) : + std::pair(p,s) {} + RawMemoryWrapper(void*& p, uint32_t& s) : + std::pair(*(uint8_t**)(&p),s) {} + + /// Maximum supported size 10MB + static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024; + + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override; + private: + void clear(); }; - /// Generic types + /// Most types are not valid sequence containers + template + struct is_sequence_container : std::false_type {}; + + /// Trait to match supported strings types + template + struct is_string : std::is_same, std::string> {}; + + /// Integral types + template + typename std::enable_if::value>::type + static /*void*/ serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + INTT& member, const std::string& member_name ) + { + const bool VLQ_ENCODING = !!( + RsSerializationFlags::INTEGER_VLQ & + ctx.mFlags.toEFT() ); + + switch(j) + { + case RsGenericSerializer::SIZE_ESTIMATE: + if(VLQ_ENCODING) ctx.mOffset += VLQ_size(member); + else ctx.mOffset += sizeof(INTT); + break; + case RsGenericSerializer::SERIALIZE: + { + if(!ctx.mOk) break; + if(VLQ_ENCODING) + ctx.mOk = VLQ_serialize( + ctx.mData, ctx.mSize, ctx.mOffset, member ); + else + { + ctx.mOk = ctx.mSize >= ctx.mOffset + sizeof(INTT); + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot serialise " + << typeid(INTT).name() << " " + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset + << " sizeof(INTT): " << sizeof(INTT) + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + break; + } + INTT netorder_num = rs_endian_fix(member); + memcpy(ctx.mData + ctx.mOffset, &netorder_num, sizeof(INTT)); + ctx.mOffset += sizeof(INTT); + } + break; + } + case RsGenericSerializer::DESERIALIZE: + if(!ctx.mOk) break; + if(VLQ_ENCODING) ctx.mOk = VLQ_deserialize( + ctx.mData, ctx.mSize, ctx.mOffset, member ); + else + { + ctx.mOk = ctx.mSize >= ctx.mOffset + sizeof(INTT); + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot deserialise " + << typeid(INTT).name() << " " + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset + << " sizeof(INTT): " << sizeof(INTT) + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + exit(-1); + break; + } + memcpy(&member, ctx.mData + ctx.mOffset, sizeof(INTT)); + member = rs_endian_fix(member); + ctx.mOffset += sizeof(INTT); + } + break; + case RsGenericSerializer::PRINT: break; + case RsGenericSerializer::TO_JSON: + ctx.mOk = ctx.mOk && to_JSON(member_name, member, ctx.mJson); + break; + case RsGenericSerializer::FROM_JSON: + ctx.mOk &= ( ctx.mOk || + !!( RsSerializationFlags::YIELDING & + ctx.mFlags.toEFT() ) ) + && from_JSON(member_name, member, ctx.mJson); + break; + default: fatalUnknownSerialJob(j); + } + } + +//============================================================================// +// Generic types // +//============================================================================// + template typename std::enable_if< std::is_same::value || !( + std::is_integral::value || std::is_base_of::value || std::is_enum::value || std::is_base_of::value || - std::is_same::value ) >::type + std::is_same::value || + is_sequence_container::value || is_string::value ) >::type static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - T& member, const std::string& member_name ) + T& memberC, const std::string& member_name ) { + // Avoid problems with const sneaking into template paramether + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: @@ -170,12 +205,16 @@ struct RsTypeSerializer } } +//============================================================================// +// Generic types + type_id // +//============================================================================// + /// Generic types + type_id - template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - uint16_t type_id, T& member, - const std::string& member_name ) + template RS_DEPRECATED + static void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + uint16_t type_id, T& member, const std::string& member_name ) { switch(j) { @@ -190,9 +229,7 @@ struct RsTypeSerializer ctx.mOk = ctx.mOk && serialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member); break; - case RsGenericSerializer::PRINT: - print_data(member_name, member); - break; + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: ctx.mOk = ctx.mOk && to_JSON(member_name, type_id, member, ctx.mJson); @@ -206,85 +243,59 @@ struct RsTypeSerializer } } +//============================================================================// +// std::map // +//============================================================================// + /// std::map template static void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - std::map& v, + std::map& member, const std::string& memberName ) { switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: + case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]] + case RsGenericSerializer::SERIALIZE: { - ctx.mOffset += 4; - for(typename std::map::iterator it(v.begin());it!=v.end();++it) + uint32_t mapSize = member.size(); + RS_SERIAL_PROCESS(mapSize); + + for( auto it = member.begin(); + ctx.mOk && it != member.end(); ++it ) { - serial_process( j, ctx, const_cast(it->first), - "map::*it->first" ); - serial_process( j,ctx,const_cast(it->second), - "map::*it->second" ); + RS_SERIAL_PROCESS(const_cast(it->first)); + RS_SERIAL_PROCESS(const_cast(it->second)); } break; } case RsGenericSerializer::DESERIALIZE: { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); + uint32_t mapSize = 0; + RS_SERIAL_PROCESS(mapSize); - for(uint32_t i=0; ifirst"); - serial_process(j, ctx, u, "map::*it->second"); - v[t] = u; - } - break; - } - case RsGenericSerializer::SERIALIZE: - { - uint32_t n=v.size(); - serial_process(j,ctx,n,"temporary size"); - - for(typename std::map::iterator it(v.begin());it!=v.end();++it) - { - serial_process( j, ctx, const_cast(it->first), - "map::*it->first" ); - serial_process( j, ctx, const_cast(it->second), - "map::*it->second" ); - } - break; - } - case RsGenericSerializer::PRINT: - { - if(v.empty()) - std::cerr << " Empty map \"" << memberName << "\"" - << std::endl; - else - std::cerr << " std::map of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; - - for(typename std::map::iterator it(v.begin());it!=v.end();++it) - { - std::cerr << " "; - - serial_process( j, ctx,const_cast(it->first), - "map::*it->first" ); - serial_process( j, ctx, const_cast(it->second), - "map::*it->second" ); + RS_SERIAL_PROCESS(t); + RS_SERIAL_PROCESS(u); + member[t] = u; } break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: { using namespace rapidjson; Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); - Value arrKey; arrKey.SetString(memberName.c_str(), - memberName.length(), allocator); + Value arrKey; arrKey.SetString( + memberName.c_str(), + static_cast(memberName.length()), allocator ); Value arr(kArrayType); - for (auto& kv : v) + for (auto& kv : member) { // Use same allocator to avoid deep copy RsGenericSerializer::SerializeContext kCtx( @@ -318,7 +329,8 @@ struct RsTypeSerializer Document::AllocatorType& allocator = jDoc.GetAllocator(); Value arrKey; - arrKey.SetString(memberName.c_str(), memberName.length()); + arrKey.SetString( memberName.c_str(), + static_cast(memberName.length()) ); ok = ok && jDoc.IsObject(); ok = ok && jDoc.HasMember(arrKey); @@ -350,35 +362,37 @@ struct RsTypeSerializer vCtx.mOk ); ctx.mOk &= ok; - if(ok) v.insert(std::pair(key,value)); + if(ok) member.insert(std::pair(key,value)); else break; } } - else - { - ctx.mOk = false; - } + else ctx.mOk = false; break; } default: fatalUnknownSerialJob(j); } } + +//============================================================================// +// std::pair // +//============================================================================// + /// std::pair template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::pair& p, - const std::string& memberName ) + static void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + std::pair& member, const std::string& memberName ) { switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough - case RsGenericSerializer::DESERIALIZE: // fallthrough - case RsGenericSerializer::SERIALIZE: // fallthrough + case RsGenericSerializer::SIZE_ESTIMATE: // [[fallthrough]] + case RsGenericSerializer::DESERIALIZE: // [[fallthrough]] + case RsGenericSerializer::SERIALIZE: // [[fallthrough]] case RsGenericSerializer::PRINT: - serial_process(j, ctx, p.first, "pair::first"); - serial_process(j, ctx, p.second, "pair::second"); + RS_SERIAL_PROCESS(member.first); + RS_SERIAL_PROCESS(member.second); break; case RsGenericSerializer::TO_JSON: { @@ -389,11 +403,13 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext lCtx( nullptr, 0, ctx.mFlags, &allocator ); - serial_process(j, lCtx, p.first, "first"); - serial_process(j, lCtx, p.second, "second"); + serial_process(j, lCtx, member.first, "first"); + serial_process(j, lCtx, member.second, "second"); rapidjson::Value key; - key.SetString(memberName.c_str(), memberName.length(), allocator); + key.SetString( memberName.c_str(), + static_cast(memberName.length()), + allocator); /* Because the passed allocator is reused it doesn't go out of scope * and there is no need of deep copy and we can take advantage of @@ -415,9 +431,9 @@ struct RsTypeSerializer { if(!yielding) { - std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName - << "\" not found in JSON:" << std::endl - << jDoc << std::endl << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName + << "\" not found in JSON:" << std::endl + << jDoc << std::endl << std::endl; print_stacktrace(); } ctx.mOk = false; @@ -429,8 +445,8 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext lCtx(nullptr, 0, ctx.mFlags); lCtx.mJson.SetObject() = v; // Beware of move semantic!! - serial_process(j, lCtx, p.first, "first"); - serial_process(j, lCtx, p.second, "second"); + serial_process(j, lCtx, member.first, "first"); + serial_process(j, lCtx, member.second, "second"); ctx.mOk &= lCtx.mOk; break; @@ -439,176 +455,225 @@ struct RsTypeSerializer } } - /// std::vector +//============================================================================// +// Sequence containers // +//============================================================================// + + /** std::list is supported */ template + struct is_sequence_container>: std::true_type {}; + + /** std::set is supported */ template + struct is_sequence_container>: std::true_type {}; + + /** std::vector is supported */ template + struct is_sequence_container>: std::true_type {}; + + + /// STL compatible sequence containers std::list, std::set, std::vector... template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::vector& v, - const std::string& memberName ) + typename std::enable_if::value>::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& member, const std::string& memberName ) { + using el_t = typename T::value_type; + switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: + case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]] + case RsGenericSerializer::SERIALIZE: { - ctx.mOffset += 4; - for(uint32_t i=0;i ctx.mSize - ctx.mOffset) { - std::cerr << " " ; - serial_process(j,ctx,v[i],memberName); + ctx.mOk = false; + RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " + << "sequence with apparently malformed elements count." + << " elCount: " << elCount + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset << " " + << std::errc::argument_out_of_domain + << std::endl; + print_stacktrace(); + break; + } + + for(uint32_t i=0; ctx.mOk && i < elCount; ++i ) + { + el_t elem; + RS_SERIAL_PROCESS(elem); + member.insert(member.end(), elem); } break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); + { + using namespace rapidjson; + + Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); + + Value arrKey; arrKey.SetString(memberName.c_str(), allocator); + Value arr(kArrayType); + + for(auto& const_el : member) + { + auto el = const_cast(const_el); + + /* Use same allocator to avoid deep copy */ + RsGenericSerializer::SerializeContext elCtx( + nullptr, 0, ctx.mFlags, &allocator ); + serial_process(j, elCtx, el, memberName); + + elCtx.mOk = elCtx.mOk && elCtx.mJson.HasMember(arrKey); + if(elCtx.mOk) arr.PushBack(elCtx.mJson[arrKey], allocator); + else + { + ctx.mOk = false; + break; + } + } + + ctx.mJson.AddMember(arrKey, arr, allocator); break; + } case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(push_back); + { + using namespace rapidjson; + + bool ok = ctx.mOk || ctx.mFlags & + RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + Document& jDoc(ctx.mJson); + Document::AllocatorType& allocator = jDoc.GetAllocator(); + + Value arrKey; + arrKey.SetString( memberName.c_str(), + static_cast(memberName.length()) ); + + ok = ok && jDoc.IsObject(); + ok = ok && jDoc.HasMember(arrKey); + + if(ok && jDoc[arrKey].IsArray()) + { + for (auto&& arrEl : jDoc[arrKey].GetArray()) + { + Value arrKeyT; + arrKeyT.SetString( + memberName.c_str(), + static_cast(memberName.length()) ); + + RsGenericSerializer::SerializeContext elCtx( + nullptr, 0, ctx.mFlags, &allocator ); + elCtx.mJson.AddMember(arrKeyT, arrEl, allocator); + + el_t el; + serial_process(j, elCtx, el, ""); + ok = ok && elCtx.mOk; + ctx.mOk &= ok; + if(ok) member.insert(member.end(), el); + else break; + } + } + else ctx.mOk = false; break; + } default: fatalUnknownSerialJob(j); } } - /// std::set - template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::set& v, const std::string& memberName ) - { - switch(j) - { - case RsGenericSerializer::SIZE_ESTIMATE: - { - ctx.mOffset += 4; - for(typename std::set::iterator it(v.begin());it!=v.end();++it) - // the const cast here is a hack to avoid serial_process to - // instantiate serialise(const T&) - serial_process(j,ctx,const_cast(*it) ,memberName); - break; - } - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); - for(uint32_t i=0; i::iterator it(v.begin());it!=v.end();++it) - // the const cast here is a hack to avoid serial_process to - // instantiate serialise(const T&) - serial_process(j,ctx,const_cast(*it) ,memberName); - break; - } - case RsGenericSerializer::PRINT: - { - if(v.empty()) - std::cerr << " Empty set \"" << memberName << "\"" - << std::endl; - else - std::cerr << " Set of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; - break; - } - case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); - break; - case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(insert); - break; - default: fatalUnknownSerialJob(j); - } - } +//============================================================================// +// Strings // +//============================================================================// - /// std::list + /// Strings template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::list& v, - const std::string& memberName ) + typename std::enable_if::value>::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& memberC, + const std::string& memberName ) { + // Avoid problems with const sneaking into template paramether + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: { - ctx.mOffset += 4; - for(typename std::list::iterator it(v.begin());it!=v.end();++it) - serial_process(j,ctx,*it ,memberName); - break; - } - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); - for(uint32_t i=0;i(member.size()); + RS_SERIAL_PROCESS(aSize); + ctx.mOffset += aSize; break; } case RsGenericSerializer::SERIALIZE: { - uint32_t n=v.size(); - serial_process(j,ctx,n,"temporary size"); - for(typename std::list::iterator it(v.begin());it!=v.end();++it) - serial_process(j,ctx,*it ,memberName); + uint32_t len = static_cast(member.length()); + RS_SERIAL_PROCESS(len); + if(len > ctx.mSize - ctx.mOffset) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + ctx.mOk = false; + } + memcpy(ctx.mData + ctx.mOffset, member.c_str(), len); + ctx.mOffset += len; break; } - case RsGenericSerializer::PRINT: + case RsGenericSerializer::DESERIALIZE: { - if(v.empty()) - std::cerr << " Empty list \"" << memberName << "\"" - << std::endl; - else - std::cerr << " List of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; + uint32_t len; + RS_SERIAL_PROCESS(len); + if(!ctx.mOk) break; + if(len > ctx.mSize - ctx.mOffset) + { + ctx.mOk = false; + RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " + << "string with apparently malformed elements count." + << " len: " << len + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset << " " + << std::errc::argument_out_of_domain + << std::endl; + print_stacktrace(); + break; + } + member.resize(len); + memcpy(&member[0], ctx.mData + ctx.mOffset, len); + ctx.mOffset += len; break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); + ctx.mOk = ctx.mOk && to_JSON(memberName, member, ctx.mJson); break; case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(push_back); + { + bool ok = ctx.mOk || !!( ctx.mFlags.toEFT() + & RsSerializationFlags::YIELDING ); + ctx.mOk = ok && from_JSON(memberName, member, ctx.mJson) && ctx.mOk; break; + } default: fatalUnknownSerialJob(j); } } @@ -619,46 +684,10 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext& ctx, t_RsFlags32& v, const std::string& memberName ) - { - switch(j) - { - case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += 4; break; - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - deserialize(ctx.mData,ctx.mSize,ctx.mOffset,n); - v = t_RsFlags32(n); - break; - } - case RsGenericSerializer::SERIALIZE: - { - uint32_t n=v.toUInt32(); - serialize(ctx.mData,ctx.mSize,ctx.mOffset,n); - break; - } - case RsGenericSerializer::PRINT: - std::cerr << " Flags of type " << std::hex << N << " : " - << v.toUInt32() << std::endl; - break; - case RsGenericSerializer::TO_JSON: - ctx.mOk = to_JSON(memberName, v.toUInt32(), ctx.mJson); - break; - case RsGenericSerializer::FROM_JSON: - { - uint32_t f = 0; - ctx.mOk &= - (ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING) - && from_JSON(memberName, f, ctx.mJson) - && (v = t_RsFlags32(f), true); - break; - } - default: fatalUnknownSerialJob(j); - } - } + { serial_process(j, ctx, v._bits, memberName); } /** * @brief serial process enum types - * * On declaration of your member of enum type you must specify the * underlying type otherwise the serialization format may differ in an * uncompatible way depending on the compiler/platform. @@ -685,11 +714,14 @@ struct RsTypeSerializer /// RsSerializable and derivatives template typename std::enable_if::value>::type - static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - T& member, - const std::string& memberName ) + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& memberC, const std::string& memberName ) { + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough @@ -710,11 +742,14 @@ struct RsTypeSerializer member.serial_process(j, lCtx); rapidjson::Value key; - key.SetString(memberName.c_str(), memberName.length(), allocator); + key.SetString( + memberName.c_str(), + static_cast(memberName.length()), + allocator ); - /* Because the passed allocator is reused it doesn't go out of scope and - * there is no need of deep copy and we can take advantage of the much - * faster rapidjson move semantic */ + /* Because the passed allocator is reused it doesn't go out of scope + * and there is no need of deep copy and we can take advantage of + * the much faster rapidjson move semantic */ jDoc.AddMember(key, lCtx.mJson, allocator); ctx.mOk = ctx.mOk && lCtx.mOk; @@ -768,15 +803,13 @@ struct RsTypeSerializer /// RsTlvItem derivatives only template typename std::enable_if< - std::is_base_of::value && !std::is_same::value - >::type - static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - T& member, - const std::string& memberName ) - { - serial_process(j, ctx, static_cast(member), memberName); - } + std::is_base_of::value && + !std::is_same::value >::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& member, const std::string& memberName ) + { serial_process(j, ctx, static_cast(member), memberName); } /** std::error_condition * supports only TO_JSON ErrConditionWrapper::serial_process will explode @@ -786,13 +819,15 @@ struct RsTypeSerializer static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - const T& cond, - const std::string& member_name ) + T& cond, const std::string& member_name ) { ErrConditionWrapper ew(cond); serial_process(j, ctx, ew, member_name); } + RS_DEPRECATED_FOR(RawMemoryWrapper) + typedef RawMemoryWrapper TlvMemBlock_proxy; + protected: //============================================================================// @@ -820,27 +855,32 @@ protected: // Generic types + type_id declarations // //============================================================================// - template static bool serialize( + template RS_DEPRECATED + static bool serialize( uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id, const T& member ); - template static bool deserialize( + template RS_DEPRECATED + static bool deserialize( const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id, T& member ); - template static uint32_t serial_size( - uint16_t type_id,const T& member ); + template RS_DEPRECATED + static uint32_t serial_size(uint16_t type_id,const T& member); - template static void print_data( const std::string& n, - uint16_t type_id,const T& member ); + template RS_DEPRECATED + static void print_data( + const std::string& n, uint16_t type_id,const T& member ); - template static bool to_JSON( const std::string& membername, - uint16_t type_id, - const T& member, RsJson& jVal ); + template RS_DEPRECATED + static bool to_JSON( + const std::string& membername, uint16_t type_id, + const T& member, RsJson& jVal ); - template static bool from_JSON( const std::string& memberName, - uint16_t type_id, - T& member, RsJson& jDoc ); + template RS_DEPRECATED + static bool from_JSON( + const std::string& memberName, uint16_t type_id, + T& member, RsJson& jDoc ); //============================================================================// // t_RsGenericId<...> declarations // @@ -910,13 +950,190 @@ protected: t_RsTlvList& member, RsJson& jDoc ); - [[noreturn]] static void fatalUnknownSerialJob(int j) +//============================================================================// +// Integral types VLQ // +//============================================================================// + + /** + * Size calculation of unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, uint32_t> + VLQ_size(T member) { - RsFatal() << " Unknown serial job: " << j << std::endl; - print_stacktrace(); - exit(EINVAL); + std::decay_t memberBackup = member; + uint32_t ret = 1; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wbool-compare" + while(member > 127) { ++ret; member >>= 7; } +#pragma GCC diagnostic pop + RsDbg() << __PRETTY_FUNCTION__ << " memberBackup: " << memberBackup + << " return: " << ret << std::endl; + return ret; } + /** + * Serialization of unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, bool> + VLQ_serialize( + uint8_t data[], uint32_t size, uint32_t &offset, T member ) + { + std::decay_t backupMember = member; + uint32_t offsetBackup = offset; + + bool ok = true; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wbool-compare" + /* Check with < and not with <= here as we write last byte after + * the loop. Order of && operands very important here! */ + while(member > 127 && (ok = offset < size)) + { + // | 128: Set the next byte flag + data[offset++] = (static_cast(member & 127)) | 128; + // Remove the seven bits we just wrote + member >>= 7; + } +#pragma GCC diagnostic pop + + if(!(ok = ok && offset <= size)) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot serialise " + << typeid(T).name() + << " member " << member + << " size: " << size + << " offset: " << offset + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + return false; + } + + data[offset++] = static_cast(member & 127); + + RsDbg() << __PRETTY_FUNCTION__ << " backupMember: " << backupMember + << " offsetBackup: " << offsetBackup << " offeset: " << offset + << " serialized as: "; + for(; offsetBackup < offset; ++offsetBackup) + std::cerr << " " << std::bitset<8>(data[offsetBackup]); + std::cerr << std::endl; + + return ok; + } + + /** + * Deserialization for unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, bool> + VLQ_deserialize( + const uint8_t data[], uint32_t size, uint32_t& offset, T& member ) + { + uint32_t backupOffset = offset; + + member = 0; + uint32_t offsetBackup = offset; + + /* In a reasonable VLQ coding representing an integer + * could take at maximum sizeof(integer) + 1 space, if it is + * not the case something fishy is happening. */ + for (size_t i = 0; offset < size && i <= sizeof(T); ++i) + { + member |= (data[offset] & 127) << (7 * i); + // If the next-byte flag is not set, ++ is after on purpose + if(!(data[offset++] & 128)) + { + RsDbg() << __PRETTY_FUNCTION__ + << " size: " << size + << " backupOffset " << backupOffset + << " offset: " << offset + << " member " << member << std::endl; + return true; + } + } + + /* If return is not triggered inside the for loop, either the buffer + * ended before we encountered the end of the number, or the number + * is VLQ encoded improperly */ + RsErr() << __PRETTY_FUNCTION__ << std::errc::illegal_byte_sequence + << " offsetBackup: " << offsetBackup + << " offset: " << offset << " bytes: "; + for(; offsetBackup < offset; ++offsetBackup) + std::cerr << " " << std::bitset<8>(data[offsetBackup]); + std::cerr << std::endl; + + print_stacktrace(); + return false; + } + + /** + * Size calculation of signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, uint32_t> + VLQ_size(T member) + { + member = (member << 1) ^ (member >> (sizeof(T)-1)); // ZigZag encoding + return VLQ_size( + static_cast::type>(member)); + } + + /** + * Serialization of signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, bool> + VLQ_serialize( + uint8_t data[], uint32_t size, uint32_t &offset, T member ) + { + member = (member << 1) ^ (member >> (sizeof(T)-1)); // ZigZag encoding + return VLQ_serialize( + data, size, offset, + static_cast::type>(member)); + } + + /** + * Deserialization for signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, bool> + VLQ_deserialize( + const uint8_t data[], uint32_t size, uint32_t& offset, T& member ) + { + using DT = std::decay_t; + typename std::make_unsigned
::type temp = 0; + bool ok = VLQ_deserialize(data, size, offset, temp); + // ZizZag decoding + member = (static_cast
(temp) >> 1) ^ -(static_cast
(temp) & 1); + return ok; + } + +//============================================================================// +// Error Condition Wrapper // +//============================================================================// + struct ErrConditionWrapper : RsSerializable { ErrConditionWrapper(const std::error_condition& ec): mec(ec) {} @@ -931,6 +1148,17 @@ protected: const std::error_condition& mec; }; +//============================================================================// +// Miscellanea // +//============================================================================// + + [[noreturn]] static void fatalUnknownSerialJob(int j) + { + RsFatal() << " Unknown serial job: " << j << std::endl; + print_stacktrace(); + exit(EINVAL); + } + RS_SET_CONTEXT_DEBUG_LEVEL(1) }; @@ -1079,7 +1307,6 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return true; } - //============================================================================// // Not implemented types macros // //============================================================================// diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index d4a22d84d..9478c0f74 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -107,6 +107,10 @@ struct t_RsLogger << std::setfill('0') << std::setw(3) << msec.count() << std::setfill(tFill) << " " << val; } + + /// needed for manipulators and things like std::endl + stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) + { return std::cerr << pf; } }; #endif // def __ANDROID__ diff --git a/libretroshare/src/util/rsendian.h b/libretroshare/src/util/rsendian.h new file mode 100644 index 000000000..925542f0d --- /dev/null +++ b/libretroshare/src/util/rsendian.h @@ -0,0 +1,59 @@ +/******************************************************************************* + * * + * libretroshare base64 encoding utilities * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +/** + * @file + * This file provide convenient integer endiannes conversion utilities. + * Instead of providing them with different names for each type (a la C htonl, + * htons, ntohl, ntohs ), which make them uncomfortable to use, expose a + * templated function `rs_endian_fix` to reorder integers representation byets + * when sending or receiving from the network. */ + +/* enforce LITTLE_ENDIAN on Windows */ +#ifdef WINDOWS_SYS +# define BYTE_ORDER 1234 +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +#else +#include +#endif + +template inline INTTYPE rs_endian_fix(INTTYPE val) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + INTTYPE swapped = 0; + for (size_t i = 0; i < sizeof(INTTYPE); ++i) + swapped |= (val >> (8*(sizeof(INTTYPE)-i-1)) & 0xFF) << (8*i); + return swapped; +#else + return hostI; +#endif +}; + +#ifndef BYTE_ORDER +# error "ENDIAN determination Failed (BYTE_ORDER not defined)" +#endif + +#if !(BYTE_ORDER == BIG_ENDIAN || BYTE_ORDER == LITTLE_ENDIAN) +# error "ENDIAN determination Failed (unkown BYTE_ORDER value)" +#endif