Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Many serialization and related stuff improvements

Browse Source 
Fix bug in array-like containers serialization which could cause almost infinite
  loop on malformed input
Implement VLQ integer serialization
Unify sequence containers serialization code
Add support for VLQ serialization also for string size
Use VLQ compression for file links
Add templated function to fix endiannes for all integer types
Use bitset to print flags in binary form
Unify serialization code for integral types
Serialize 64bit integers types to JSON object with both string and integer
  representation, so it is posible to have this representation also for
  containers types like std::vetor or std::map this breaks retrocompatibility
  but is necessary to support clients written in languages which doesn't have
  64 bit integers support such as JavaScript or Dart
This commit is contained in:
Gioacchino Mazzurco 2020-03-16 16:20:06 +01:00
parent d203f31d0c
commit 39bde58c29
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
9 changed files with 985 additions and 752 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libretroshare/src/file_sharing/file_tree.cc
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2018 by Retroshare Team <retroshare.project@gmail.com> * * Copyright (C) 2018 Retroshare Team <contact@retroshare.cc> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -58,7 +60,8 @@ RsFileTree::fromBase64(const std::string& base64)
if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec); if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec);
RsGenericSerializer::SerializeContext ctx( RsGenericSerializer::SerializeContext ctx(
mem.data(), static_cast<uint32_t>(mem.size()) ); mem.data(), static_cast<uint32_t>(mem.size()),
SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ) );
std::unique_ptr<RsFileTree> ft(new RsFileTree); std::unique_ptr<RsFileTree> ft(new RsFileTree);
ft->serial_process( ft->serial_process(
RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); RsGenericSerializer::SerializeJob::DESERIALIZE, ctx);
@ -70,6 +73,7 @@ RsFileTree::fromBase64(const std::string& base64)
std::string RsFileTree::toBase64() const std::string RsFileTree::toBase64() const
{ {
RsGenericSerializer::SerializeContext ctx; RsGenericSerializer::SerializeContext ctx;
ctx.mFlags = SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ);
RsFileTree* ncThis = const_cast<RsFileTree*>(this); RsFileTree* ncThis = const_cast<RsFileTree*>(this);
ncThis->serial_process( ncThis->serial_process(
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx );

1
libretroshare/src/libretroshare.pro
View File

@ -481,6 +481,7 @@ HEADERS += util/folderiterator.h \
util/radix32.h \ util/radix32.h \
util/radix64.h \ util/radix64.h \
util/rsbase64.h \ util/rsbase64.h \
util/rsendian.h \
util/rsinitedptr.h \ util/rsinitedptr.h \
util/rsprint.h \ util/rsprint.h \
util/rsstring.h \ util/rsstring.h \

4
libretroshare/src/retroshare/rsfiles.h
View File

@ -309,10 +309,6 @@ public:
RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) RsGenericSerializer::SerializeContext& ctx )
{ {
/* TODO: most of the time handles are smaller then 64bit use VLQ for
* binary serialization of those
* https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding
*/
RS_SERIAL_PROCESS(mFiles); RS_SERIAL_PROCESS(mFiles);
RS_SERIAL_PROCESS(mDirs); RS_SERIAL_PROCESS(mDirs);
RS_SERIAL_PROCESS(mTotalFiles); RS_SERIAL_PROCESS(mTotalFiles);

29
libretroshare/src/retroshare/rsflags.h
View File

@ -24,6 +24,7 @@
#include <type_traits> #include <type_traits>
#include <ostream> #include <ostream>
#include <bitset>
/** Check if given type is a scoped enum */ /** Check if given type is a scoped enum */
template<typename E> template<typename E>
@ -128,13 +129,7 @@ typename std::enable_if<Rs__BitFlagsOps<EFT>::enabled, std::ostream>::type&
operator <<(std::ostream& stream, EFT flags) operator <<(std::ostream& stream, EFT flags)
{ {
using u_t = typename std::underlying_type<EFT>::type; using u_t = typename std::underlying_type<EFT>::type;
return stream << std::bitset<sizeof(u_t)>(static_cast<u_t>(flags));
for(int i = sizeof(u_t); i>=0; --i)
{
stream << (flags & ( 1 << i ) ? "1" : "0");
if( i % 8 == 0 ) stream << " ";
}
return stream;
} }
#include <cstdint> #include <cstdint>
@ -170,6 +165,7 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
inline t_RsFlags32() : _bits(0) {} inline t_RsFlags32() : _bits(0) {}
inline explicit t_RsFlags32(uint32_t N) : _bits(N) {} // allows initialization from a set of uint32_t inline explicit t_RsFlags32(uint32_t N) : _bits(N) {} // allows initialization from a set of uint32_t
inline t_RsFlags32<n> operator| (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits | f._bits) ; } inline t_RsFlags32<n> operator| (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits | f._bits) ; }
inline t_RsFlags32<n> operator^ (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits ^ f._bits) ; } inline t_RsFlags32<n> operator^ (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits ^ f._bits) ; }
inline t_RsFlags32<n> operator* (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits & f._bits) ; } inline t_RsFlags32<n> operator* (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits & f._bits) ; }
@ -187,6 +183,19 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
//inline explicit operator bool() const { return _bits>0; } //inline explicit operator bool() const { return _bits>0; }
inline uint32_t toUInt32() const { return _bits ; } inline uint32_t toUInt32() const { return _bits ; }
/// Easier porting to new flag system
template<typename EFT> inline
typename std::enable_if<(Rs__BitFlagsOps<EFT>::enabled &&
sizeof(EFT) >= sizeof(uint32_t) ), EFT>::type
toEFT() { return static_cast<EFT>(_bits); }
/// Easier porting to new flag system
template<typename EFT>
static inline typename std::enable_if<
Rs__BitFlagsOps<EFT>::enabled &&
sizeof(EFT) <= sizeof(uint32_t), t_RsFlags32>::type
fromEFT(EFT e) { return t_RsFlags32(static_cast<uint32_t>(e)); }
void clear() { _bits = 0 ; } void clear() { _bits = 0 ; }
friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32<n>& f) // friendly print with 0 and I friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32<n>& f) // friendly print with 0 and I
@ -199,7 +208,9 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
} }
return o ; return o ;
} }
private: private:
friend struct RsTypeSerializer;
uint32_t _bits; uint32_t _bits;
}; };
@ -230,7 +241,7 @@ typedef t_RsFlags32<FLAGS_TAG_SERVICE_PERM > ServicePermissionFlags ;
// //
typedef t_RsFlags32<FLAGS_TAG_SERVICE_CHAT > ChatLobbyFlags ; typedef t_RsFlags32<FLAGS_TAG_SERVICE_CHAT > ChatLobbyFlags ;
// Flags for serializer /// @deprecated Flags for serializer
// RS_DEPRECATED_FOR(RsSerializationFlags)
typedef t_RsFlags32<FLAGS_TAG_SERIALIZER > SerializationFlags ; typedef t_RsFlags32<FLAGS_TAG_SERIALIZER > SerializationFlags ;

37
libretroshare/src/serialiser/rsserializer.h
View File

@ -192,6 +192,39 @@ class RsRawSerialiser: public RsSerialType
virtual RsItem * deserialise(void *data, uint32_t *size); virtual RsItem * deserialise(void *data, uint32_t *size);
}; };
/** These are convenience flags to be used by the items when processing the
* data. The names of the flags are not very important. What matters is that
* the serial_process() method of each item correctly deals with the data
* when it sees the flags, if the serialiser sets them.
* By default the flags are not set and shouldn't be handled.
* When deriving a new serializer, the user can set his own flags, using
* compatible values
*/
enum class RsSerializationFlags
{
NONE = 0,
CONFIG = 1,
SIGNATURE = 2,
SKIP_HEADER = 4,
/** Used for JSON deserialization in JSON API, it causes the deserialization
* to continue even if some field is missing (or incorrect), this way the
* API is more user friendly as some methods need just part of the structs
* they take as parameters. */
YIELDING = 8,
/** When set integers typer are serialized/deserialized in Variable Length
* Quantity mode
* @see https://en.wikipedia.org/wiki/Variable-length_quantity
* This type of encoding is efficent when absoulte value is usually much
* smaller then the maximum representable with the original type.
* This encoding is also capable of representing big values at expences of a
* one more byte used.
*/
INTEGER_VLQ = 16
};
RS_REGISTER_ENUM_FLAGS_TYPE(RsSerializationFlags);
/// Top class for all services and config serializers. /// Top class for all services and config serializers.
struct RsGenericSerializer : RsSerialType struct RsGenericSerializer : RsSerialType
{ {
@ -200,7 +233,7 @@ struct RsGenericSerializer : RsSerialType
SIZE_ESTIMATE = 0x01, SIZE_ESTIMATE = 0x01,
SERIALIZE = 0x02, SERIALIZE = 0x02,
DESERIALIZE = 0x03, DESERIALIZE = 0x03,
PRINT = 0x04, PRINT = 0x04, /// @deprecated use rsdebug.h << operator instead
TO_JSON, TO_JSON,
FROM_JSON FROM_JSON
} SerializeJob; } SerializeJob;
@ -227,7 +260,7 @@ struct RsGenericSerializer : RsSerialType
SerializationFlags flags, SerializationFlags flags,
RsJson::AllocatorType* allocator = nullptr) : RsJson::AllocatorType* allocator = nullptr) :
mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format), mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format),
mFlags(flags), mJson(rapidjson::kObjectType, allocator) {} mFlags{flags}, mJson(rapidjson::kObjectType, allocator) {}
unsigned char *mData; unsigned char *mData;
uint32_t mSize; uint32_t mSize;

572
libretroshare/src/serialiser/rstypeserializer.cc
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2017 Cyril Soler <csoler@users.sourceforge.net> * * Copyright (C) 2017 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -25,7 +26,7 @@
#include "serialiser/rsbaseserial.h" #include "serialiser/rsbaseserial.h"
#include "serialiser/rstlvkeys.h" #include "serialiser/rstlvkeys.h"
#include "serialiser/rsserializable.h" #include "serialiser/rsserializable.h"
#include "util/radix64.h" #include "util/rsbase64.h"
#include "util/rsprint.h" #include "util/rsprint.h"
#include "util/rstime.h" #include "util/rstime.h"
@ -34,8 +35,6 @@
#include <typeinfo> // for typeid #include <typeinfo> // for typeid
#include <rapidjson/prettywriter.h> #include <rapidjson/prettywriter.h>
static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.
#ifdef RSSERIAL_DEBUG #ifdef RSSERIAL_DEBUG
# define SAFE_GET_JSON_V() \ # define SAFE_GET_JSON_V() \
const char* mName = memberName.c_str(); \ const char* mName = memberName.c_str(); \
@ -56,32 +55,10 @@ static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.
rapidjson::Value& v = jDoc[mName] rapidjson::Value& v = jDoc[mName]
#endif // ifdef RSSERIAL_DEBUG #endif // ifdef RSSERIAL_DEBUG
//============================================================================// //============================================================================//
// std::string // // std::string //
//============================================================================// //============================================================================//
template<> uint32_t RsTypeSerializer::serial_size(const std::string& str)
{
return getRawStringSize(str);
}
template<> bool RsTypeSerializer::serialize( uint8_t data[], uint32_t size,
uint32_t& offset,
const std::string& str )
{
return setRawString(data, size, &offset, str);
}
template<> bool RsTypeSerializer::deserialize( const uint8_t data[],
uint32_t size, uint32_t &offset,
std::string& str )
{
return getRawString(data, size, &offset, str);
}
template<> void RsTypeSerializer::print_data( const std::string& n,
const std::string& str )
{
std::cerr << " [std::string] " << n << ": " << str << std::endl;
}
template<> /*static*/ template<> /*static*/
bool RsTypeSerializer::to_JSON( const std::string& membername, bool RsTypeSerializer::to_JSON( const std::string& membername,
const std::string& member, RsJson& jDoc ) const std::string& member, RsJson& jDoc )
@ -112,135 +89,11 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName,
return ret; return ret;
} }
//============================================================================// //============================================================================//
// Integer types // // Integral types //
//============================================================================// //============================================================================//
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const bool& member)
{
return setRawUInt8(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, const int32_t& /*member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return false;
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint8_t& member)
{
return setRawUInt8(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint16_t& member)
{
return setRawUInt16(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint32_t& member)
{
return setRawUInt32(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint64_t& member)
{
return setRawUInt64(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const rstime_t& member)
{
return setRawTimeT(data,size,&offset,member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, bool& member)
{
uint8_t m;
bool ok = getRawUInt8(data,size,&offset,&m);
member = m;
return ok;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, int32_t& /*member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return false;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint8_t& member)
{
return getRawUInt8(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t& member)
{
return getRawUInt16(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint32_t& member)
{
return getRawUInt32(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint64_t& member)
{
return getRawUInt64(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, rstime_t& member)
{
return getRawTimeT(data,size,&offset,member);
}
template<> uint32_t RsTypeSerializer::serial_size(const bool& /* member*/)
{
return 1;
}
template<> uint32_t RsTypeSerializer::serial_size(const int32_t& /* member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return 0;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint8_t& /* member*/)
{
return 1;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint16_t& /* member*/)
{
return 2;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint32_t& /* member*/)
{
return 4;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint64_t& /* member*/)
{
return 8;
}
template<> uint32_t RsTypeSerializer::serial_size(const rstime_t& /* member*/)
{
return 8;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const bool & V)
{
std::cerr << " [bool ] " << n << ": " << V << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const int32_t& V)
{
std::cerr << " [int32_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint8_t & V)
{
std::cerr << " [uint8_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint16_t& V)
{
std::cerr << " [uint16_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint32_t& V)
{
std::cerr << " [uint32_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint64_t& V)
{
std::cerr << " [uint64_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const rstime_t& V)
{
std::cerr << " [rstime_t ] " << n << ": " << V << " (" << time(NULL)-V << " secs ago)" << std::endl;
}
#define SIMPLE_TO_JSON_DEF(T) \ #define SIMPLE_TO_JSON_DEF(T) \
template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \
const T& member, RsJson& jDoc ) \ const T& member, RsJson& jDoc ) \
@ -268,28 +121,45 @@ SIMPLE_TO_JSON_DEF(uint32_t)
/** Be very careful in changing this constant as it would break 64 bit integers /** Be very careful in changing this constant as it would break 64 bit integers
* members JSON string representation retrocompatibility */ * members JSON string representation retrocompatibility */
static constexpr char strReprSuffix[] = "_sixtyfour_str"; static constexpr char strReprKey[] = "xstr64";
/** While JSON doesn't have problems representing 64 bit integers JavaScript /** Be very careful in changing this constant as it would break 64 bit integers
* standard represents numbers in a double-like format thus it is not capable to * members JSON string representation retrocompatibility */
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we add static constexpr char intReprKey[] = "xint64";
* to JSON also the string representation for this types as a workaround for the
* sake of JavaScript clients @see https://stackoverflow.com/a/34989371 /** While JSON doesn't have problems representing 64 bits integers JavaScript,
* Dart and other languages represents numbers in a double-like format thus they
* are not capable to handle safely integers outside the range
* [-(2^53 - 1), 2^53 - 1].
* To overcome this limitation we represent 64 bit integers as an object with
* two keys, one as integer and one as string representation.
* In our case we need to wrap those into an object instead of just adding a key
* with a suffix so support well also containers like std::map or std::vector.
* More discussion on the topic at @see https://stackoverflow.com/a/34989371
*/ */
#define SIXTYFOUR_INTEGERS_TO_JSON_DEF(T) \ #define SIXTYFOUR_INTEGERS_TO_JSON_DEF(T) \
template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \
const T& member, RsJson& jDoc ) \ const T& member, RsJson& jDoc ) \
{ \ { \
rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); \ using namespace rapidjson; \
Document::AllocatorType& allocator = jDoc.GetAllocator(); \
\ \
rapidjson::Value key; \ Document wrapper(rapidjson::kObjectType, &allocator); \
\
Value intKey; \
intKey.SetString(intReprKey, allocator ); \
Value intValue(member); \
wrapper.AddMember(intKey, intValue, allocator); \
\
bool ok = to_JSON(strReprKey, std::to_string(member), wrapper); \
\
Value key; \
key.SetString( memberName.c_str(), \ key.SetString( memberName.c_str(), \
static_cast<rapidjson::SizeType>(memberName.length()), \ static_cast<rapidjson::SizeType>(memberName.length()), \
allocator ); \ allocator ); \
rapidjson::Value value(member); \ jDoc.AddMember(key, wrapper, allocator); \
jDoc.AddMember(key, value, allocator); \
\ \
return to_JSON(memberName + strReprSuffix, std::to_string(member), jDoc); \ return ok; \
} }
SIXTYFOUR_INTEGERS_TO_JSON_DEF(int64_t); SIXTYFOUR_INTEGERS_TO_JSON_DEF(int64_t);
@ -345,100 +215,71 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName,
return ret; return ret;
} }
/** While JSON doesn't have problems representing 64 bit integers JavaScript /** inverse of @see SIXTYFOUR_INTEGERS_TO_JSON_DEF */
* standard represents numbers in a double-like format thus it is not capable to #define SIXTYFOUR_INTEGERS_FROM_JSON_DEF(T, PRED, GET, CONV) \
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look template<> bool RsTypeSerializer::from_JSON( \
* for the string representation in the JSON for this types as a workaround for const std::string& memberName, T& member, RsJson& jDoc ) \
* the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 { \
*/ using namespace rapidjson; \
template<> /*static*/ \
bool RsTypeSerializer::from_JSON( SAFE_GET_JSON_V(); \
const std::string& memberName, int64_t& member, RsJson& jDoc ) \
{ /* For retro-compatibility take it directly if it is passed as integer */ \
const char* mName = memberName.c_str(); if(v.PRED()) \
if(jDoc.HasMember(mName)) { \
{ member = v.GET(); \
rapidjson::Value& v = jDoc[mName]; return true; \
if(v.IsInt64()) } \
{ \
member = v.GetInt64(); ret = ret && v.IsObject(); \
return true; \
} if(!ret) \
} { \
Dbg3() << __PRETTY_FUNCTION__ << " " << memberName << " not found" \
Dbg4() << __PRETTY_FUNCTION__ << " int64_t " << memberName << " not found " << std::endl; \
<< "in JSON then attempt to look for string representation" return false; \
<< std::endl; } \
\
const std::string str_key = memberName + strReprSuffix; if(v.HasMember(intReprKey)) \
std::string str_value; { \
if(from_JSON(str_key, str_value, jDoc)) Value& iVal = v[intReprKey]; \
{ if(iVal.PRED()) \
try { member = std::stoll(str_value); } { \
catch (...) member = iVal.GET(); \
{ return true; \
RsErr() << __PRETTY_FUNCTION__ << " cannot convert " } \
<< str_value << " to int64_t" << std::endl; } \
return false; \
} Dbg4() << __PRETTY_FUNCTION__ << " integer representation of " << memberName \
<< " not found in JSON then attempt to look for string representation" \
return true; << std::endl; \
} \
\
Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its " if(v.HasMember(strReprKey)) \
<< "string representation " << str_key << " has been found " { \
<< "in JSON" << std::endl; Value& sVal = v[strReprKey]; \
if(sVal.IsString()) \
return false; { \
} try { member = CONV(sVal.GetString()); } \
catch (...) \
/** While JSON doesn't have problems representing 64 bit integers JavaScript { \
* standard represents numbers in a double-like format thus it is not capable to RsErr() << __PRETTY_FUNCTION__ << " cannot convert " \
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look << sVal.GetString() << " to integral type" << std::endl; \
* for the string representation in the JSON for this types as a workaround for return false; \
* the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 } \
*/ \
template<> /*static*/ return true; \
bool RsTypeSerializer::from_JSON( } \
const std::string& memberName, uint64_t& member, RsJson& jDoc ) } \
{ \
const char* mName = memberName.c_str(); Dbg3() << __PRETTY_FUNCTION__ << " neither integral representation nor " \
if(jDoc.HasMember(mName)) << "string representation found for: " << memberName << std::endl; \
{ \
rapidjson::Value& v = jDoc[mName]; return false; \
if(v.IsUint64())
{
member = v.GetUint64();
return true;
}
}
Dbg4() << __PRETTY_FUNCTION__ << " uint64_t " << memberName << " not found "
<< "in JSON then attempt to look for string representation"
<< std::endl;
const std::string str_key = memberName + strReprSuffix;
std::string str_value;
if(from_JSON(str_key, str_value, jDoc))
{
try { member = std::stoull(str_value); }
catch (...)
{
RsErr() << __PRETTY_FUNCTION__ << " cannot convert "
<< str_value << " to uint64_t" << std::endl;
return false;
}
return true;
}
Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its "
<< "string representation " << str_key << " has been found "
<< "in JSON" << std::endl;
return false;
} }
SIXTYFOUR_INTEGERS_FROM_JSON_DEF(uint64_t, IsUint64, GetUint64, std::stoull)
SIXTYFOUR_INTEGERS_FROM_JSON_DEF( int64_t, IsInt64, GetInt64, std::stoll)
//============================================================================// //============================================================================//
// Floats // // Floats //
@ -662,113 +503,170 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/,
// Binary blocks // // Binary blocks //
//============================================================================// //============================================================================//
template<> uint32_t RsTypeSerializer::serial_size(const RsTypeSerializer::TlvMemBlock_proxy& r) { return 4 + r.second ; } void RsTypeSerializer::RawMemoryWrapper::serial_process(
RsGenericSerializer::SerializeJob j,
template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,RsTypeSerializer::TlvMemBlock_proxy& r) RsGenericSerializer::SerializeContext& ctx )
{ {
uint32_t saved_offset = offset ; switch(j)
bool ok = deserialize<uint32_t>(data,size,offset,r.second) ;
if(r.second == 0)
{ {
r.first = NULL ; case RsGenericSerializer::SIZE_ESTIMATE:
RS_SERIAL_PROCESS(second);
if(!ok) ctx.mOffset += second;
offset = saved_offset ; break;
case RsGenericSerializer::SERIALIZE:
return ok ; if(!ctx.mOk) break;
if(second > MAX_SERIALIZED_CHUNK_SIZE)
{
RsErr() << __PRETTY_FUNCTION__
<< std::errc::message_size << " "
<< second << " > " << MAX_SERIALIZED_CHUNK_SIZE
<< std::endl;
print_stacktrace();
break;
} }
if(r.second > MAX_SERIALIZED_CHUNK_SIZE) RS_SERIAL_PROCESS(second);
if(!ctx.mOk) break;
ctx.mOk = ctx.mSize >= ctx.mOffset + second;
if(!ctx.mOk)
{ {
std::cerr << "(EE) RsTypeSerializer::deserialize<TlvMemBlock_proxy>(): data chunk has size larger than safety size (" << MAX_SERIALIZED_CHUNK_SIZE << "). Item will be dropped." << std::endl; RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space
offset = saved_offset ; << std::endl;
return false ; print_stacktrace();
break;
}
memcpy(ctx.mData + ctx.mOffset, first, second);
ctx.mOffset += second;
break;
case RsGenericSerializer::DESERIALIZE:
{
uint32_t serialSize = 0;
RS_SERIAL_PROCESS(serialSize);
if(!ctx.mOk) break;
ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE;
if(!ctx.mOk)
{
RsErr() << __PRETTY_FUNCTION__
<< std::errc::message_size << " "
<< serialSize << " > " << MAX_SERIALIZED_CHUNK_SIZE
<< std::endl;
clear();
break;
} }
r.first = (uint8_t*)rs_malloc(r.second) ; if(!serialSize)
{
ok = ok && (NULL != r.first); clear();
break;
memcpy(r.first,&data[offset],r.second) ;
offset += r.second ;
if(!ok)
offset = saved_offset ;
return ok;
} }
template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const RsTypeSerializer::TlvMemBlock_proxy& r) ctx.mOk = ctx.mSize >= ctx.mOffset + serialSize;
if(!ctx.mOk)
{ {
uint32_t saved_offset = offset ; RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space
<< std::endl;
print_stacktrace();
bool ok = serialize<uint32_t>(data,size,offset,r.second) ; clear();
break;
memcpy(&data[offset],r.first,r.second) ;
offset += r.second ;
if(!ok)
offset = saved_offset ;
return ok;
} }
template<> void RsTypeSerializer::print_data(const std::string& n, const RsTypeSerializer::TlvMemBlock_proxy& s) if(serialSize != second)
{ {
std::cerr << " [Binary data] " << n << ", length=" << s.second << " data=" << RsUtil::BinToHex((uint8_t*)s.first,std::min(50u,s.second)) << ((s.second>50)?"...":"") << std::endl; first = reinterpret_cast<uint8_t*>(realloc(first, serialSize));
second = static_cast<uint32_t>(serialSize);
} }
template<> /*static*/ memcpy(first, ctx.mData + ctx.mOffset, second);
bool RsTypeSerializer::to_JSON( ctx.mOffset += second;
const std::string& memberName, break;
const RsTypeSerializer::TlvMemBlock_proxy& member, RsJson& jDoc ) }
case RsGenericSerializer::PRINT: break;
case RsGenericSerializer::TO_JSON:
{ {
rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); if(!ctx.mOk) break;
rapidjson::Value key;
key.SetString( memberName.c_str(),
static_cast<rapidjson::SizeType>(memberName.length()),
allocator );
std::string encodedValue; std::string encodedValue;
Radix64::encode( reinterpret_cast<uint8_t*>(member.first), RsBase64::encode(first, second, encodedValue, true, false);
static_cast<int>(member.second), encodedValue ); ctx.mJson.SetString(
encodedValue.data(),
rapidjson::Value value; static_cast<rapidjson::SizeType>(encodedValue.length()) );
value.SetString(encodedValue.data(), allocator); break;
jDoc.AddMember(key, value, allocator);
return true;
} }
case RsGenericSerializer::FROM_JSON:
template<> /*static*/
bool RsTypeSerializer::from_JSON( const std::string& memberName,
RsTypeSerializer::TlvMemBlock_proxy& member,
RsJson& jDoc)
{ {
SAFE_GET_JSON_V(); const bool yelding = !!( RsSerializationFlags::YIELDING &
ret = ret && v.IsString(); ctx.mFlags.toEFT<RsSerializationFlags>() );
if(ret) if(!(ctx.mOk || yelding))
{ {
std::string encodedValue = v.GetString(); clear();
std::vector<uint8_t> decodedValue = Radix64::decode(encodedValue); break;
member.second = decodedValue.size(); }
if(!ctx.mJson.IsString())
{
RsErr() << __PRETTY_FUNCTION__ << " "
<< std::errc::invalid_argument << std::endl;
print_stacktrace();
if(member.second == 0) ctx.mOk = false;
clear();
break;
}
if( ctx.mJson.GetStringLength() >
RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) )
{ {
member.first = nullptr; RsErr() << __PRETTY_FUNCTION__ << " "
return ret; << std::errc::message_size << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
} }
member.first = rs_malloc(member.second); std::string encodedValue = ctx.mJson.GetString();
ret = ret && member.first && std::vector<uint8_t> decoded;
memcpy(member.first, decodedValue.data(), member.second); auto ec = RsBase64::decode(encodedValue, decoded);
if(ec)
{
RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
} }
return ret;
const auto decodedSize = decoded.size();
if(!decodedSize)
{
clear();
break;
} }
if(decodedSize != second)
{
first = reinterpret_cast<uint8_t*>(realloc(first, decodedSize));
second = static_cast<uint32_t>(decodedSize);
}
memcpy(first, decoded.data(), second);
break;
}
default: RsTypeSerializer::fatalUnknownSerialJob(j);
}
}
void RsTypeSerializer::RawMemoryWrapper::clear()
{
free(first);
first = nullptr;
second = 0;
}
//============================================================================//
// std::error_condition //
//============================================================================//
void RsTypeSerializer::ErrConditionWrapper::serial_process( void RsTypeSerializer::ErrConditionWrapper::serial_process(
RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) RsGenericSerializer::SerializeContext& ctx )
@ -778,12 +676,12 @@ void RsTypeSerializer::ErrConditionWrapper::serial_process(
case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough
case RsGenericSerializer::DESERIALIZE: // fallthrough case RsGenericSerializer::DESERIALIZE: // fallthrough
case RsGenericSerializer::SERIALIZE: // fallthrough case RsGenericSerializer::SERIALIZE: // fallthrough
case RsGenericSerializer::FROM_JSON: case RsGenericSerializer::FROM_JSON: // [[fallthrough]]
case RsGenericSerializer::PRINT:
RsFatal() << __PRETTY_FUNCTION__ << " SerializeJob: " << j RsFatal() << __PRETTY_FUNCTION__ << " SerializeJob: " << j
<< "is not supported on std::error_condition " << std::endl; << "is not supported on std::error_condition " << std::endl;
print_stacktrace(); print_stacktrace();
exit(-2); exit(-2);
case RsGenericSerializer::PRINT: // fallthrough
case RsGenericSerializer::TO_JSON: case RsGenericSerializer::TO_JSON:
{ {
constexpr RsGenericSerializer::SerializeJob rj = constexpr RsGenericSerializer::SerializeJob rj =

961
libretroshare/src/serialiser/rstypeserializer.h
View File

File diff suppressed because it is too large Load Diff

4
libretroshare/src/util/rsdebug.h
View File

@ -107,6 +107,10 @@ struct t_RsLogger
<< std::setfill('0') << std::setw(3) << msec.count() << std::setfill('0') << std::setw(3) << msec.count()
<< std::setfill(tFill) << " " << val; << std::setfill(tFill) << " " << val;
} }
/// needed for manipulators and things like std::endl
stream_type& operator<<(std::ostream& (*pf)(std::ostream&))
{ return std::cerr << pf; }
}; };
#endif // def __ANDROID__ #endif // def __ANDROID__

59
libretroshare/src/util/rsendian.h Normal file
View File

@ -0,0 +1,59 @@
/*******************************************************************************
* *
* libretroshare base64 encoding utilities *
* *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
/**
* @file
* This file provide convenient integer endiannes conversion utilities.
* Instead of providing them with different names for each type (a la C htonl,
* htons, ntohl, ntohs ), which make them uncomfortable to use, expose a
* templated function `rs_endian_fix` to reorder integers representation byets
* when sending or receiving from the network. */
/* enforce LITTLE_ENDIAN on Windows */
#ifdef WINDOWS_SYS
# define BYTE_ORDER 1234
# define LITTLE_ENDIAN 1234
# define BIG_ENDIAN 4321
#else
#include <arpa/inet.h>
#endif
template<typename INTTYPE> inline INTTYPE rs_endian_fix(INTTYPE val)
{
#if BYTE_ORDER == LITTLE_ENDIAN
INTTYPE swapped = 0;
for (size_t i = 0; i < sizeof(INTTYPE); ++i)
swapped |= (val >> (8*(sizeof(INTTYPE)-i-1)) & 0xFF) << (8*i);
return swapped;
#else
return hostI;
#endif
};
#ifndef BYTE_ORDER
# error "ENDIAN determination Failed (BYTE_ORDER not defined)"
#endif
#if !(BYTE_ORDER == BIG_ENDIAN || BYTE_ORDER == LITTLE_ENDIAN)
# error "ENDIAN determination Failed (unkown BYTE_ORDER value)"
#endif