Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Many serialization and related stuff improvements

Browse Source 
Fix bug in array-like containers serialization which could cause almost infinite
  loop on malformed input
Implement VLQ integer serialization
Unify sequence containers serialization code
Add support for VLQ serialization also for string size
Use VLQ compression for file links
Add templated function to fix endiannes for all integer types
Use bitset to print flags in binary form
Unify serialization code for integral types
Serialize 64bit integers types to JSON object with both string and integer
  representation, so it is posible to have this representation also for
  containers types like std::vetor or std::map this breaks retrocompatibility
  but is necessary to support clients written in languages which doesn't have
  64 bit integers support such as JavaScript or Dart
This commit is contained in:
Gioacchino Mazzurco 2020-03-16 16:20:06 +01:00
parent d203f31d0c
commit 39bde58c29
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
9 changed files with 985 additions and 752 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libretroshare/src/file_sharing/file_tree.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Retroshare Team <retroshare.project@gmail.com> *
* Copyright (C) 2018 Retroshare Team <contact@retroshare.cc> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -58,7 +60,8 @@ RsFileTree::fromBase64(const std::string& base64)
if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec);
RsGenericSerializer::SerializeContext ctx(
mem.data(), static_cast<uint32_t>(mem.size()) );
mem.data(), static_cast<uint32_t>(mem.size()),
SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ) );
std::unique_ptr<RsFileTree> ft(new RsFileTree);
ft->serial_process(
RsGenericSerializer::SerializeJob::DESERIALIZE, ctx);
@ -70,6 +73,7 @@ RsFileTree::fromBase64(const std::string& base64)
std::string RsFileTree::toBase64() const
{
RsGenericSerializer::SerializeContext ctx;
ctx.mFlags = SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ);
RsFileTree* ncThis = const_cast<RsFileTree*>(this);
ncThis->serial_process(
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx );

1
libretroshare/src/libretroshare.pro
View File

@ -481,6 +481,7 @@ HEADERS += util/folderiterator.h \
util/radix32.h \
util/radix64.h \
util/rsbase64.h \
util/rsendian.h \
util/rsinitedptr.h \
util/rsprint.h \
util/rsstring.h \

4
libretroshare/src/retroshare/rsfiles.h
View File

@ -309,10 +309,6 @@ public:
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
/* TODO: most of the time handles are smaller then 64bit use VLQ for
* binary serialization of those
* https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding
*/
RS_SERIAL_PROCESS(mFiles);
RS_SERIAL_PROCESS(mDirs);
RS_SERIAL_PROCESS(mTotalFiles);

33
libretroshare/src/retroshare/rsflags.h
View File

@ -24,6 +24,7 @@
#include <type_traits>
#include <ostream>
#include <bitset>
/** Check if given type is a scoped enum */
template<typename E>
@ -128,13 +129,7 @@ typename std::enable_if<Rs__BitFlagsOps<EFT>::enabled, std::ostream>::type&
operator <<(std::ostream& stream, EFT flags)
{
using u_t = typename std::underlying_type<EFT>::type;
for(int i = sizeof(u_t); i>=0; --i)
{
stream << (flags & ( 1 << i ) ? "1" : "0");
if( i % 8 == 0 ) stream << " ";
}
return stream;
return stream << std::bitset<sizeof(u_t)>(static_cast<u_t>(flags));
}
#include <cstdint>
@ -170,6 +165,7 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
inline t_RsFlags32() : _bits(0) {}
inline explicit t_RsFlags32(uint32_t N) : _bits(N) {} // allows initialization from a set of uint32_t
inline t_RsFlags32<n> operator| (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits | f._bits) ; }
inline t_RsFlags32<n> operator^ (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits ^ f._bits) ; }
inline t_RsFlags32<n> operator* (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits & f._bits) ; }
@ -187,6 +183,19 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
//inline explicit operator bool() const { return _bits>0; }
inline uint32_t toUInt32() const { return _bits ; }
/// Easier porting to new flag system
template<typename EFT> inline
typename std::enable_if<(Rs__BitFlagsOps<EFT>::enabled &&
sizeof(EFT) >= sizeof(uint32_t) ), EFT>::type
toEFT() { return static_cast<EFT>(_bits); }
/// Easier porting to new flag system
template<typename EFT>
static inline typename std::enable_if<
Rs__BitFlagsOps<EFT>::enabled &&
sizeof(EFT) <= sizeof(uint32_t), t_RsFlags32>::type
fromEFT(EFT e) { return t_RsFlags32(static_cast<uint32_t>(e)); }
void clear() { _bits = 0 ; }
friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32<n>& f) // friendly print with 0 and I
@ -199,8 +208,10 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
}
return o ;
}
private:
uint32_t _bits ;
private:
friend struct RsTypeSerializer;
uint32_t _bits;
};
#define FLAGS_TAG_TRANSFER_REQS 0x4228af
@ -230,7 +241,7 @@ typedef t_RsFlags32<FLAGS_TAG_SERVICE_PERM > ServicePermissionFlags ;
//
typedef t_RsFlags32<FLAGS_TAG_SERVICE_CHAT > ChatLobbyFlags ;
// Flags for serializer
//
/// @deprecated Flags for serializer
RS_DEPRECATED_FOR(RsSerializationFlags)
typedef t_RsFlags32<FLAGS_TAG_SERIALIZER > SerializationFlags ;

37
libretroshare/src/serialiser/rsserializer.h
View File

@ -192,6 +192,39 @@ class RsRawSerialiser: public RsSerialType
virtual RsItem * deserialise(void *data, uint32_t *size);
};
/** These are convenience flags to be used by the items when processing the
* data. The names of the flags are not very important. What matters is that
* the serial_process() method of each item correctly deals with the data
* when it sees the flags, if the serialiser sets them.
* By default the flags are not set and shouldn't be handled.
* When deriving a new serializer, the user can set his own flags, using
* compatible values
*/
enum class RsSerializationFlags
{
NONE = 0,
CONFIG = 1,
SIGNATURE = 2,
SKIP_HEADER = 4,
/** Used for JSON deserialization in JSON API, it causes the deserialization
* to continue even if some field is missing (or incorrect), this way the
* API is more user friendly as some methods need just part of the structs
* they take as parameters. */
YIELDING = 8,
/** When set integers typer are serialized/deserialized in Variable Length
* Quantity mode
* @see https://en.wikipedia.org/wiki/Variable-length_quantity
* This type of encoding is efficent when absoulte value is usually much
* smaller then the maximum representable with the original type.
* This encoding is also capable of representing big values at expences of a
* one more byte used.
*/
INTEGER_VLQ = 16
};
RS_REGISTER_ENUM_FLAGS_TYPE(RsSerializationFlags);
/// Top class for all services and config serializers.
struct RsGenericSerializer : RsSerialType
{
@ -200,7 +233,7 @@ struct RsGenericSerializer : RsSerialType
SIZE_ESTIMATE = 0x01,
SERIALIZE = 0x02,
DESERIALIZE = 0x03,
PRINT = 0x04,
PRINT = 0x04, /// @deprecated use rsdebug.h << operator instead
TO_JSON,
FROM_JSON
} SerializeJob;
@ -227,7 +260,7 @@ struct RsGenericSerializer : RsSerialType
SerializationFlags flags,
RsJson::AllocatorType* allocator = nullptr) :
mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format),
mFlags(flags), mJson(rapidjson::kObjectType, allocator) {}
mFlags{flags}, mJson(rapidjson::kObjectType, allocator) {}
unsigned char *mData;
uint32_t mSize;

610
libretroshare/src/serialiser/rstypeserializer.cc
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2017 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -25,7 +26,7 @@
#include "serialiser/rsbaseserial.h"
#include "serialiser/rstlvkeys.h"
#include "serialiser/rsserializable.h"
#include "util/radix64.h"
#include "util/rsbase64.h"
#include "util/rsprint.h"
#include "util/rstime.h"
@ -34,8 +35,6 @@
#include <typeinfo> // for typeid
#include <rapidjson/prettywriter.h>
static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.
#ifdef RSSERIAL_DEBUG
# define SAFE_GET_JSON_V() \
const char* mName = memberName.c_str(); \
@ -56,32 +55,10 @@ static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.
rapidjson::Value& v = jDoc[mName]
#endif // ifdef RSSERIAL_DEBUG
//============================================================================//
// std::string //
//============================================================================//
template<> uint32_t RsTypeSerializer::serial_size(const std::string& str)
{
return getRawStringSize(str);
}
template<> bool RsTypeSerializer::serialize( uint8_t data[], uint32_t size,
uint32_t& offset,
const std::string& str )
{
return setRawString(data, size, &offset, str);
}
template<> bool RsTypeSerializer::deserialize( const uint8_t data[],
uint32_t size, uint32_t &offset,
std::string& str )
{
return getRawString(data, size, &offset, str);
}
template<> void RsTypeSerializer::print_data( const std::string& n,
const std::string& str )
{
std::cerr << " [std::string] " << n << ": " << str << std::endl;
}
template<> /*static*/
bool RsTypeSerializer::to_JSON( const std::string& membername,
const std::string& member, RsJson& jDoc )
@ -112,135 +89,11 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName,
return ret;
}
//============================================================================//
// Integer types //
// Integral types //
//============================================================================//
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const bool& member)
{
return setRawUInt8(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, const int32_t& /*member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return false;
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint8_t& member)
{
return setRawUInt8(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint16_t& member)
{
return setRawUInt16(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint32_t& member)
{
return setRawUInt32(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint64_t& member)
{
return setRawUInt64(data,size,&offset,member);
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const rstime_t& member)
{
return setRawTimeT(data,size,&offset,member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, bool& member)
{
uint8_t m;
bool ok = getRawUInt8(data,size,&offset,&m);
member = m;
return ok;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, int32_t& /*member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return false;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint8_t& member)
{
return getRawUInt8(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t& member)
{
return getRawUInt16(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint32_t& member)
{
return getRawUInt32(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint64_t& member)
{
return getRawUInt64(data,size,&offset,&member);
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, rstime_t& member)
{
return getRawTimeT(data,size,&offset,member);
}
template<> uint32_t RsTypeSerializer::serial_size(const bool& /* member*/)
{
return 1;
}
template<> uint32_t RsTypeSerializer::serial_size(const int32_t& /* member*/)
{
std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl;
print_stacktrace();
return 0;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint8_t& /* member*/)
{
return 1;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint16_t& /* member*/)
{
return 2;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint32_t& /* member*/)
{
return 4;
}
template<> uint32_t RsTypeSerializer::serial_size(const uint64_t& /* member*/)
{
return 8;
}
template<> uint32_t RsTypeSerializer::serial_size(const rstime_t& /* member*/)
{
return 8;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const bool & V)
{
std::cerr << " [bool ] " << n << ": " << V << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const int32_t& V)
{
std::cerr << " [int32_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint8_t & V)
{
std::cerr << " [uint8_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint16_t& V)
{
std::cerr << " [uint16_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint32_t& V)
{
std::cerr << " [uint32_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const uint64_t& V)
{
std::cerr << " [uint64_t ] " << n << ": " << std::to_string(V) << std::endl;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const rstime_t& V)
{
std::cerr << " [rstime_t ] " << n << ": " << V << " (" << time(NULL)-V << " secs ago)" << std::endl;
}
#define SIMPLE_TO_JSON_DEF(T) \
template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \
const T& member, RsJson& jDoc ) \
@ -268,28 +121,45 @@ SIMPLE_TO_JSON_DEF(uint32_t)
/** Be very careful in changing this constant as it would break 64 bit integers
* members JSON string representation retrocompatibility */
static constexpr char strReprSuffix[] = "_sixtyfour_str";
static constexpr char strReprKey[] = "xstr64";
/** While JSON doesn't have problems representing 64 bit integers JavaScript
* standard represents numbers in a double-like format thus it is not capable to
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we add
* to JSON also the string representation for this types as a workaround for the
* sake of JavaScript clients @see https://stackoverflow.com/a/34989371
/** Be very careful in changing this constant as it would break 64 bit integers
* members JSON string representation retrocompatibility */
static constexpr char intReprKey[] = "xint64";
/** While JSON doesn't have problems representing 64 bits integers JavaScript,
* Dart and other languages represents numbers in a double-like format thus they
* are not capable to handle safely integers outside the range
* [-(2^53 - 1), 2^53 - 1].
* To overcome this limitation we represent 64 bit integers as an object with
* two keys, one as integer and one as string representation.
* In our case we need to wrap those into an object instead of just adding a key
* with a suffix so support well also containers like std::map or std::vector.
* More discussion on the topic at @see https://stackoverflow.com/a/34989371
*/
#define SIXTYFOUR_INTEGERS_TO_JSON_DEF(T) \
template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \
const T& member, RsJson& jDoc ) \
{ \
rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); \
using namespace rapidjson; \
Document::AllocatorType& allocator = jDoc.GetAllocator(); \
\
rapidjson::Value key; \
Document wrapper(rapidjson::kObjectType, &allocator); \
\
Value intKey; \
intKey.SetString(intReprKey, allocator ); \
Value intValue(member); \
wrapper.AddMember(intKey, intValue, allocator); \
\
bool ok = to_JSON(strReprKey, std::to_string(member), wrapper); \
\
Value key; \
key.SetString( memberName.c_str(), \
static_cast<rapidjson::SizeType>(memberName.length()), \
allocator ); \
rapidjson::Value value(member); \
jDoc.AddMember(key, value, allocator); \
jDoc.AddMember(key, wrapper, allocator); \
\
return to_JSON(memberName + strReprSuffix, std::to_string(member), jDoc); \
return ok; \
}
SIXTYFOUR_INTEGERS_TO_JSON_DEF(int64_t);
@ -345,100 +215,71 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName,
return ret;
}
/** While JSON doesn't have problems representing 64 bit integers JavaScript
* standard represents numbers in a double-like format thus it is not capable to
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look
* for the string representation in the JSON for this types as a workaround for
* the sake of JavaScript clients @see https://stackoverflow.com/a/34989371
*/
template<> /*static*/
bool RsTypeSerializer::from_JSON(
const std::string& memberName, int64_t& member, RsJson& jDoc )
{
const char* mName = memberName.c_str();
if(jDoc.HasMember(mName))
{
rapidjson::Value& v = jDoc[mName];
if(v.IsInt64())
{
member = v.GetInt64();
return true;
}
}
Dbg4() << __PRETTY_FUNCTION__ << " int64_t " << memberName << " not found "
<< "in JSON then attempt to look for string representation"
<< std::endl;
const std::string str_key = memberName + strReprSuffix;
std::string str_value;
if(from_JSON(str_key, str_value, jDoc))
{
try { member = std::stoll(str_value); }
catch (...)
{
RsErr() << __PRETTY_FUNCTION__ << " cannot convert "
<< str_value << " to int64_t" << std::endl;
return false;
}
return true;
}
Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its "
<< "string representation " << str_key << " has been found "
<< "in JSON" << std::endl;
return false;
}
/** While JSON doesn't have problems representing 64 bit integers JavaScript
* standard represents numbers in a double-like format thus it is not capable to
* handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look
* for the string representation in the JSON for this types as a workaround for
* the sake of JavaScript clients @see https://stackoverflow.com/a/34989371
*/
template<> /*static*/
bool RsTypeSerializer::from_JSON(
const std::string& memberName, uint64_t& member, RsJson& jDoc )
{
const char* mName = memberName.c_str();
if(jDoc.HasMember(mName))
{
rapidjson::Value& v = jDoc[mName];
if(v.IsUint64())
{
member = v.GetUint64();
return true;
}
}
Dbg4() << __PRETTY_FUNCTION__ << " uint64_t " << memberName << " not found "
<< "in JSON then attempt to look for string representation"
<< std::endl;
const std::string str_key = memberName + strReprSuffix;
std::string str_value;
if(from_JSON(str_key, str_value, jDoc))
{
try { member = std::stoull(str_value); }
catch (...)
{
RsErr() << __PRETTY_FUNCTION__ << " cannot convert "
<< str_value << " to uint64_t" << std::endl;
return false;
}
return true;
}
Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its "
<< "string representation " << str_key << " has been found "
<< "in JSON" << std::endl;
return false;
/** inverse of @see SIXTYFOUR_INTEGERS_TO_JSON_DEF */
#define SIXTYFOUR_INTEGERS_FROM_JSON_DEF(T, PRED, GET, CONV) \
template<> bool RsTypeSerializer::from_JSON( \
const std::string& memberName, T& member, RsJson& jDoc ) \
{ \
using namespace rapidjson; \
\
SAFE_GET_JSON_V(); \
\
/* For retro-compatibility take it directly if it is passed as integer */ \
if(v.PRED()) \
{ \
member = v.GET(); \
return true; \
} \
\
ret = ret && v.IsObject(); \
\
if(!ret) \
{ \
Dbg3() << __PRETTY_FUNCTION__ << " " << memberName << " not found" \
<< std::endl; \
return false; \
} \
\
if(v.HasMember(intReprKey)) \
{ \
Value& iVal = v[intReprKey]; \
if(iVal.PRED()) \
{ \
member = iVal.GET(); \
return true; \
} \
} \
\
Dbg4() << __PRETTY_FUNCTION__ << " integer representation of " << memberName \
<< " not found in JSON then attempt to look for string representation" \
<< std::endl; \
\
\
if(v.HasMember(strReprKey)) \
{ \
Value& sVal = v[strReprKey]; \
if(sVal.IsString()) \
{ \
try { member = CONV(sVal.GetString()); } \
catch (...) \
{ \
RsErr() << __PRETTY_FUNCTION__ << " cannot convert " \
<< sVal.GetString() << " to integral type" << std::endl; \
return false; \
} \
\
return true; \
} \
} \
\
Dbg3() << __PRETTY_FUNCTION__ << " neither integral representation nor " \
<< "string representation found for: " << memberName << std::endl; \
\
return false; \
}
SIXTYFOUR_INTEGERS_FROM_JSON_DEF(uint64_t, IsUint64, GetUint64, std::stoull)
SIXTYFOUR_INTEGERS_FROM_JSON_DEF( int64_t, IsInt64, GetInt64, std::stoll)
//============================================================================//
// Floats //
@ -662,113 +503,170 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/,
// Binary blocks //
//============================================================================//
template<> uint32_t RsTypeSerializer::serial_size(const RsTypeSerializer::TlvMemBlock_proxy& r) { return 4 + r.second ; }
template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,RsTypeSerializer::TlvMemBlock_proxy& r)
void RsTypeSerializer::RawMemoryWrapper::serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
uint32_t saved_offset = offset ;
bool ok = deserialize<uint32_t>(data,size,offset,r.second) ;
if(r.second == 0)
{
r.first = NULL ;
if(!ok)
offset = saved_offset ;
return ok ;
}
if(r.second > MAX_SERIALIZED_CHUNK_SIZE)
{
std::cerr << "(EE) RsTypeSerializer::deserialize<TlvMemBlock_proxy>(): data chunk has size larger than safety size (" << MAX_SERIALIZED_CHUNK_SIZE << "). Item will be dropped." << std::endl;
offset = saved_offset ;
return false ;
}
r.first = (uint8_t*)rs_malloc(r.second) ;
ok = ok && (NULL != r.first);
memcpy(r.first,&data[offset],r.second) ;
offset += r.second ;
if(!ok)
offset = saved_offset ;
return ok;
}
template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const RsTypeSerializer::TlvMemBlock_proxy& r)
{
uint32_t saved_offset = offset ;
bool ok = serialize<uint32_t>(data,size,offset,r.second) ;
memcpy(&data[offset],r.first,r.second) ;
offset += r.second ;
if(!ok)
offset = saved_offset ;
return ok;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const RsTypeSerializer::TlvMemBlock_proxy& s)
{
std::cerr << " [Binary data] " << n << ", length=" << s.second << " data=" << RsUtil::BinToHex((uint8_t*)s.first,std::min(50u,s.second)) << ((s.second>50)?"...":"") << std::endl;
}
template<> /*static*/
bool RsTypeSerializer::to_JSON(
const std::string& memberName,
const RsTypeSerializer::TlvMemBlock_proxy& member, RsJson& jDoc )
{
rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator();
rapidjson::Value key;
key.SetString( memberName.c_str(),
static_cast<rapidjson::SizeType>(memberName.length()),
allocator );
std::string encodedValue;
Radix64::encode( reinterpret_cast<uint8_t*>(member.first),
static_cast<int>(member.second), encodedValue );
rapidjson::Value value;
value.SetString(encodedValue.data(), allocator);
jDoc.AddMember(key, value, allocator);
return true;
}
template<> /*static*/
bool RsTypeSerializer::from_JSON( const std::string& memberName,
RsTypeSerializer::TlvMemBlock_proxy& member,
RsJson& jDoc)
{
SAFE_GET_JSON_V();
ret = ret && v.IsString();
if(ret)
switch(j)
{
std::string encodedValue = v.GetString();
std::vector<uint8_t> decodedValue = Radix64::decode(encodedValue);
member.second = decodedValue.size();
if(member.second == 0)
case RsGenericSerializer::SIZE_ESTIMATE:
RS_SERIAL_PROCESS(second);
ctx.mOffset += second;
break;
case RsGenericSerializer::SERIALIZE:
if(!ctx.mOk) break;
if(second > MAX_SERIALIZED_CHUNK_SIZE)
{
member.first = nullptr;
return ret;
RsErr() << __PRETTY_FUNCTION__
<< std::errc::message_size << " "
<< second << " > " << MAX_SERIALIZED_CHUNK_SIZE
<< std::endl;
print_stacktrace();
break;
}
RS_SERIAL_PROCESS(second);
if(!ctx.mOk) break;
ctx.mOk = ctx.mSize >= ctx.mOffset + second;
if(!ctx.mOk)
{
RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space
<< std::endl;
print_stacktrace();
break;
}
memcpy(ctx.mData + ctx.mOffset, first, second);
ctx.mOffset += second;
break;
case RsGenericSerializer::DESERIALIZE:
{
uint32_t serialSize = 0;
RS_SERIAL_PROCESS(serialSize);
if(!ctx.mOk) break;
ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE;
if(!ctx.mOk)
{
RsErr() << __PRETTY_FUNCTION__
<< std::errc::message_size << " "
<< serialSize << " > " << MAX_SERIALIZED_CHUNK_SIZE
<< std::endl;
clear();
break;
}
member.first = rs_malloc(member.second);
ret = ret && member.first &&
memcpy(member.first, decodedValue.data(), member.second);
if(!serialSize)
{
clear();
break;
}
ctx.mOk = ctx.mSize >= ctx.mOffset + serialSize;
if(!ctx.mOk)
{
RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space
<< std::endl;
print_stacktrace();
clear();
break;
}
if(serialSize != second)
{
first = reinterpret_cast<uint8_t*>(realloc(first, serialSize));
second = static_cast<uint32_t>(serialSize);
}
memcpy(first, ctx.mData + ctx.mOffset, second);
ctx.mOffset += second;
break;
}
case RsGenericSerializer::PRINT: break;
case RsGenericSerializer::TO_JSON:
{
if(!ctx.mOk) break;
std::string encodedValue;
RsBase64::encode(first, second, encodedValue, true, false);
ctx.mJson.SetString(
encodedValue.data(),
static_cast<rapidjson::SizeType>(encodedValue.length()) );
break;
}
case RsGenericSerializer::FROM_JSON:
{
const bool yelding = !!( RsSerializationFlags::YIELDING &
ctx.mFlags.toEFT<RsSerializationFlags>() );
if(!(ctx.mOk || yelding))
{
clear();
break;
}
if(!ctx.mJson.IsString())
{
RsErr() << __PRETTY_FUNCTION__ << " "
<< std::errc::invalid_argument << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
}
if( ctx.mJson.GetStringLength() >
RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) )
{
RsErr() << __PRETTY_FUNCTION__ << " "
<< std::errc::message_size << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
}
std::string encodedValue = ctx.mJson.GetString();
std::vector<uint8_t> decoded;
auto ec = RsBase64::decode(encodedValue, decoded);
if(ec)
{
RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
}
const auto decodedSize = decoded.size();
if(!decodedSize)
{
clear();
break;
}
if(decodedSize != second)
{
first = reinterpret_cast<uint8_t*>(realloc(first, decodedSize));
second = static_cast<uint32_t>(decodedSize);
}
memcpy(first, decoded.data(), second);
break;
}
default: RsTypeSerializer::fatalUnknownSerialJob(j);
}
return ret;
}
void RsTypeSerializer::RawMemoryWrapper::clear()
{
free(first);
first = nullptr;
second = 0;
}
//============================================================================//
// std::error_condition //
//============================================================================//
void RsTypeSerializer::ErrConditionWrapper::serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
@ -778,12 +676,12 @@ void RsTypeSerializer::ErrConditionWrapper::serial_process(
case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough
case RsGenericSerializer::DESERIALIZE: // fallthrough
case RsGenericSerializer::SERIALIZE: // fallthrough
case RsGenericSerializer::FROM_JSON:
case RsGenericSerializer::FROM_JSON: // [[fallthrough]]
case RsGenericSerializer::PRINT:
RsFatal() << __PRETTY_FUNCTION__ << " SerializeJob: " << j
<< "is not supported on std::error_condition " << std::endl;
print_stacktrace();
exit(-2);
case RsGenericSerializer::PRINT: // fallthrough
case RsGenericSerializer::TO_JSON:
{
constexpr RsGenericSerializer::SerializeJob rj =

981
libretroshare/src/serialiser/rstypeserializer.h
View File

File diff suppressed because it is too large Load Diff

4
libretroshare/src/util/rsdebug.h
View File

@ -107,6 +107,10 @@ struct t_RsLogger
<< std::setfill('0') << std::setw(3) << msec.count()
<< std::setfill(tFill) << " " << val;
}
/// needed for manipulators and things like std::endl
stream_type& operator<<(std::ostream& (*pf)(std::ostream&))
{ return std::cerr << pf; }
};
#endif // def __ANDROID__

59
libretroshare/src/util/rsendian.h Normal file
View File

@ -0,0 +1,59 @@
/*******************************************************************************
* *
* libretroshare base64 encoding utilities *
* *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
/**
* @file
* This file provide convenient integer endiannes conversion utilities.
* Instead of providing them with different names for each type (a la C htonl,
* htons, ntohl, ntohs ), which make them uncomfortable to use, expose a
* templated function `rs_endian_fix` to reorder integers representation byets
* when sending or receiving from the network. */
/* enforce LITTLE_ENDIAN on Windows */
#ifdef WINDOWS_SYS
# define BYTE_ORDER 1234
# define LITTLE_ENDIAN 1234
# define BIG_ENDIAN 4321
#else
#include <arpa/inet.h>
#endif
template<typename INTTYPE> inline INTTYPE rs_endian_fix(INTTYPE val)
{
#if BYTE_ORDER == LITTLE_ENDIAN
INTTYPE swapped = 0;
for (size_t i = 0; i < sizeof(INTTYPE); ++i)
swapped |= (val >> (8*(sizeof(INTTYPE)-i-1)) & 0xFF) << (8*i);
return swapped;
#else
return hostI;
#endif
};
#ifndef BYTE_ORDER
# error "ENDIAN determination Failed (BYTE_ORDER not defined)"
#endif
#if !(BYTE_ORDER == BIG_ENDIAN || BYTE_ORDER == LITTLE_ENDIAN)
# error "ENDIAN determination Failed (unkown BYTE_ORDER value)"
#endif