From 55d466f79b657b3ad8e7409c656237d16594d8d4 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 11 Mar 2020 23:11:59 +0100 Subject: [PATCH 01/40] Initial work on RsFiles links support in libretroshare --- libretroshare/src/file_sharing/file_tree.cc | 154 ++++++----- libretroshare/src/file_sharing/file_tree.h | 54 ---- libretroshare/src/ft/ftserver.cc | 73 +++++- libretroshare/src/ft/ftserver.h | 14 +- libretroshare/src/libretroshare.pro | 1 - libretroshare/src/retroshare/rsfiles.h | 245 ++++++++++++++++-- libretroshare/src/retroshare/rsflags.h | 6 +- libretroshare/src/retroshare/rsgxsforums.h | 4 +- libretroshare/src/util/rsmemory.h | 11 +- libretroshare/src/util/rsurl.cc | 2 +- libretroshare/src/util/rsurl.h | 3 +- .../gui/FileTransfer/SharedFilesDialog.cpp | 7 +- .../src/gui/FileTransfer/TransfersDialog.cpp | 5 +- retroshare-gui/src/gui/RetroShareLink.cpp | 14 +- .../src/gui/common/RsCollection.cpp | 16 +- retroshare-gui/src/gui/common/RsCollection.h | 8 +- 16 files changed, 438 insertions(+), 179 deletions(-) delete mode 100644 libretroshare/src/file_sharing/file_tree.h diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 4953b192b..3002da6c6 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -20,40 +20,89 @@ * * ******************************************************************************/ #include -#include -#include +#include "util/radix64.h" +#include "util/rsdir.h" +#include "retroshare/rsfiles.h" #include "file_sharing_defaults.h" #include "filelist_io.h" -#include "file_tree.h" -std::string FileTreeImpl::toRadix64() const +void RsFileTree::DirData::serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) { - unsigned char *buff = NULL ; - uint32_t size = 0 ; - - serialise(buff,size) ; - - std::string res ; - - Radix64::encode(buff,size,res) ; - - free(buff) ; - return res ; + RS_SERIAL_PROCESS(name); + RS_SERIAL_PROCESS(subdirs); + RS_SERIAL_PROCESS(subfiles); } -FileTree *FileTree::create(const std::string& radix64_string) +void RsFileTree::FileData::serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) { - FileTreeImpl *ft = new FileTreeImpl ; + RS_SERIAL_PROCESS(name); + RS_SERIAL_PROCESS(size); + RS_SERIAL_PROCESS(hash); +} +/*static*/ std::unique_ptr RsFileTree::fromBase64( + const std::string& base64 ) +{ + std::unique_ptr ft(new RsFileTree); + std::vector mem = Radix64::decode(base64); + RsGenericSerializer::SerializeContext ctx( + mem.data(), static_cast(mem.size()) ); + ft->serial_process( + RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); + if(ctx.mOk) return ft; + return nullptr; +} + +std::string RsFileTree::toBase64() const +{ + RsGenericSerializer::SerializeContext ctx; + RsFileTree* ncThis = const_cast(this); + ncThis->serial_process( + RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); + RsDbg() << __PRETTY_FUNCTION__ << " ctx.mOffset: " << ctx.mOffset + << std::endl; + + std::vector buf(ctx.mOffset); + ctx.mSize = ctx.mOffset; ctx.mOffset = 0; ctx.mData = buf.data(); + + ncThis->serial_process( + RsGenericSerializer::SerializeJob::SERIALIZE, ctx ); + std::string result; + Radix64::encode(ctx.mData, ctx.mSize, result); + return result; +} + +std::string RsFileTree::toRadix64() const +{ + unsigned char* buff = nullptr; + uint32_t size = 0; + serialise(buff, size); + std::string res; + Radix64::encode(buff,size,res); + free(buff); + return res; +} + +std::unique_ptr RsFileTree::fromRadix64( + const std::string& radix64_string ) +{ + std::unique_ptr ft(new RsFileTree); std::vector mem = Radix64::decode(radix64_string); - ft->deserialise(mem.data(),mem.size()) ; - - return ft ; + if(ft->deserialise(mem.data(), static_cast(mem.size()))) + return std::move(ft); + return nullptr; } -void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const DirDetails& dd,bool remote,bool remove_top_dirs) +void RsFileTree::recurs_buildFileTree( + RsFileTree& ft, uint32_t index, const DirDetails& dd, bool remote, + bool remove_top_dirs ) { + RsDbg() << __PRETTY_FUNCTION__ << " index: " << index << std::endl; if(ft.mDirs.size() <= index) ft.mDirs.resize(index+1) ; @@ -67,14 +116,14 @@ void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const Di DirDetails dd2 ; - FileSearchFlags flags = remote?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL ; + FileSearchFlags flags = remote ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; for(uint32_t i=0;iRequestDirDetails(dd.children[i].ref,dd2,flags)) { if(dd.children[i].type == DIR_TYPE_FILE) { - FileTree::FileData f ; + FileData f ; f.name = dd2.name ; f.size = dd2.count ; f.hash = dd2.hash ; @@ -97,7 +146,9 @@ void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const Di std::cerr << "(EE) Cannot request dir details for pointer " << dd.children[i].ref << std::endl; } -bool FileTreeImpl::getDirectoryContent(uint32_t index,std::string& name,std::vector& subdirs,std::vector& subfiles) const +bool RsFileTree::getDirectoryContent( + std::string& name, std::vector& subdirs, + std::vector& subfiles, std::uintptr_t index ) const { if(index >= mDirs.size()) return false ; @@ -112,18 +163,25 @@ bool FileTreeImpl::getDirectoryContent(uint32_t index,std::string& name,std::vec return true ; } -FileTree *FileTree::create(const DirDetails& dd, bool remote,bool remove_top_dirs) +std::unique_ptr RsFileTree::fromDirDetails( + const DirDetails& dd, bool remote ,bool remove_top_dirs ) { - FileTreeImpl *ft = new FileTreeImpl ; - - FileTreeImpl::recurs_buildFileTree(*ft,0,dd,remote,remove_top_dirs) ; - - return ft ; + std::unique_ptrft(new RsFileTree); + if(dd.type == DIR_TYPE_FILE) + { + FileData fd; + fd.name = dd.name; fd.hash = dd.hash; fd.size = dd.count; + ft->mFiles.push_back(fd); + ft->mTotalFiles = 1; + ft->mTotalSize = fd.size; + } + else recurs_buildFileTree(*ft, 0, dd, remote, remove_top_dirs ); + return ft; } typedef FileListIO::read_error read_error ; -bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size) +bool RsFileTree::deserialise(unsigned char *buffer,uint32_t buffer_size) { uint32_t buffer_offset = 0 ; @@ -218,7 +276,7 @@ bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size) return true ; } -bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) const +bool RsFileTree::serialise(unsigned char *& buffer,uint32_t& buffer_size) const { buffer = 0 ; uint32_t buffer_offset = 0 ; @@ -234,7 +292,11 @@ bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) cons { // Write some header - if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,(uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001)) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField( + buffer, buffer_size, buffer_offset, + FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION, + (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001 ) ) + throw std::runtime_error("Write error") ; if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mFiles.size())) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mDirs.size())) throw std::runtime_error("Write error") ; @@ -291,29 +353,3 @@ bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) cons return false; } } - -void FileTreeImpl::print() const -{ - std::cerr << "File hierarchy: name=" << mDirs[0].name << " size=" << mTotalSize << std::endl; - recurs_print(0," ") ; -} - -void FileTreeImpl::recurs_print(uint32_t index,const std::string& indent) const -{ - if(index >= mDirs.size()) - { - std::cerr << "(EE) inconsistent FileTree structure" << std::endl; - return; - } - std::cerr << indent << mDirs[index].name << std::endl; - - for(uint32_t i=0;i * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public License * - * along with this program. If not, see . * - * * - ******************************************************************************/ -#include "retroshare/rsfiles.h" - -class FileTreeImpl: public FileTree -{ -public: - FileTreeImpl() - { - mTotalFiles = 0 ; - mTotalSize = 0 ; - } - - virtual std::string toRadix64() const ; - virtual bool getDirectoryContent(uint32_t index,std::string& name,std::vector& subdirs,std::vector& subfiles) const ; - virtual void print() const ; - - bool serialise(unsigned char *& data,uint32_t& data_size) const ; - bool deserialise(unsigned char* data, uint32_t data_size) ; - -protected: - void recurs_print(uint32_t index,const std::string& indent) const; - - static void recurs_buildFileTree(FileTreeImpl& ft, uint32_t index, const DirDetails& dd, bool remote, bool remove_top_dirs); - - struct DirData { - std::string name; - std::vector subdirs ; - std::vector subfiles ; - }; - std::vector mFiles ; - std::vector mDirs ; - - friend class FileTree ; -}; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 256543da6..8be6ac661 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -43,10 +43,10 @@ #include "rsitems/rsfiletransferitems.h" #include "rsitems/rsserviceids.h" - +#include "util/rsmemory.h" #include "rsserver/p3face.h" #include "turtle/p3turtle.h" - +#include "util/rsurl.h" #include "util/rsdebug.h" #include "util/rsdir.h" #include "util/rsprint.h" @@ -66,6 +66,16 @@ #define FTSERVER_DEBUG() std::cerr << time(NULL) << " : FILE_SERVER : " << __FUNCTION__ << " : " #define FTSERVER_ERROR() std::cerr << "(EE) FILE_SERVER ERROR : " +/*static*/ const RsFilesErrorCategory RsFilesErrorCategory::instance; + +/*static*/ const std::string RsFiles::DEFAULT_FILES_BASE_URL = + "retroshare:///files"; +/*static*/ const std::string RsFiles::FILES_URL_COUNT_FIELD = "filesCount"; +/*static*/ const std::string RsFiles::FILES_URL_DATA_FIELD = "filesData"; +/*static*/ const std::string RsFiles::FILES_URL_NAME_FIELD = "filesName"; +/*static*/ const std::string RsFiles::FILES_URL_SIZE_FIELD = "filesSize"; + + static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most. @@ -2101,3 +2111,62 @@ RsFileItem::RsFileItem(RsFileItemType subtype) : void RsFileSearchRequestItem::clear() { queryString.clear(); } void RsFileSearchResultItem::clear() { mResults.clear(); } + + +std::error_condition RsFilesErrorCategory::default_error_condition(int ev) +const noexcept +{ + switch(static_cast(ev)) + { + case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: // [[fallthrough]]; + case RsFilesErrorNum::INVALID_FILES_URL: + return std::errc::invalid_argument; + default: + return std::error_condition(ev, *this); + } +} + +std::error_condition ftServer::exportFilesLink( + std::string& link, std::uintptr_t handle, const std::string& baseUrl ) +{ + DirDetails tDirDet; + if(!requestDirDetails(tDirDet, handle)) + return RsFilesErrorNum::FILES_HANDLE_NOT_FOUND; + + std::unique_ptr tFileTree = + RsFileTree::fromDirDetails(tDirDet, false); + link = tFileTree->toBase64(); + + if(!baseUrl.empty()) + { + RsUrl tUrl(baseUrl); + tUrl.setQueryKV(FILES_URL_COUNT_FIELD, + std::to_string(tFileTree->mTotalFiles) ) + .setQueryKV(FILES_URL_DATA_FIELD, link) + .setQueryKV(FILES_URL_NAME_FIELD, tDirDet.name) + .setQueryKV( FILES_URL_SIZE_FIELD, + std::to_string(tFileTree->mTotalSize) ); + link = tUrl.toString(); + } + + return std::error_condition(); +} + +std::error_condition ftServer::parseFilesLink( + const std::string& link, RsFileTree& collection ) +{ + RsUrl tUrl(link); + rs_view_ptr radixPtr = + tUrl.getQueryV(FILES_URL_DATA_FIELD); + if(!radixPtr) radixPtr = &link; + + std::unique_ptr tft = + RsFileTree::fromBase64(*radixPtr); + if(tft) + { + collection = *tft; + return std::error_condition(); + } + + return RsFilesErrorNum::INVALID_FILES_URL; +} diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 6f719c04e..76ce8d416 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -207,14 +207,24 @@ public: virtual uint32_t filePermDirectDL() ; /// @see RsFiles - virtual bool turtleSearchRequest( + bool turtleSearchRequest( const std::string& matchString, const std::function& results)>& multiCallback, - rstime_t maxWait = 300 ); + rstime_t maxWait = 300 ) override; virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; + /// @see RsFiles + std::error_condition exportFilesLink( + std::string& link, std::uintptr_t handle, + const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL + ) override; + + /// @see RsFiles + std::error_condition parseFilesLink( + const std::string& link, RsFileTree& collection ) override; + /*** * Control of Downloads Priority. ***/ diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index bb8fe05b0..3b5dad851 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -45,7 +45,6 @@ file_lists { file_sharing/directory_updater.h \ file_sharing/rsfilelistitems.h \ file_sharing/dir_hierarchy.h \ - file_sharing/file_tree.h \ file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 3818a8f45..3f1ce2eab 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -28,12 +28,14 @@ #include #include #include +#include #include "rstypes.h" #include "serialiser/rsserializable.h" #include "rsturtle.h" #include "util/rstime.h" #include "retroshare/rsevents.h" +#include "util/rsmemory.h" class RsFiles; @@ -43,6 +45,53 @@ class RsFiles; */ extern RsFiles* rsFiles; +enum class RsFilesErrorNum : int32_t +{ + FILES_HANDLE_NOT_FOUND = 2004, + INVALID_FILES_URL = 2005 +}; + +struct RsFilesErrorCategory: std::error_category +{ + const char* name() const noexcept override + { return "RetroShare Files"; } + + std::string message(int ev) const override + { + switch (static_cast(ev)) + { + case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: + return "Files handle not found"; + case RsFilesErrorNum::INVALID_FILES_URL: + return "Invalid files url"; + default: + return "Error message for error: " + std::to_string(ev) + + " not available in category: " + name(); + } + } + + std::error_condition default_error_condition(int ev) const noexcept override; + + const static RsFilesErrorCategory instance; +}; + + +namespace std +{ +/** Register RsFilesErrorNum as an error condition enum, must be in std + * namespace */ +template<> struct is_error_condition_enum : true_type {}; +} + +/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in + * same namespace of RsJsonApiErrorNum */ +inline std::error_condition make_error_condition(RsFilesErrorNum e) noexcept +{ + return std::error_condition( + static_cast(e), RsFilesErrorCategory::instance ); +}; + + namespace RsRegularExpression { class Expression; } /* These are used mainly by ftController at the moment */ @@ -96,8 +145,28 @@ const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// bro const FileSearchFlags RS_FILE_HINTS_SEARCHABLE ( 0x00000200 );// can be searched anonymously const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000380 );// OR of the last tree flags. Used to filter out. -// Flags used when requesting a transfer -// +/** Flags used when requesting a new file transfer */ +enum class FileRequestFlags: uint32_t +{ + /// Enable requesting file via turtle routing. + ANONYMOUS_ROUTING = 0x00000040, + + /// Asks (TODO: or enforce?) for end-to-end encryption of file trasfer + ENCRYPTED = 0x00000080, + + /// Asks (TODO: or enforce?) no end-to-end encryption of file trasfer + UNENCRYPTED = 0x00000100, + + /// Start trasfer very slow + SLOW = 0x00002000, + + /// Disable searching for potential direct sources + NO_DIRECT_SEARCH = 0x02000000 +}; +RS_REGISTER_ENUM_FLAGS_TYPE(FileRequestFlags) + +/// @deprecated Flags used when requesting a transfer +/// @see FileRequestFlags instead const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING ( 0x00000040 ); // Use to ask turtle router to download the file. const TransferRequestFlags RS_FILE_REQ_ENCRYPTED ( 0x00000080 ); // Asks for end-to-end encryption of file at the level of ftServer const TransferRequestFlags RS_FILE_REQ_UNENCRYPTED ( 0x00000100 ); // Asks for no end-to-end encryption of file at the level of ftServer @@ -202,36 +271,119 @@ struct SharedDirStats uint64_t total_shared_size ; }; -// This class represents a tree of directories and files, only with their names size and hash. It is used to create collection links in the GUI -// and to transmit directory information between services. This class is independent from the existing FileHierarchy classes used in storage because -// we need a very copact serialization and storage size since we create links with it. Besides, we cannot afford to risk the leak of other local information -// by using the orignal classes. - -class FileTree +/** This class represents a tree of directories and files, only with their names + * size and hash. It is used to create collection links in the GUI and to + * transmit directory information between services. This class is independent + * from the existing FileHierarchy classes used in storage because we need a + * very copact serialization and storage size since we create links with it. + * Besides, we cannot afford to risk the leak of other local information + * by using the orignal classes. + */ +struct RsFileTree : RsSerializable { public: - virtual ~FileTree() {} + RsFileTree() : mTotalFiles(0), mTotalSize(0) {} - static FileTree *create(const DirDetails& dd, bool remote, bool remove_top_dirs = true) ; - static FileTree *create(const std::string& radix64_string) ; + /** + * @brief Create a RsFileTree from directory details + * @param dd + * @param remote + * @param remove_top_dirs + * @return + */ + static std::unique_ptr fromDirDetails( + const DirDetails& dd, bool remote, bool remove_top_dirs = true ); - virtual std::string toRadix64() const =0 ; + /** + * @brief Create a RsFileTree from Radix64 representation + * @param base64 + * @return nullptr if on failure, pointer to the created FileTree on success + */ + static std::unique_ptr fromBase64( + const std::string& base64 ); - // These methods allow the user to browse the hierarchy + /** @brief Convert to base64 representetion */ + std::string toBase64() const; - struct FileData { - std::string name ; - uint64_t size ; - RsFileHash hash ; + /// @see RsSerializable + virtual void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) + { + /* TODO: most of the time handles are smaller then 64bit use VLQ for + * binary serialization of those + * https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + */ + RS_SERIAL_PROCESS(mFiles); + RS_SERIAL_PROCESS(mDirs); + RS_SERIAL_PROCESS(mTotalFiles); + RS_SERIAL_PROCESS(mTotalSize); + } + + struct FileData: RsSerializable + { + std::string name; + uint64_t size; + RsFileHash hash; + + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override; }; - virtual uint32_t root() const { return 0;} - virtual bool getDirectoryContent(uint32_t index,std::string& name,std::vector& subdirs,std::vector& subfiles) const = 0; + /// Allow browsing the hierarchy + bool getDirectoryContent( + std::string& name, std::vector& subdirs, + std::vector& subfiles, std::uintptr_t handle = 0 ) const; - virtual void print() const=0; + struct DirData: RsSerializable + { + std::string name; + std::vector subdirs; + std::vector subfiles; - uint32_t mTotalFiles ; - uint64_t mTotalSize ; + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override; + }; + + static void recurs_buildFileTree( + RsFileTree& ft, uint32_t index, const DirDetails& dd, bool remote, + bool remove_top_dirs ); + + std::vector mFiles; + std::vector mDirs; + + uint32_t mTotalFiles; + uint64_t mTotalSize; + + ~RsFileTree() = default; + + /** + * @brief Create a RsFileTree from Radix64 representation + * @deprecated kept for retrocompatibility with RetroShare-gui + * The format is not guardanted to be compatible with the new methods + * @param radix64_string + * @return nullptr if on failure, pointer to the created FileTree on success + */ + RS_DEPRECATED + static std::unique_ptr fromRadix64( + const std::string& radix64_string ); + + /** @brief Convert to radix64 representetion + * @deprecated kept for retrocompatibility with RetroShare-gui + * The format is not guardanted to be compatible with the new methods + */ + RS_DEPRECATED std::string toRadix64() const; + +private: + /** @deprecated kept for retrocompatibility with RetroShare-gui */ + RS_DEPRECATED_FOR(serial_process) + bool serialise(unsigned char *& data,uint32_t& data_size) const; + + /** @deprecated kept for retrocompatibility with RetroShare-gui */ + RS_DEPRECATED_FOR(serial_process) + bool deserialise(unsigned char* data, uint32_t data_size); }; struct BannedFileEntry : RsSerializable @@ -301,9 +453,6 @@ struct TurtleFileInfoV2 : RsSerializable class RsFiles { public: - RsFiles() {} - virtual ~RsFiles() {} - /** * @brief Provides file data for the GUI, media streaming or API clients. * It may return unverified chunks. This allows streaming without having to @@ -726,6 +875,50 @@ public: */ virtual bool removeSharedDirectory(std::string dir) = 0; + /// Default base URL used for files links @see exportFilesLink + static const std::string DEFAULT_FILES_BASE_URL; + + /** Link query field used to store collection files count + * @see exportFilesLink */ + static const std::string FILES_URL_COUNT_FIELD; + + /// Link query field used to store collection data @see exportFilesLink + static const std::string FILES_URL_DATA_FIELD; + + /// Link query field used to store collection name @see exportFilesLink + static const std::string FILES_URL_NAME_FIELD; + + /// Link query field used to store collection size @see exportFilesLink + static const std::string FILES_URL_SIZE_FIELD; + + /** + * @brief Get link to a forum + * @jsonapi{development} + * @param[out] link storage for the generated link + * @param[in] handle file/directory RetroShare handle + * @param[in] baseUrl URL into which to sneak in the RetroShare link + * radix, this is primarly useful to induce applications into making the + * link clickable, or to disguise the RetroShare link into a + * "normal" looking web link. If empty the collection data link will be + * outputted in plain base64 format. + * @return error information if some error occurred, 0/SUCCESS otherwise + */ + virtual std::error_condition exportFilesLink( + std::string& link, std::uintptr_t handle, + const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0; + + /** + * @brief Parse RetroShare files link + * @jsonapi{development} + * @param[in] link files link either in radix or URL format + * @param[out] collection optional storage for parsed files link + * @return error information if some error occurred, 0/SUCCESS otherwise + */ + virtual std::error_condition parseFilesLink( + const std::string& link, RsFileTree& collection ) = 0; + + // virtual std::error_condition downloadFilesLink(FileRequestFlags) + /** * @brief Get list of ignored file name prefixes and suffixes * @param[out] ignoredPrefixes storage for ingored prefixes @@ -757,4 +950,6 @@ public: virtual bool ignoreDuplicates() = 0; virtual void setIgnoreDuplicates(bool ignore) = 0; + + virtual ~RsFiles() = default; }; diff --git a/libretroshare/src/retroshare/rsflags.h b/libretroshare/src/retroshare/rsflags.h index cdea3f5be..4222855fa 100644 --- a/libretroshare/src/retroshare/rsflags.h +++ b/libretroshare/src/retroshare/rsflags.h @@ -210,9 +210,9 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 #define FLAGS_TAG_SERVICE_CHAT 0x839042 #define FLAGS_TAG_SERIALIZER 0xa0338d -// Flags for requesting transfers, ask for turtle, cache, speed, etc. -// -typedef t_RsFlags32 TransferRequestFlags ; +/// @deprecated Flags for requesting transfers, ask for turtle, cache, speed, etc. +RS_DEPRECATED_FOR(FileRequestFlags) +typedef t_RsFlags32 TransferRequestFlags; // Flags for file storage. Mainly permissions like BROWSABLE/NETWORK_WIDE for groups and peers. // diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 11a47cf7b..f7c8153ba 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -305,10 +305,10 @@ public: static const std::string FORUM_URL_DATA_FIELD; /** Link query field used to store forum message title - * @see exportChannelLink */ + * @see exportForumLink */ static const std::string FORUM_URL_MSG_TITLE_FIELD; - /// Link query field used to store forum message id @see exportChannelLink + /// Link query field used to store forum message id @see exportForumLink static const std::string FORUM_URL_MSG_ID_FIELD; /** diff --git a/libretroshare/src/util/rsmemory.h b/libretroshare/src/util/rsmemory.h index 01dc4d8d8..769163ca5 100644 --- a/libretroshare/src/util/rsmemory.h +++ b/libretroshare/src/util/rsmemory.h @@ -62,7 +62,6 @@ bool myFunnyFunction( */ #define RS_DEFAULT_STORAGE_PARAM(Type,...) *std::unique_ptr(new Type(__VA_ARGS__)) - /** @brief Safely dynamic cast between std::unique_ptr of different types * std::unique_ptr semantic rely on the invariant that only one instance own * the object, when casting between differents types one would be tempted to do @@ -90,6 +89,16 @@ bool rs_unique_cast( return false; } +/** Mark a pointer as non-owned aka you are not in charge of deleting it and + * must not delete it. + * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ +template using rs_view_ptr = T*; + +/** Mark a pointer as owned aka you are in charge of deletingonce finished + * dealing with it. + * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ +template using rs_owner_ptr = T*; + void *rs_malloc(size_t size) ; diff --git a/libretroshare/src/util/rsurl.cc b/libretroshare/src/util/rsurl.cc index dd01d7384..5da7bbaf4 100644 --- a/libretroshare/src/util/rsurl.cc +++ b/libretroshare/src/util/rsurl.cc @@ -224,7 +224,7 @@ RsUrl& RsUrl::delQueryK(const std::string& key) } bool RsUrl::hasQueryK(const std::string& key) { return (mQuery.find(key) != mQuery.end()); } -const std::string* RsUrl::getQueryV(const std::string& key) +rs_view_ptr RsUrl::getQueryV(const std::string& key) { if(hasQueryK(key)) return &(mQuery.find(key)->second); return nullptr; diff --git a/libretroshare/src/util/rsurl.h b/libretroshare/src/util/rsurl.h index 0610585c5..6e6d124c3 100644 --- a/libretroshare/src/util/rsurl.h +++ b/libretroshare/src/util/rsurl.h @@ -20,6 +20,7 @@ #include #include +#include "util/rsmemory.h" #include "serialiser/rsserializable.h" struct sockaddr_storage; @@ -63,7 +64,7 @@ struct RsUrl : RsSerializable RsUrl& setQueryKV(const std::string& key, const std::string& value); RsUrl& delQueryK(const std::string& key); bool hasQueryK(const std::string& key); - const std::string* getQueryV(const std::string& key); + rs_view_ptr getQueryV(const std::string& key); const std::string& fragment() const; RsUrl& setFragment(const std::string& fragment); diff --git a/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp index 22c9c2021..608ac451c 100644 --- a/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp @@ -623,10 +623,7 @@ void SharedFilesDialog::copyLinks(const QModelIndexList& lst, bool remote,QList< if (details.type == DIR_TYPE_DIR) { - FileTree *ft = FileTree::create(details,remote) ; - - std::cerr << "Created collection file tree:" << std::endl; - ft->print(); + auto ft = RsFileTree::fromDirDetails(details,remote); QString dir_name = QDir(QString::fromUtf8(details.name.c_str())).dirName(); @@ -634,8 +631,6 @@ void SharedFilesDialog::copyLinks(const QModelIndexList& lst, bool remote,QList< if(link.valid()) urls.push_back(link) ; - - delete ft ; } else { diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index 9a901b809..3c00f2122 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -2225,9 +2225,8 @@ void TransfersDialog::pasteLink() for(QList::const_iterator it(links.begin());it!=links.end();++it) { - FileTree *ft = FileTree::create((*it).radix().toStdString()) ; - - col.merge_in(*ft) ; + auto ft = RsFileTree::fromRadix64((*it).radix().toStdString()); + col.merge_in(*ft); } links.clear(); RSLinkClipboard::pasteLinks(links,RetroShareLink::TYPE_FILE); diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index fe7374c90..72d6d7464 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -1706,15 +1706,13 @@ static void processList(const QStringList &list, const QString &textSingular, co } break; - case TYPE_FILE_TREE: - { - FileTree *ft = FileTree::create(link.radix().toStdString()) ; - - RsCollection(*ft).downloadFiles() ; - - delete ft; - } + case TYPE_FILE_TREE: + { + auto ft = RsFileTree::fromRadix64( + link.radix().toStdString() ); + RsCollection(*ft).downloadFiles(); break; + } case TYPE_CHAT_ROOM: { diff --git a/retroshare-gui/src/gui/common/RsCollection.cpp b/retroshare-gui/src/gui/common/RsCollection.cpp index 53661fb75..ec9fec59b 100644 --- a/retroshare-gui/src/gui/common/RsCollection.cpp +++ b/retroshare-gui/src/gui/common/RsCollection.cpp @@ -44,7 +44,7 @@ RsCollection::RsCollection(QObject *parent) _xml_doc.appendChild(_root); } -RsCollection::RsCollection(const FileTree& fr) +RsCollection::RsCollection(const RsFileTree& fr) : _xml_doc("RsCollection") { _root = _xml_doc.createElement("RsCollection"); @@ -153,7 +153,7 @@ void RsCollection::merge_in(const QString& fname,uint64_t size,const RsFileHash& recursAddElements(_xml_doc,info,_root) ; } -void RsCollection::merge_in(const FileTree& tree) +void RsCollection::merge_in(const RsFileTree& tree) { recursAddElements(_xml_doc,tree,0,_root) ; } @@ -273,14 +273,14 @@ void RsCollection::recursAddElements(QDomDocument& doc,const ColFileInfo& colFil } } -void RsCollection::recursAddElements(QDomDocument& doc,const FileTree& ft,uint32_t index,QDomElement& e) const +void RsCollection::recursAddElements( + QDomDocument& doc, const RsFileTree& ft, uint32_t index, + QDomElement& e ) const { - std::vector subdirs ; - std::vector subfiles ; + std::vector subdirs; + std::vector subfiles ; std::string name; - - if(!ft.getDirectoryContent(index,name,subdirs,subfiles)) - return ; + if(!ft.getDirectoryContent(name, subdirs, subfiles, index)) return; QDomElement d = doc.createElement("Directory") ; d.setAttribute(QString("name"),QString::fromUtf8(name.c_str())) ; diff --git a/retroshare-gui/src/gui/common/RsCollection.h b/retroshare-gui/src/gui/common/RsCollection.h index f1a8cb866..492d407d8 100644 --- a/retroshare-gui/src/gui/common/RsCollection.h +++ b/retroshare-gui/src/gui/common/RsCollection.h @@ -62,11 +62,11 @@ public: RsCollection(QObject *parent = 0) ; // create from list of files and directories RsCollection(const std::vector& file_entries, FileSearchFlags flags, QObject *parent = 0) ; - RsCollection(const FileTree& fr); + RsCollection(const RsFileTree& fr); virtual ~RsCollection() ; void merge_in(const QString& fname,uint64_t size,const RsFileHash& hash) ; - void merge_in(const FileTree& tree) ; + void merge_in(const RsFileTree& tree) ; static const QString ExtensionString ; @@ -99,7 +99,9 @@ private: void recursAddElements(QDomDocument&, const DirDetails&, QDomElement&, FileSearchFlags flags) const ; void recursAddElements(QDomDocument&,const ColFileInfo&,QDomElement&) const; - void recursAddElements(QDomDocument& doc,const FileTree& ft,uint32_t index,QDomElement& e) const; + void recursAddElements( + QDomDocument& doc, const RsFileTree& ft, uint32_t index, + QDomElement& e ) const; void recursCollectColFileInfos(const QDomElement&,std::vector& colFileInfos,const QString& current_dir,bool bad_chars_in_parent) const ; // check that the file is a valid rscollection file, and not a lol bomb or some shit like this From d203f31d0cedc677113b436e69332d9b9df7ce1a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 12 Mar 2020 18:57:07 +0100 Subject: [PATCH 02/40] Improve files links generation parsing and errors Implement URL safe base64 for file links Implement sneaking file data into URL fragment Deprecate Radix65 in favore of RsBase64 which supports also URL safe encoding --- libretroshare/src/file_sharing/file_tree.cc | 29 ++- libretroshare/src/ft/ftserver.cc | 27 +-- libretroshare/src/ft/ftserver.h | 2 +- libretroshare/src/libretroshare.pro | 2 + libretroshare/src/retroshare/rsfiles.h | 55 +++--- libretroshare/src/util/radix64.h | 7 +- libretroshare/src/util/rsbase64.cc | 191 ++++++++++++++++++++ libretroshare/src/util/rsbase64.h | 139 ++++++++++++++ 8 files changed, 403 insertions(+), 49 deletions(-) create mode 100644 libretroshare/src/util/rsbase64.cc create mode 100644 libretroshare/src/util/rsbase64.h diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 3002da6c6..a33c82fe6 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -22,10 +22,12 @@ #include #include "util/radix64.h" +#include "util/rsbase64.h" #include "util/rsdir.h" #include "retroshare/rsfiles.h" #include "file_sharing_defaults.h" #include "filelist_io.h" +#include "serialiser/rstypeserializer.h" void RsFileTree::DirData::serial_process( RsGenericSerializer::SerializeJob j, @@ -45,17 +47,24 @@ void RsFileTree::FileData::serial_process( RS_SERIAL_PROCESS(hash); } -/*static*/ std::unique_ptr RsFileTree::fromBase64( - const std::string& base64 ) +/*static*/ std::tuple, std::error_condition> +RsFileTree::fromBase64(const std::string& base64) { - std::unique_ptr ft(new RsFileTree); - std::vector mem = Radix64::decode(base64); + const auto failure = [](std::error_condition ec) + { return std::make_tuple(nullptr, ec); }; + + std::error_condition ec; + std::vector mem; + if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec); + RsGenericSerializer::SerializeContext ctx( mem.data(), static_cast(mem.size()) ); + std::unique_ptr ft(new RsFileTree); ft->serial_process( RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); - if(ctx.mOk) return ft; - return nullptr; + if(ctx.mOk) return std::make_tuple(std::move(ft), std::error_condition()); + + return failure(std::errc::invalid_argument); } std::string RsFileTree::toBase64() const @@ -64,8 +73,6 @@ std::string RsFileTree::toBase64() const RsFileTree* ncThis = const_cast(this); ncThis->serial_process( RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); - RsDbg() << __PRETTY_FUNCTION__ << " ctx.mOffset: " << ctx.mOffset - << std::endl; std::vector buf(ctx.mOffset); ctx.mSize = ctx.mOffset; ctx.mOffset = 0; ctx.mData = buf.data(); @@ -73,7 +80,7 @@ std::string RsFileTree::toBase64() const ncThis->serial_process( RsGenericSerializer::SerializeJob::SERIALIZE, ctx ); std::string result; - Radix64::encode(ctx.mData, ctx.mSize, result); + RsBase64::encode(ctx.mData, ctx.mSize, result, false, true); return result; } @@ -94,7 +101,7 @@ std::unique_ptr RsFileTree::fromRadix64( std::unique_ptr ft(new RsFileTree); std::vector mem = Radix64::decode(radix64_string); if(ft->deserialise(mem.data(), static_cast(mem.size()))) - return std::move(ft); + return ft; return nullptr; } @@ -353,3 +360,5 @@ bool RsFileTree::serialise(unsigned char *& buffer,uint32_t& buffer_size) const return false; } } + +RsFileTree::~RsFileTree() = default; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 8be6ac661..678ac4a02 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -72,6 +72,8 @@ "retroshare:///files"; /*static*/ const std::string RsFiles::FILES_URL_COUNT_FIELD = "filesCount"; /*static*/ const std::string RsFiles::FILES_URL_DATA_FIELD = "filesData"; +// Use a 5 character word so there is no mistake it isn't base64 as 5%4=1 +/*static*/ const std::string RsFiles::FILES_URL_FAGMENT_FORWARD = "fragm"; /*static*/ const std::string RsFiles::FILES_URL_NAME_FIELD = "filesName"; /*static*/ const std::string RsFiles::FILES_URL_SIZE_FIELD = "filesSize"; @@ -2118,8 +2120,7 @@ const noexcept { switch(static_cast(ev)) { - case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: // [[fallthrough]]; - case RsFilesErrorNum::INVALID_FILES_URL: + case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: return std::errc::invalid_argument; default: return std::error_condition(ev, *this); @@ -2127,7 +2128,8 @@ const noexcept } std::error_condition ftServer::exportFilesLink( - std::string& link, std::uintptr_t handle, const std::string& baseUrl ) + std::string& link, std::uintptr_t handle, bool fragSneak, + const std::string& baseUrl ) { DirDetails tDirDet; if(!requestDirDetails(tDirDet, handle)) @@ -2142,10 +2144,13 @@ std::error_condition ftServer::exportFilesLink( RsUrl tUrl(baseUrl); tUrl.setQueryKV(FILES_URL_COUNT_FIELD, std::to_string(tFileTree->mTotalFiles) ) - .setQueryKV(FILES_URL_DATA_FIELD, link) .setQueryKV(FILES_URL_NAME_FIELD, tDirDet.name) .setQueryKV( FILES_URL_SIZE_FIELD, std::to_string(tFileTree->mTotalSize) ); + if(fragSneak) + tUrl.setQueryKV(FILES_URL_DATA_FIELD, FILES_URL_FAGMENT_FORWARD) + .setFragment(link); + else tUrl.setQueryKV(FILES_URL_DATA_FIELD, link); link = tUrl.toString(); } @@ -2159,14 +2164,10 @@ std::error_condition ftServer::parseFilesLink( rs_view_ptr radixPtr = tUrl.getQueryV(FILES_URL_DATA_FIELD); if(!radixPtr) radixPtr = &link; + else if(*radixPtr == FILES_URL_FAGMENT_FORWARD) radixPtr = &tUrl.fragment(); - std::unique_ptr tft = - RsFileTree::fromBase64(*radixPtr); - if(tft) - { - collection = *tft; - return std::error_condition(); - } - - return RsFilesErrorNum::INVALID_FILES_URL; + std::unique_ptr tft; std::error_condition ec; + std::tie(tft, ec) = RsFileTree::fromBase64(*radixPtr); + if(tft) collection = *tft; + return ec; } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 76ce8d416..f06f0f5bd 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -217,7 +217,7 @@ public: /// @see RsFiles std::error_condition exportFilesLink( - std::string& link, std::uintptr_t handle, + std::string& link, std::uintptr_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) override; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 3b5dad851..7842e03cc 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -480,6 +480,7 @@ HEADERS += util/folderiterator.h \ util/dnsresolver.h \ util/radix32.h \ util/radix64.h \ + util/rsbase64.h \ util/rsinitedptr.h \ util/rsprint.h \ util/rsstring.h \ @@ -636,6 +637,7 @@ SOURCES += util/folderiterator.cc \ util/rsrecogn.cc \ util/rstime.cc \ util/rsurl.cc \ + util/rsbase64.cc \ util/rserrno.cc equals(RS_UPNP_LIB, miniupnpc) { diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 3f1ce2eab..35d13c729 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2008 Robert Fernie * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -48,7 +49,6 @@ extern RsFiles* rsFiles; enum class RsFilesErrorNum : int32_t { FILES_HANDLE_NOT_FOUND = 2004, - INVALID_FILES_URL = 2005 }; struct RsFilesErrorCategory: std::error_category @@ -62,8 +62,6 @@ struct RsFilesErrorCategory: std::error_category { case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: return "Files handle not found"; - case RsFilesErrorNum::INVALID_FILES_URL: - return "Invalid files url"; default: return "Error message for error: " + std::to_string(ev) + " not available in category: " + name(); @@ -286,21 +284,22 @@ public: /** * @brief Create a RsFileTree from directory details - * @param dd + * @param dd directory or file details * @param remote * @param remove_top_dirs - * @return + * @return pointer to the created file-tree */ static std::unique_ptr fromDirDetails( const DirDetails& dd, bool remote, bool remove_top_dirs = true ); /** - * @brief Create a RsFileTree from Radix64 representation - * @param base64 - * @return nullptr if on failure, pointer to the created FileTree on success + * @brief Create a RsFileTree from Base64 representation + * @param base64 base64 or base64url string representation of the file-tree + * @return pointer to the parsed file-tree on success, nullptr plus error + * details on failure */ - static std::unique_ptr fromBase64( - const std::string& base64 ); + static std::tuple, std::error_condition> + fromBase64(const std::string& base64); /** @brief Convert to base64 representetion */ std::string toBase64() const; @@ -357,24 +356,25 @@ public: uint32_t mTotalFiles; uint64_t mTotalSize; - ~RsFileTree() = default; + virtual ~RsFileTree(); /** * @brief Create a RsFileTree from Radix64 representation * @deprecated kept for retrocompatibility with RetroShare-gui - * The format is not guardanted to be compatible with the new methods + * The format is not compatible with the new methods * @param radix64_string * @return nullptr if on failure, pointer to the created FileTree on success */ - RS_DEPRECATED + RS_DEPRECATED_FOR(fromBase64) static std::unique_ptr fromRadix64( const std::string& radix64_string ); /** @brief Convert to radix64 representetion * @deprecated kept for retrocompatibility with RetroShare-gui - * The format is not guardanted to be compatible with the new methods + * The format is not compatible with the new methods */ - RS_DEPRECATED std::string toRadix64() const; + RS_DEPRECATED_FOR(toBase64) + std::string toRadix64() const; private: /** @deprecated kept for retrocompatibility with RetroShare-gui */ @@ -885,6 +885,11 @@ public: /// Link query field used to store collection data @see exportFilesLink static const std::string FILES_URL_DATA_FIELD; + /** Link query FILES_URL_DATA_FIELD field value used to instruct the parser + * to look for the data into the link fragment + * @see exportFilesLink and parseFilesLink */ + static const std::string FILES_URL_FAGMENT_FORWARD; + /// Link query field used to store collection name @see exportFilesLink static const std::string FILES_URL_NAME_FIELD; @@ -896,15 +901,21 @@ public: * @jsonapi{development} * @param[out] link storage for the generated link * @param[in] handle file/directory RetroShare handle - * @param[in] baseUrl URL into which to sneak in the RetroShare link - * radix, this is primarly useful to induce applications into making the - * link clickable, or to disguise the RetroShare link into a - * "normal" looking web link. If empty the collection data link will be - * outputted in plain base64 format. + * @param[in] fragSneak when true the file data is sneaked into fragment + * instead of FILES_URL_DATA_FIELD query field, this way if using an + * http[s] link to pass around a disguised file link a misconfigured host + * attempting to visit that link with a web browser will not send the file + * data to the server thus protecting at least some of the privacy of the + * user even in a misconfiguration scenario. + * @param[in] baseUrl URL into which to sneak in the RetroShare file link + * base64, this is primarly useful to induce applications into making the + * link clickable, or to disguise the RetroShare link into a "normal" + * looking web link. If empty the collection data link will be outputted in + * plain base64 format. * @return error information if some error occurred, 0/SUCCESS otherwise */ virtual std::error_condition exportFilesLink( - std::string& link, std::uintptr_t handle, + std::string& link, std::uintptr_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0; /** diff --git a/libretroshare/src/util/radix64.h b/libretroshare/src/util/radix64.h index de6eb21f4..53dd7c256 100644 --- a/libretroshare/src/util/radix64.h +++ b/libretroshare/src/util/radix64.h @@ -26,7 +26,10 @@ #include #include -class Radix64 +#include "util/rsdeprecate.h" + +/** @deprecated use RsBase64 instead which supports also URL safe encoding */ +class RS_DEPRECATED_FOR(RsBase64) Radix64 { public: static std::vector decode(const std::string& buffer) @@ -195,5 +198,3 @@ again: return true ; } }; - - diff --git a/libretroshare/src/util/rsbase64.cc b/libretroshare/src/util/rsbase64.cc new file mode 100644 index 000000000..80e0f20d1 --- /dev/null +++ b/libretroshare/src/util/rsbase64.cc @@ -0,0 +1,191 @@ +/******************************************************************************* + * * + * libretroshare base64 encoding utilities * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include + +#include "util/rsbase64.h" +#include "util/rsdebug.h" + +#if __cplusplus < 201703L +/* Solve weird undefined reference error with C++ < 17 see: + * https://stackoverflow.com/questions/8016780/undefined-reference-to-static-constexpr-char + */ +/*static*/ decltype(RsBase64::bDict) constexpr RsBase64::bDict; +/*static*/ decltype(RsBase64::uDict) constexpr RsBase64::uDict; +/*static*/ decltype(RsBase64::rDict) constexpr RsBase64::rDict; +/*static*/ decltype(RsBase64::sPad) constexpr RsBase64::sPad; +#endif + +/*static*/ void RsBase64::encode( + rs_view_ptr data, size_t len, std::string& outString, + bool padding, bool urlSafe ) +{ + const char* sDict = urlSafe ? uDict : bDict; + + // Workaround if input and output are the same buffer. + bool inplace = (outString.data() == reinterpret_cast(data)); + std::string tBuff; + std::string& outStr = inplace ? tBuff : outString; + + auto encSize = encodedSize(len, padding); + outStr.resize(encSize); + char* p = &outStr[0]; + + for (; len >= 3; len -= 3, data += 3) + { + *p++ = sDict[ (data[0] >> 2) & 077 ]; + *p++ = sDict[ + (((data[0] << 4) & 060) | ((data[1] >> 4) & 017)) & 077 ]; + *p++ = sDict[ + (((data[1] << 2) & 074) | ((data[2] >> 6) & 03)) & 077 ]; + *p++ = sDict[ data[2] & 077 ]; + } + if (len == 2) + { + *p++ = sDict[ (data[0] >> 2) & 077 ]; + *p++ = sDict[ + (((data[0] << 4) & 060) | ((data[1] >> 4) & 017)) & 077 ]; + *p++ = sDict[ ((data[1] << 2) & 074) ]; + if(padding) *p++ = sPad; + } + else if (len == 1) + { + *p++ = sDict[ (data[0] >> 2) & 077 ]; + *p++ = sDict[ (data[0] << 4) & 060 ]; + if(padding) { *p++ = sPad; *p++ = sPad; } + } + + if(inplace) outString = tBuff; +} + +/*static*/ std::error_condition RsBase64::decode( + const std::string& encoded, std::vector& decoded ) +{ + size_t decSize; std::error_condition ec; + std::tie(decSize, ec) = decodedSize(encoded); + if(!decSize || ec) return ec; + + size_t encSize = encoded.size(); + decoded.resize(decSize); + + for (size_t i = 0, o = 0; i < encSize; i += 4, o += 3) + { + char input0 = encoded[i + 0]; + char input1 = encoded[i + 1]; + + /* At the end of the string, missing bytes 2 and 3 are considered + * padding '=' */ + char input2 = i + 2 < encoded.size() ? encoded[i + 2] : sPad; + char input3 = i + 3 < encSize ? encoded[i + 3] : sPad; + + // If any unknown characters appear, it's an error. + if(!( isBase64Char(input0) && isBase64Char(input1) && + isBase64Char(input2) && isBase64Char(input3) )) + return std::errc::argument_out_of_domain; + + /* If padding appears anywhere but the last 1 or 2 characters, or if + * it appears but encoded.size() % 4 != 0, it's an error. */ + bool at_end = (i + 4 >= encSize); + if ( (input0 == sPad) || (input1 == sPad) || + ( input2 == sPad && !at_end ) || + ( input2 == sPad && input3 != sPad ) || + ( input3 == sPad && !at_end) ) + return std::errc::illegal_byte_sequence; + + uint32_t b0 = rDict[static_cast(input0)] & 0x3f; + uint32_t b1 = rDict[static_cast(input1)] & 0x3f; + uint32_t b2 = rDict[static_cast(input2)] & 0x3f; + uint32_t b3 = rDict[static_cast(input3)] & 0x3f; + + uint32_t stream = (b0 << 18) | (b1 << 12) | (b2 << 6) | b3; + decoded[o + 0] = (stream >> 16) & 0xFF; + if (input2 != sPad) decoded[o + 1] = (stream >> 8) & 0xFF; + /* If there are any stale bits in this from input1, the text is + * malformed. */ + else if (((stream >> 8) & 0xFF) != 0) + return std::errc::invalid_argument; + + if (input3 != sPad) decoded[o + 2] = (stream >> 0) & 0xFF; + /* If there are any stale bits in this from input2, the text is + * malformed. */ + else if (((stream >> 0) & 0xFF) != 0) + return std::errc::invalid_argument; + } + + return std::error_condition(); +} + +/*static*/ size_t RsBase64::encodedSize(size_t decodedSize, bool padding) +{ + if(padding) return 4 * (decodedSize + 2) / 3; + return static_cast( + std::ceil(4L * static_cast(decodedSize) / 3L) ); +} + +/*static*/ std::tuple RsBase64::decodedSize( + const std::string& input ) +{ + const auto success = [](size_t val) + { return std::make_tuple(val, std::error_condition()); }; + + if(input.empty()) return success(0); + + auto mod = input.size() % 4; + if(mod == 1) std::make_tuple(0, std::errc::invalid_argument); + + size_t padded_size = ((input.size() + 3) / 4) * 3; + if (mod >= 2 || (mod == 0 && input[input.size() - 1] == sPad)) + { + /* If the last byte is '=', or the input size % 4 is 2 or 3 (thus + * there are implied '='s), then the actual size is 1-2 bytes + * smaller. */ + if ( mod == 2 || (mod == 0 && input[input.size() - 2] == sPad) ) + { + /* If the second-to-last byte is also '=', or the input + * size % 4 is 2 (implying a second '='), then the actual size + * is 2 bytes smaller. */ + return success(padded_size - 2); + } + else + { + /* Otherwise it's just the last character and the actual size is + * 1 byte smaller. */ + return success(padded_size - 1); + } + } + return success(padded_size); +} + +/*static*/ size_t RsBase64::stripInvalid( + const std::string& in, std::string& out ) +{ + size_t strippedCnt = 0; + auto inSize = in.size(); + out.resize(inSize); + for(size_t i = 0; i < inSize; ++i) + { + if(isBase64Char(in[i])) out[i-strippedCnt] = in[i]; + else ++strippedCnt; + } + out.resize(inSize-strippedCnt); + return strippedCnt; +} diff --git a/libretroshare/src/util/rsbase64.h b/libretroshare/src/util/rsbase64.h new file mode 100644 index 000000000..b11ec08cc --- /dev/null +++ b/libretroshare/src/util/rsbase64.h @@ -0,0 +1,139 @@ +/******************************************************************************* + * * + * libretroshare base64 encoding utilities * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include + +#include "util/rsmemory.h" + +/** + * Implement methods to encode e decode to base64 format as per RFC 4648 + * This implementation support also the file name and URL safe base64url format + * @see https://tools.ietf.org/html/rfc4648#section-5 + */ +class RsBase64 +{ +public: + /// Enable base64url by default + static constexpr bool DEFAULT_URL_SAFE = true; + + /// Disable padding by default + static constexpr bool DEFAULT_PADDING = false; + + /** + * @brief Encode arbitrary data to base64 + * @param[in] data pointer to the input data buffer + * @param[in] len lenght of the input buffer + * @param[out] outString storage for the resulting base64 encoded string + * @param[in] padding set to true to enable padding to 32 bits + * @param[in] urlSafe pass true for base64url format, false for base64 format + */ + static void encode( + rs_view_ptr data, size_t len, + std::string& outString, + bool padding = DEFAULT_PADDING, bool urlSafe = DEFAULT_URL_SAFE ); + + /** + * @brief Decode data from a base64 encoded string + * @param[in] encoded encoded string + * @param[out] decoded storage for decoded data + * @return success or error details + */ + static std::error_condition decode( + const std::string& encoded, std::vector& decoded ); + + /** + * Remove invalid characters from base64 encoded string. + * Often when copy and pasting from one progam to another long base64 + * strings, new lines, spaces or other characters end up polluting the + * original text. This function is useful to cleanup the pollution before + * attempting to decode the message. + * @param in input string + * @param out storage for cleaned string. In-place operation in supported so + * the same input string may be passed. + * @return count of stripped invalid characters + */ + static size_t stripInvalid(const std::string& in, std::string& out); + + /** + * Calculate how much bytes are needed to store the base64 encoded version + * of some data. + * @param decodedSize size of the original decoded data + * @param padding true to enable base64 padding + * @return how much bytes would take to store the encoded version + */ + static size_t encodedSize( + size_t decodedSize, bool padding = DEFAULT_PADDING ); + + /** + * @brief Calculate how much space is needed to store the decoded version of + * a base64 encoded string + * @param input encoded string + * @return decoded size, plus error information on failure + */ + static std::tuple decodedSize( + const std::string& input ); + +private: + /// base64 conversion table + static constexpr char bDict[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /// base64url conversion table + static constexpr char uDict[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + + /// This reverse table supports both base64 and base64url + static constexpr int8_t rDict[256] = { + /* index +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 */ + /* 0 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 16 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 32 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, 62, -1, 63, + /* 48 */ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, 0, -1, -1, + /* 64 */ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + /* 80 */ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, + /* 96 */ -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + /* 112 */ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + /* 128 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 144 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 160 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 176 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 192 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 208 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 224 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + /* 240 */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /// base64 padding character + static constexpr char sPad = '='; + + /** Check if given character is valid either for base64 or for base64url + * @param c character to check + * @return true if valid false otherwise + */ + static inline bool isBase64Char(char c) + { return rDict[static_cast(c)] >= 0; } +}; From 39bde58c2949538508cc0cb7725c497abb06bf9a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 16 Mar 2020 16:20:06 +0100 Subject: [PATCH 03/40] Many serialization and related stuff improvements Fix bug in array-like containers serialization which could cause almost infinite loop on malformed input Implement VLQ integer serialization Unify sequence containers serialization code Add support for VLQ serialization also for string size Use VLQ compression for file links Add templated function to fix endiannes for all integer types Use bitset to print flags in binary form Unify serialization code for integral types Serialize 64bit integers types to JSON object with both string and integer representation, so it is posible to have this representation also for containers types like std::vetor or std::map this breaks retrocompatibility but is necessary to support clients written in languages which doesn't have 64 bit integers support such as JavaScript or Dart --- libretroshare/src/file_sharing/file_tree.cc | 8 +- libretroshare/src/libretroshare.pro | 1 + libretroshare/src/retroshare/rsfiles.h | 4 - libretroshare/src/retroshare/rsflags.h | 33 +- libretroshare/src/serialiser/rsserializer.h | 37 +- .../src/serialiser/rstypeserializer.cc | 610 +++++------ .../src/serialiser/rstypeserializer.h | 981 +++++++++++------- libretroshare/src/util/rsdebug.h | 4 + libretroshare/src/util/rsendian.h | 59 ++ 9 files changed, 985 insertions(+), 752 deletions(-) create mode 100644 libretroshare/src/util/rsendian.h diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index a33c82fe6..2dbea4cc4 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2018 by Retroshare Team * + * Copyright (C) 2018 Retroshare Team * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -58,7 +60,8 @@ RsFileTree::fromBase64(const std::string& base64) if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec); RsGenericSerializer::SerializeContext ctx( - mem.data(), static_cast(mem.size()) ); + mem.data(), static_cast(mem.size()), + SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ) ); std::unique_ptr ft(new RsFileTree); ft->serial_process( RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); @@ -70,6 +73,7 @@ RsFileTree::fromBase64(const std::string& base64) std::string RsFileTree::toBase64() const { RsGenericSerializer::SerializeContext ctx; + ctx.mFlags = SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ); RsFileTree* ncThis = const_cast(this); ncThis->serial_process( RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 7842e03cc..a923475d5 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -481,6 +481,7 @@ HEADERS += util/folderiterator.h \ util/radix32.h \ util/radix64.h \ util/rsbase64.h \ + util/rsendian.h \ util/rsinitedptr.h \ util/rsprint.h \ util/rsstring.h \ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 35d13c729..ae5c00dea 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -309,10 +309,6 @@ public: RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) { - /* TODO: most of the time handles are smaller then 64bit use VLQ for - * binary serialization of those - * https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding - */ RS_SERIAL_PROCESS(mFiles); RS_SERIAL_PROCESS(mDirs); RS_SERIAL_PROCESS(mTotalFiles); diff --git a/libretroshare/src/retroshare/rsflags.h b/libretroshare/src/retroshare/rsflags.h index 4222855fa..37320f08b 100644 --- a/libretroshare/src/retroshare/rsflags.h +++ b/libretroshare/src/retroshare/rsflags.h @@ -24,6 +24,7 @@ #include #include +#include /** Check if given type is a scoped enum */ template @@ -128,13 +129,7 @@ typename std::enable_if::enabled, std::ostream>::type& operator <<(std::ostream& stream, EFT flags) { using u_t = typename std::underlying_type::type; - - for(int i = sizeof(u_t); i>=0; --i) - { - stream << (flags & ( 1 << i ) ? "1" : "0"); - if( i % 8 == 0 ) stream << " "; - } - return stream; + return stream << std::bitset(static_cast(flags)); } #include @@ -170,6 +165,7 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 inline t_RsFlags32() : _bits(0) {} inline explicit t_RsFlags32(uint32_t N) : _bits(N) {} // allows initialization from a set of uint32_t + inline t_RsFlags32 operator| (const t_RsFlags32& f) const { return t_RsFlags32(_bits | f._bits) ; } inline t_RsFlags32 operator^ (const t_RsFlags32& f) const { return t_RsFlags32(_bits ^ f._bits) ; } inline t_RsFlags32 operator* (const t_RsFlags32& f) const { return t_RsFlags32(_bits & f._bits) ; } @@ -187,6 +183,19 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 //inline explicit operator bool() const { return _bits>0; } inline uint32_t toUInt32() const { return _bits ; } + /// Easier porting to new flag system + template inline + typename std::enable_if<(Rs__BitFlagsOps::enabled && + sizeof(EFT) >= sizeof(uint32_t) ), EFT>::type + toEFT() { return static_cast(_bits); } + + /// Easier porting to new flag system + template + static inline typename std::enable_if< + Rs__BitFlagsOps::enabled && + sizeof(EFT) <= sizeof(uint32_t), t_RsFlags32>::type + fromEFT(EFT e) { return t_RsFlags32(static_cast(e)); } + void clear() { _bits = 0 ; } friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32& f) // friendly print with 0 and I @@ -199,8 +208,10 @@ template class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32 } return o ; } - private: - uint32_t _bits ; + +private: + friend struct RsTypeSerializer; + uint32_t _bits; }; #define FLAGS_TAG_TRANSFER_REQS 0x4228af @@ -230,7 +241,7 @@ typedef t_RsFlags32 ServicePermissionFlags ; // typedef t_RsFlags32 ChatLobbyFlags ; -// Flags for serializer -// +/// @deprecated Flags for serializer +RS_DEPRECATED_FOR(RsSerializationFlags) typedef t_RsFlags32 SerializationFlags ; diff --git a/libretroshare/src/serialiser/rsserializer.h b/libretroshare/src/serialiser/rsserializer.h index e9b60a0aa..60f4a21d8 100644 --- a/libretroshare/src/serialiser/rsserializer.h +++ b/libretroshare/src/serialiser/rsserializer.h @@ -192,6 +192,39 @@ class RsRawSerialiser: public RsSerialType virtual RsItem * deserialise(void *data, uint32_t *size); }; +/** These are convenience flags to be used by the items when processing the + * data. The names of the flags are not very important. What matters is that + * the serial_process() method of each item correctly deals with the data + * when it sees the flags, if the serialiser sets them. + * By default the flags are not set and shouldn't be handled. + * When deriving a new serializer, the user can set his own flags, using + * compatible values + */ +enum class RsSerializationFlags +{ + NONE = 0, + CONFIG = 1, + SIGNATURE = 2, + SKIP_HEADER = 4, + + /** Used for JSON deserialization in JSON API, it causes the deserialization + * to continue even if some field is missing (or incorrect), this way the + * API is more user friendly as some methods need just part of the structs + * they take as parameters. */ + YIELDING = 8, + + /** When set integers typer are serialized/deserialized in Variable Length + * Quantity mode + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * This type of encoding is efficent when absoulte value is usually much + * smaller then the maximum representable with the original type. + * This encoding is also capable of representing big values at expences of a + * one more byte used. + */ + INTEGER_VLQ = 16 +}; +RS_REGISTER_ENUM_FLAGS_TYPE(RsSerializationFlags); + /// Top class for all services and config serializers. struct RsGenericSerializer : RsSerialType { @@ -200,7 +233,7 @@ struct RsGenericSerializer : RsSerialType SIZE_ESTIMATE = 0x01, SERIALIZE = 0x02, DESERIALIZE = 0x03, - PRINT = 0x04, + PRINT = 0x04, /// @deprecated use rsdebug.h << operator instead TO_JSON, FROM_JSON } SerializeJob; @@ -227,7 +260,7 @@ struct RsGenericSerializer : RsSerialType SerializationFlags flags, RsJson::AllocatorType* allocator = nullptr) : mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format), - mFlags(flags), mJson(rapidjson::kObjectType, allocator) {} + mFlags{flags}, mJson(rapidjson::kObjectType, allocator) {} unsigned char *mData; uint32_t mSize; diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 1b086576a..d57f3d0d4 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2017 Cyril Soler * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -25,7 +26,7 @@ #include "serialiser/rsbaseserial.h" #include "serialiser/rstlvkeys.h" #include "serialiser/rsserializable.h" -#include "util/radix64.h" +#include "util/rsbase64.h" #include "util/rsprint.h" #include "util/rstime.h" @@ -34,8 +35,6 @@ #include // for typeid #include -static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB. - #ifdef RSSERIAL_DEBUG # define SAFE_GET_JSON_V() \ const char* mName = memberName.c_str(); \ @@ -56,32 +55,10 @@ static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB. rapidjson::Value& v = jDoc[mName] #endif // ifdef RSSERIAL_DEBUG - //============================================================================// // std::string // //============================================================================// -template<> uint32_t RsTypeSerializer::serial_size(const std::string& str) -{ - return getRawStringSize(str); -} -template<> bool RsTypeSerializer::serialize( uint8_t data[], uint32_t size, - uint32_t& offset, - const std::string& str ) -{ - return setRawString(data, size, &offset, str); -} -template<> bool RsTypeSerializer::deserialize( const uint8_t data[], - uint32_t size, uint32_t &offset, - std::string& str ) -{ - return getRawString(data, size, &offset, str); -} -template<> void RsTypeSerializer::print_data( const std::string& n, - const std::string& str ) -{ - std::cerr << " [std::string] " << n << ": " << str << std::endl; -} template<> /*static*/ bool RsTypeSerializer::to_JSON( const std::string& membername, const std::string& member, RsJson& jDoc ) @@ -112,135 +89,11 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return ret; } + //============================================================================// -// Integer types // +// Integral types // //============================================================================// -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const bool& member) -{ - return setRawUInt8(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, const int32_t& /*member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return false; -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint8_t& member) -{ - return setRawUInt8(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint16_t& member) -{ - return setRawUInt16(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint32_t& member) -{ - return setRawUInt32(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const uint64_t& member) -{ - return setRawUInt64(data,size,&offset,member); -} -template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const rstime_t& member) -{ - return setRawTimeT(data,size,&offset,member); -} - -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, bool& member) -{ - uint8_t m; - bool ok = getRawUInt8(data,size,&offset,&m); - member = m; - return ok; -} -template<> bool RsTypeSerializer::deserialize(const uint8_t /*data*/[], uint32_t /*size*/, uint32_t& /*offset*/, int32_t& /*member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return false; -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint8_t& member) -{ - return getRawUInt8(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t& member) -{ - return getRawUInt16(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint32_t& member) -{ - return getRawUInt32(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, uint64_t& member) -{ - return getRawUInt64(data,size,&offset,&member); -} -template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, rstime_t& member) -{ - return getRawTimeT(data,size,&offset,member); -} - -template<> uint32_t RsTypeSerializer::serial_size(const bool& /* member*/) -{ - return 1; -} -template<> uint32_t RsTypeSerializer::serial_size(const int32_t& /* member*/) -{ - std::cerr << __PRETTY_FUNCTION__ << " Not implemented!" << std::endl; - print_stacktrace(); - return 0; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint8_t& /* member*/) -{ - return 1; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint16_t& /* member*/) -{ - return 2; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint32_t& /* member*/) -{ - return 4; -} -template<> uint32_t RsTypeSerializer::serial_size(const uint64_t& /* member*/) -{ - return 8; -} -template<> uint32_t RsTypeSerializer::serial_size(const rstime_t& /* member*/) -{ - return 8; -} - -template<> void RsTypeSerializer::print_data(const std::string& n, const bool & V) -{ - std::cerr << " [bool ] " << n << ": " << V << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const int32_t& V) -{ - std::cerr << " [int32_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint8_t & V) -{ - std::cerr << " [uint8_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint16_t& V) -{ - std::cerr << " [uint16_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint32_t& V) -{ - std::cerr << " [uint32_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const uint64_t& V) -{ - std::cerr << " [uint64_t ] " << n << ": " << std::to_string(V) << std::endl; -} -template<> void RsTypeSerializer::print_data(const std::string& n, const rstime_t& V) -{ - std::cerr << " [rstime_t ] " << n << ": " << V << " (" << time(NULL)-V << " secs ago)" << std::endl; -} - #define SIMPLE_TO_JSON_DEF(T) \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ const T& member, RsJson& jDoc ) \ @@ -268,28 +121,45 @@ SIMPLE_TO_JSON_DEF(uint32_t) /** Be very careful in changing this constant as it would break 64 bit integers * members JSON string representation retrocompatibility */ -static constexpr char strReprSuffix[] = "_sixtyfour_str"; +static constexpr char strReprKey[] = "xstr64"; -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we add - * to JSON also the string representation for this types as a workaround for the - * sake of JavaScript clients @see https://stackoverflow.com/a/34989371 +/** Be very careful in changing this constant as it would break 64 bit integers + * members JSON string representation retrocompatibility */ +static constexpr char intReprKey[] = "xint64"; + +/** While JSON doesn't have problems representing 64 bits integers JavaScript, + * Dart and other languages represents numbers in a double-like format thus they + * are not capable to handle safely integers outside the range + * [-(2^53 - 1), 2^53 - 1]. + * To overcome this limitation we represent 64 bit integers as an object with + * two keys, one as integer and one as string representation. + * In our case we need to wrap those into an object instead of just adding a key + * with a suffix so support well also containers like std::map or std::vector. + * More discussion on the topic at @see https://stackoverflow.com/a/34989371 */ #define SIXTYFOUR_INTEGERS_TO_JSON_DEF(T) \ template<> bool RsTypeSerializer::to_JSON( const std::string& memberName, \ const T& member, RsJson& jDoc ) \ { \ - rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); \ + using namespace rapidjson; \ + Document::AllocatorType& allocator = jDoc.GetAllocator(); \ \ - rapidjson::Value key; \ + Document wrapper(rapidjson::kObjectType, &allocator); \ + \ + Value intKey; \ + intKey.SetString(intReprKey, allocator ); \ + Value intValue(member); \ + wrapper.AddMember(intKey, intValue, allocator); \ + \ + bool ok = to_JSON(strReprKey, std::to_string(member), wrapper); \ + \ + Value key; \ key.SetString( memberName.c_str(), \ static_cast(memberName.length()), \ allocator ); \ - rapidjson::Value value(member); \ - jDoc.AddMember(key, value, allocator); \ + jDoc.AddMember(key, wrapper, allocator); \ \ - return to_JSON(memberName + strReprSuffix, std::to_string(member), jDoc); \ + return ok; \ } SIXTYFOUR_INTEGERS_TO_JSON_DEF(int64_t); @@ -345,100 +215,71 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return ret; } -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look - * for the string representation in the JSON for this types as a workaround for - * the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 - */ -template<> /*static*/ -bool RsTypeSerializer::from_JSON( - const std::string& memberName, int64_t& member, RsJson& jDoc ) -{ - const char* mName = memberName.c_str(); - if(jDoc.HasMember(mName)) - { - rapidjson::Value& v = jDoc[mName]; - if(v.IsInt64()) - { - member = v.GetInt64(); - return true; - } - } - - Dbg4() << __PRETTY_FUNCTION__ << " int64_t " << memberName << " not found " - << "in JSON then attempt to look for string representation" - << std::endl; - - const std::string str_key = memberName + strReprSuffix; - std::string str_value; - if(from_JSON(str_key, str_value, jDoc)) - { - try { member = std::stoll(str_value); } - catch (...) - { - RsErr() << __PRETTY_FUNCTION__ << " cannot convert " - << str_value << " to int64_t" << std::endl; - return false; - } - - return true; - } - - Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its " - << "string representation " << str_key << " has been found " - << "in JSON" << std::endl; - - return false; -} - -/** While JSON doesn't have problems representing 64 bit integers JavaScript - * standard represents numbers in a double-like format thus it is not capable to - * handle safely integers outside the range [-(2^53 - 1), 2^53 - 1], so we look - * for the string representation in the JSON for this types as a workaround for - * the sake of JavaScript clients @see https://stackoverflow.com/a/34989371 - */ -template<> /*static*/ -bool RsTypeSerializer::from_JSON( - const std::string& memberName, uint64_t& member, RsJson& jDoc ) -{ - const char* mName = memberName.c_str(); - if(jDoc.HasMember(mName)) - { - rapidjson::Value& v = jDoc[mName]; - if(v.IsUint64()) - { - member = v.GetUint64(); - return true; - } - } - - Dbg4() << __PRETTY_FUNCTION__ << " uint64_t " << memberName << " not found " - << "in JSON then attempt to look for string representation" - << std::endl; - - const std::string str_key = memberName + strReprSuffix; - std::string str_value; - if(from_JSON(str_key, str_value, jDoc)) - { - try { member = std::stoull(str_value); } - catch (...) - { - RsErr() << __PRETTY_FUNCTION__ << " cannot convert " - << str_value << " to uint64_t" << std::endl; - return false; - } - - return true; - } - - Dbg3() << __PRETTY_FUNCTION__ << " neither " << memberName << " nor its " - << "string representation " << str_key << " has been found " - << "in JSON" << std::endl; - - return false; +/** inverse of @see SIXTYFOUR_INTEGERS_TO_JSON_DEF */ +#define SIXTYFOUR_INTEGERS_FROM_JSON_DEF(T, PRED, GET, CONV) \ +template<> bool RsTypeSerializer::from_JSON( \ + const std::string& memberName, T& member, RsJson& jDoc ) \ +{ \ + using namespace rapidjson; \ + \ + SAFE_GET_JSON_V(); \ + \ + /* For retro-compatibility take it directly if it is passed as integer */ \ + if(v.PRED()) \ + { \ + member = v.GET(); \ + return true; \ + } \ + \ + ret = ret && v.IsObject(); \ + \ + if(!ret) \ + { \ + Dbg3() << __PRETTY_FUNCTION__ << " " << memberName << " not found" \ + << std::endl; \ + return false; \ + } \ + \ + if(v.HasMember(intReprKey)) \ + { \ + Value& iVal = v[intReprKey]; \ + if(iVal.PRED()) \ + { \ + member = iVal.GET(); \ + return true; \ + } \ + } \ + \ + Dbg4() << __PRETTY_FUNCTION__ << " integer representation of " << memberName \ + << " not found in JSON then attempt to look for string representation" \ + << std::endl; \ + \ + \ + if(v.HasMember(strReprKey)) \ + { \ + Value& sVal = v[strReprKey]; \ + if(sVal.IsString()) \ + { \ + try { member = CONV(sVal.GetString()); } \ + catch (...) \ + { \ + RsErr() << __PRETTY_FUNCTION__ << " cannot convert " \ + << sVal.GetString() << " to integral type" << std::endl; \ + return false; \ + } \ + \ + return true; \ + } \ + } \ + \ + Dbg3() << __PRETTY_FUNCTION__ << " neither integral representation nor " \ + << "string representation found for: " << memberName << std::endl; \ + \ + return false; \ } +SIXTYFOUR_INTEGERS_FROM_JSON_DEF(uint64_t, IsUint64, GetUint64, std::stoull) +SIXTYFOUR_INTEGERS_FROM_JSON_DEF( int64_t, IsInt64, GetInt64, std::stoll) //============================================================================// // Floats // @@ -662,113 +503,170 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/, // Binary blocks // //============================================================================// -template<> uint32_t RsTypeSerializer::serial_size(const RsTypeSerializer::TlvMemBlock_proxy& r) { return 4 + r.second ; } - -template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,RsTypeSerializer::TlvMemBlock_proxy& r) +void RsTypeSerializer::RawMemoryWrapper::serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) { - uint32_t saved_offset = offset ; - - bool ok = deserialize(data,size,offset,r.second) ; - - if(r.second == 0) - { - r.first = NULL ; - - if(!ok) - offset = saved_offset ; - - return ok ; - } - if(r.second > MAX_SERIALIZED_CHUNK_SIZE) - { - std::cerr << "(EE) RsTypeSerializer::deserialize(): data chunk has size larger than safety size (" << MAX_SERIALIZED_CHUNK_SIZE << "). Item will be dropped." << std::endl; - offset = saved_offset ; - return false ; - } - - r.first = (uint8_t*)rs_malloc(r.second) ; - - ok = ok && (NULL != r.first); - - memcpy(r.first,&data[offset],r.second) ; - offset += r.second ; - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const RsTypeSerializer::TlvMemBlock_proxy& r) -{ - uint32_t saved_offset = offset ; - - bool ok = serialize(data,size,offset,r.second) ; - - memcpy(&data[offset],r.first,r.second) ; - offset += r.second ; - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> void RsTypeSerializer::print_data(const std::string& n, const RsTypeSerializer::TlvMemBlock_proxy& s) -{ - std::cerr << " [Binary data] " << n << ", length=" << s.second << " data=" << RsUtil::BinToHex((uint8_t*)s.first,std::min(50u,s.second)) << ((s.second>50)?"...":"") << std::endl; -} - -template<> /*static*/ -bool RsTypeSerializer::to_JSON( - const std::string& memberName, - const RsTypeSerializer::TlvMemBlock_proxy& member, RsJson& jDoc ) -{ - rapidjson::Document::AllocatorType& allocator = jDoc.GetAllocator(); - - rapidjson::Value key; - key.SetString( memberName.c_str(), - static_cast(memberName.length()), - allocator ); - - std::string encodedValue; - Radix64::encode( reinterpret_cast(member.first), - static_cast(member.second), encodedValue ); - - rapidjson::Value value; - value.SetString(encodedValue.data(), allocator); - - jDoc.AddMember(key, value, allocator); - - return true; -} - -template<> /*static*/ -bool RsTypeSerializer::from_JSON( const std::string& memberName, - RsTypeSerializer::TlvMemBlock_proxy& member, - RsJson& jDoc) -{ - SAFE_GET_JSON_V(); - ret = ret && v.IsString(); - if(ret) + switch(j) { - std::string encodedValue = v.GetString(); - std::vector decodedValue = Radix64::decode(encodedValue); - member.second = decodedValue.size(); - - if(member.second == 0) + case RsGenericSerializer::SIZE_ESTIMATE: + RS_SERIAL_PROCESS(second); + ctx.mOffset += second; + break; + case RsGenericSerializer::SERIALIZE: + if(!ctx.mOk) break; + if(second > MAX_SERIALIZED_CHUNK_SIZE) { - member.first = nullptr; - return ret; + RsErr() << __PRETTY_FUNCTION__ + << std::errc::message_size << " " + << second << " > " << MAX_SERIALIZED_CHUNK_SIZE + << std::endl; + print_stacktrace(); + break; + } + RS_SERIAL_PROCESS(second); + if(!ctx.mOk) break; + ctx.mOk = ctx.mSize >= ctx.mOffset + second; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + print_stacktrace(); + break; + } + memcpy(ctx.mData + ctx.mOffset, first, second); + ctx.mOffset += second; + break; + case RsGenericSerializer::DESERIALIZE: + { + uint32_t serialSize = 0; + RS_SERIAL_PROCESS(serialSize); + if(!ctx.mOk) break; + ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ + << std::errc::message_size << " " + << serialSize << " > " << MAX_SERIALIZED_CHUNK_SIZE + << std::endl; + clear(); + break; } - member.first = rs_malloc(member.second); - ret = ret && member.first && - memcpy(member.first, decodedValue.data(), member.second); + if(!serialSize) + { + clear(); + break; + } + + ctx.mOk = ctx.mSize >= ctx.mOffset + serialSize; + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + print_stacktrace(); + + clear(); + break; + } + + if(serialSize != second) + { + first = reinterpret_cast(realloc(first, serialSize)); + second = static_cast(serialSize); + } + + memcpy(first, ctx.mData + ctx.mOffset, second); + ctx.mOffset += second; + break; + } + case RsGenericSerializer::PRINT: break; + case RsGenericSerializer::TO_JSON: + { + if(!ctx.mOk) break; + std::string encodedValue; + RsBase64::encode(first, second, encodedValue, true, false); + ctx.mJson.SetString( + encodedValue.data(), + static_cast(encodedValue.length()) ); + break; + } + case RsGenericSerializer::FROM_JSON: + { + const bool yelding = !!( RsSerializationFlags::YIELDING & + ctx.mFlags.toEFT() ); + if(!(ctx.mOk || yelding)) + { + clear(); + break; + } + if(!ctx.mJson.IsString()) + { + RsErr() << __PRETTY_FUNCTION__ << " " + << std::errc::invalid_argument << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + if( ctx.mJson.GetStringLength() > + RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) ) + { + RsErr() << __PRETTY_FUNCTION__ << " " + << std::errc::message_size << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + + std::string encodedValue = ctx.mJson.GetString(); + std::vector decoded; + auto ec = RsBase64::decode(encodedValue, decoded); + if(ec) + { + RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl; + print_stacktrace(); + + ctx.mOk = false; + clear(); + break; + } + + const auto decodedSize = decoded.size(); + + if(!decodedSize) + { + clear(); + break; + } + + if(decodedSize != second) + { + first = reinterpret_cast(realloc(first, decodedSize)); + second = static_cast(decodedSize); + } + + memcpy(first, decoded.data(), second); + break; + } + default: RsTypeSerializer::fatalUnknownSerialJob(j); } - return ret; } +void RsTypeSerializer::RawMemoryWrapper::clear() +{ + free(first); + first = nullptr; + second = 0; +} + +//============================================================================// +// std::error_condition // +//============================================================================// + void RsTypeSerializer::ErrConditionWrapper::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) @@ -778,12 +676,12 @@ void RsTypeSerializer::ErrConditionWrapper::serial_process( case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough case RsGenericSerializer::DESERIALIZE: // fallthrough case RsGenericSerializer::SERIALIZE: // fallthrough - case RsGenericSerializer::FROM_JSON: + case RsGenericSerializer::FROM_JSON: // [[fallthrough]] + case RsGenericSerializer::PRINT: RsFatal() << __PRETTY_FUNCTION__ << " SerializeJob: " << j << "is not supported on std::error_condition " << std::endl; print_stacktrace(); exit(-2); - case RsGenericSerializer::PRINT: // fallthrough case RsGenericSerializer::TO_JSON: { constexpr RsGenericSerializer::SerializeJob rj = diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 55568632a..a8d61fdeb 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2017 Cyril Soler * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -24,125 +25,159 @@ #include // for typeid #include -#include +#include #include - +#include +#include #include "serialiser/rsserial.h" #include "serialiser/rstlvbase.h" #include "serialiser/rstlvlist.h" - #include "retroshare/rsflags.h" #include "retroshare/rsids.h" - +#include "util/rsendian.h" #include "serialiser/rsserializer.h" #include "serialiser/rsserializable.h" #include "util/rsjson.h" #include "util/rsdebug.h" -/* INTERNAL ONLY helper to avoid copy paste code for std::{vector,list,set} - * Can't use a template function because T is needed for const_cast */ -#define RsTypeSerializer_PRIVATE_TO_JSON_ARRAY() do \ -{ \ - using namespace rapidjson; \ -\ - Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); \ -\ - Value arrKey; arrKey.SetString(memberName.c_str(), allocator); \ -\ - Value arr(kArrayType); \ -\ - for (auto& el : v) \ - { \ - /* Use same allocator to avoid deep copy */\ - RsGenericSerializer::SerializeContext elCtx(\ - nullptr, 0, ctx.mFlags, &allocator );\ -\ - /* If el is const the default serial_process template is matched */ \ - /* also when specialization is necessary so the compilation break */ \ - serial_process(j, elCtx, const_cast(el), memberName); \ -\ - elCtx.mOk = elCtx.mOk && elCtx.mJson.HasMember(arrKey);\ - if(elCtx.mOk) arr.PushBack(elCtx.mJson[arrKey], allocator);\ - else\ - {\ - ctx.mOk = false;\ - break;\ - }\ - }\ -\ - ctx.mJson.AddMember(arrKey, arr, allocator);\ -} while (false) - -/* INTERNAL ONLY helper to avoid copy paste code for std::{vector,list,set} - * Can't use a template function because std::{vector,list,set} has different - * name for insert/push_back function - */ -#define RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(INSERT_FUN) do\ -{\ - using namespace rapidjson;\ -\ - bool ok = ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING;\ - Document& jDoc(ctx.mJson);\ - Document::AllocatorType& allocator = jDoc.GetAllocator();\ -\ - Value arrKey;\ - arrKey.SetString(memberName.c_str(), memberName.length());\ -\ - ok = ok && jDoc.IsObject();\ - ok = ok && jDoc.HasMember(arrKey);\ -\ - if(ok && jDoc[arrKey].IsArray())\ - {\ - for (auto&& arrEl : jDoc[arrKey].GetArray())\ - {\ - Value arrKeyT;\ - arrKeyT.SetString(memberName.c_str(), memberName.length());\ -\ - RsGenericSerializer::SerializeContext elCtx(\ - nullptr, 0, ctx.mFlags, &allocator );\ - elCtx.mJson.AddMember(arrKeyT, arrEl, allocator);\ -\ - T el;\ - serial_process(j, elCtx, el, memberName); \ - ok = ok && elCtx.mOk;\ - ctx.mOk &= ok;\ - if(ok) v.INSERT_FUN(el);\ - else break;\ - }\ - }\ - else\ - {\ - ctx.mOk = false;\ - }\ -\ -} while(false) - - struct RsTypeSerializer { - /** This type should be used to pass a parameter to drive the serialisation - * if needed */ - struct TlvMemBlock_proxy : std::pair + /** Use this wrapper to serialize raw memory chunks + * RsTypeSerializer::RawMemoryWrapper chunkWrapper(chunk_data, chunk_size); + * RsTypeSerializer::serial_process(j, ctx, chunkWrapper, "chunk_data"); + **/ + struct RawMemoryWrapper: std::pair, RsSerializable { - TlvMemBlock_proxy(void*& p, uint32_t& s) : - std::pair(p,s) {} - TlvMemBlock_proxy(uint8_t*& p,uint32_t& s) : - std::pair(*(void**)&p,s) {} + RawMemoryWrapper(uint8_t*& p,uint32_t& s) : + std::pair(p,s) {} + RawMemoryWrapper(void*& p, uint32_t& s) : + std::pair(*(uint8_t**)(&p),s) {} + + /// Maximum supported size 10MB + static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024; + + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override; + private: + void clear(); }; - /// Generic types + /// Most types are not valid sequence containers + template + struct is_sequence_container : std::false_type {}; + + /// Trait to match supported strings types + template + struct is_string : std::is_same, std::string> {}; + + /// Integral types + template + typename std::enable_if::value>::type + static /*void*/ serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + INTT& member, const std::string& member_name ) + { + const bool VLQ_ENCODING = !!( + RsSerializationFlags::INTEGER_VLQ & + ctx.mFlags.toEFT() ); + + switch(j) + { + case RsGenericSerializer::SIZE_ESTIMATE: + if(VLQ_ENCODING) ctx.mOffset += VLQ_size(member); + else ctx.mOffset += sizeof(INTT); + break; + case RsGenericSerializer::SERIALIZE: + { + if(!ctx.mOk) break; + if(VLQ_ENCODING) + ctx.mOk = VLQ_serialize( + ctx.mData, ctx.mSize, ctx.mOffset, member ); + else + { + ctx.mOk = ctx.mSize >= ctx.mOffset + sizeof(INTT); + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot serialise " + << typeid(INTT).name() << " " + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset + << " sizeof(INTT): " << sizeof(INTT) + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + break; + } + INTT netorder_num = rs_endian_fix(member); + memcpy(ctx.mData + ctx.mOffset, &netorder_num, sizeof(INTT)); + ctx.mOffset += sizeof(INTT); + } + break; + } + case RsGenericSerializer::DESERIALIZE: + if(!ctx.mOk) break; + if(VLQ_ENCODING) ctx.mOk = VLQ_deserialize( + ctx.mData, ctx.mSize, ctx.mOffset, member ); + else + { + ctx.mOk = ctx.mSize >= ctx.mOffset + sizeof(INTT); + if(!ctx.mOk) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot deserialise " + << typeid(INTT).name() << " " + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset + << " sizeof(INTT): " << sizeof(INTT) + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + exit(-1); + break; + } + memcpy(&member, ctx.mData + ctx.mOffset, sizeof(INTT)); + member = rs_endian_fix(member); + ctx.mOffset += sizeof(INTT); + } + break; + case RsGenericSerializer::PRINT: break; + case RsGenericSerializer::TO_JSON: + ctx.mOk = ctx.mOk && to_JSON(member_name, member, ctx.mJson); + break; + case RsGenericSerializer::FROM_JSON: + ctx.mOk &= ( ctx.mOk || + !!( RsSerializationFlags::YIELDING & + ctx.mFlags.toEFT() ) ) + && from_JSON(member_name, member, ctx.mJson); + break; + default: fatalUnknownSerialJob(j); + } + } + +//============================================================================// +// Generic types // +//============================================================================// + template typename std::enable_if< std::is_same::value || !( + std::is_integral::value || std::is_base_of::value || std::is_enum::value || std::is_base_of::value || - std::is_same::value ) >::type + std::is_same::value || + is_sequence_container::value || is_string::value ) >::type static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - T& member, const std::string& member_name ) + T& memberC, const std::string& member_name ) { + // Avoid problems with const sneaking into template paramether + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: @@ -170,12 +205,16 @@ struct RsTypeSerializer } } +//============================================================================// +// Generic types + type_id // +//============================================================================// + /// Generic types + type_id - template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - uint16_t type_id, T& member, - const std::string& member_name ) + template RS_DEPRECATED + static void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + uint16_t type_id, T& member, const std::string& member_name ) { switch(j) { @@ -190,9 +229,7 @@ struct RsTypeSerializer ctx.mOk = ctx.mOk && serialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member); break; - case RsGenericSerializer::PRINT: - print_data(member_name, member); - break; + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: ctx.mOk = ctx.mOk && to_JSON(member_name, type_id, member, ctx.mJson); @@ -206,85 +243,59 @@ struct RsTypeSerializer } } +//============================================================================// +// std::map // +//============================================================================// + /// std::map template static void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - std::map& v, + std::map& member, const std::string& memberName ) { switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: + case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]] + case RsGenericSerializer::SERIALIZE: { - ctx.mOffset += 4; - for(typename std::map::iterator it(v.begin());it!=v.end();++it) + uint32_t mapSize = member.size(); + RS_SERIAL_PROCESS(mapSize); + + for( auto it = member.begin(); + ctx.mOk && it != member.end(); ++it ) { - serial_process( j, ctx, const_cast(it->first), - "map::*it->first" ); - serial_process( j,ctx,const_cast(it->second), - "map::*it->second" ); + RS_SERIAL_PROCESS(const_cast(it->first)); + RS_SERIAL_PROCESS(const_cast(it->second)); } break; } case RsGenericSerializer::DESERIALIZE: { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); + uint32_t mapSize = 0; + RS_SERIAL_PROCESS(mapSize); - for(uint32_t i=0; ifirst"); - serial_process(j, ctx, u, "map::*it->second"); - v[t] = u; - } - break; - } - case RsGenericSerializer::SERIALIZE: - { - uint32_t n=v.size(); - serial_process(j,ctx,n,"temporary size"); - - for(typename std::map::iterator it(v.begin());it!=v.end();++it) - { - serial_process( j, ctx, const_cast(it->first), - "map::*it->first" ); - serial_process( j, ctx, const_cast(it->second), - "map::*it->second" ); - } - break; - } - case RsGenericSerializer::PRINT: - { - if(v.empty()) - std::cerr << " Empty map \"" << memberName << "\"" - << std::endl; - else - std::cerr << " std::map of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; - - for(typename std::map::iterator it(v.begin());it!=v.end();++it) - { - std::cerr << " "; - - serial_process( j, ctx,const_cast(it->first), - "map::*it->first" ); - serial_process( j, ctx, const_cast(it->second), - "map::*it->second" ); + RS_SERIAL_PROCESS(t); + RS_SERIAL_PROCESS(u); + member[t] = u; } break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: { using namespace rapidjson; Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); - Value arrKey; arrKey.SetString(memberName.c_str(), - memberName.length(), allocator); + Value arrKey; arrKey.SetString( + memberName.c_str(), + static_cast(memberName.length()), allocator ); Value arr(kArrayType); - for (auto& kv : v) + for (auto& kv : member) { // Use same allocator to avoid deep copy RsGenericSerializer::SerializeContext kCtx( @@ -318,7 +329,8 @@ struct RsTypeSerializer Document::AllocatorType& allocator = jDoc.GetAllocator(); Value arrKey; - arrKey.SetString(memberName.c_str(), memberName.length()); + arrKey.SetString( memberName.c_str(), + static_cast(memberName.length()) ); ok = ok && jDoc.IsObject(); ok = ok && jDoc.HasMember(arrKey); @@ -350,35 +362,37 @@ struct RsTypeSerializer vCtx.mOk ); ctx.mOk &= ok; - if(ok) v.insert(std::pair(key,value)); + if(ok) member.insert(std::pair(key,value)); else break; } } - else - { - ctx.mOk = false; - } + else ctx.mOk = false; break; } default: fatalUnknownSerialJob(j); } } + +//============================================================================// +// std::pair // +//============================================================================// + /// std::pair template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::pair& p, - const std::string& memberName ) + static void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + std::pair& member, const std::string& memberName ) { switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough - case RsGenericSerializer::DESERIALIZE: // fallthrough - case RsGenericSerializer::SERIALIZE: // fallthrough + case RsGenericSerializer::SIZE_ESTIMATE: // [[fallthrough]] + case RsGenericSerializer::DESERIALIZE: // [[fallthrough]] + case RsGenericSerializer::SERIALIZE: // [[fallthrough]] case RsGenericSerializer::PRINT: - serial_process(j, ctx, p.first, "pair::first"); - serial_process(j, ctx, p.second, "pair::second"); + RS_SERIAL_PROCESS(member.first); + RS_SERIAL_PROCESS(member.second); break; case RsGenericSerializer::TO_JSON: { @@ -389,11 +403,13 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext lCtx( nullptr, 0, ctx.mFlags, &allocator ); - serial_process(j, lCtx, p.first, "first"); - serial_process(j, lCtx, p.second, "second"); + serial_process(j, lCtx, member.first, "first"); + serial_process(j, lCtx, member.second, "second"); rapidjson::Value key; - key.SetString(memberName.c_str(), memberName.length(), allocator); + key.SetString( memberName.c_str(), + static_cast(memberName.length()), + allocator); /* Because the passed allocator is reused it doesn't go out of scope * and there is no need of deep copy and we can take advantage of @@ -415,9 +431,9 @@ struct RsTypeSerializer { if(!yielding) { - std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName - << "\" not found in JSON:" << std::endl - << jDoc << std::endl << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName + << "\" not found in JSON:" << std::endl + << jDoc << std::endl << std::endl; print_stacktrace(); } ctx.mOk = false; @@ -429,8 +445,8 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext lCtx(nullptr, 0, ctx.mFlags); lCtx.mJson.SetObject() = v; // Beware of move semantic!! - serial_process(j, lCtx, p.first, "first"); - serial_process(j, lCtx, p.second, "second"); + serial_process(j, lCtx, member.first, "first"); + serial_process(j, lCtx, member.second, "second"); ctx.mOk &= lCtx.mOk; break; @@ -439,176 +455,225 @@ struct RsTypeSerializer } } - /// std::vector +//============================================================================// +// Sequence containers // +//============================================================================// + + /** std::list is supported */ template + struct is_sequence_container>: std::true_type {}; + + /** std::set is supported */ template + struct is_sequence_container>: std::true_type {}; + + /** std::vector is supported */ template + struct is_sequence_container>: std::true_type {}; + + + /// STL compatible sequence containers std::list, std::set, std::vector... template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::vector& v, - const std::string& memberName ) + typename std::enable_if::value>::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& member, const std::string& memberName ) { + using el_t = typename T::value_type; + switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: + case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]] + case RsGenericSerializer::SERIALIZE: { - ctx.mOffset += 4; - for(uint32_t i=0;i ctx.mSize - ctx.mOffset) { - std::cerr << " " ; - serial_process(j,ctx,v[i],memberName); + ctx.mOk = false; + RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " + << "sequence with apparently malformed elements count." + << " elCount: " << elCount + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset << " " + << std::errc::argument_out_of_domain + << std::endl; + print_stacktrace(); + break; + } + + for(uint32_t i=0; ctx.mOk && i < elCount; ++i ) + { + el_t elem; + RS_SERIAL_PROCESS(elem); + member.insert(member.end(), elem); } break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); + { + using namespace rapidjson; + + Document::AllocatorType& allocator = ctx.mJson.GetAllocator(); + + Value arrKey; arrKey.SetString(memberName.c_str(), allocator); + Value arr(kArrayType); + + for(auto& const_el : member) + { + auto el = const_cast(const_el); + + /* Use same allocator to avoid deep copy */ + RsGenericSerializer::SerializeContext elCtx( + nullptr, 0, ctx.mFlags, &allocator ); + serial_process(j, elCtx, el, memberName); + + elCtx.mOk = elCtx.mOk && elCtx.mJson.HasMember(arrKey); + if(elCtx.mOk) arr.PushBack(elCtx.mJson[arrKey], allocator); + else + { + ctx.mOk = false; + break; + } + } + + ctx.mJson.AddMember(arrKey, arr, allocator); break; + } case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(push_back); + { + using namespace rapidjson; + + bool ok = ctx.mOk || ctx.mFlags & + RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + Document& jDoc(ctx.mJson); + Document::AllocatorType& allocator = jDoc.GetAllocator(); + + Value arrKey; + arrKey.SetString( memberName.c_str(), + static_cast(memberName.length()) ); + + ok = ok && jDoc.IsObject(); + ok = ok && jDoc.HasMember(arrKey); + + if(ok && jDoc[arrKey].IsArray()) + { + for (auto&& arrEl : jDoc[arrKey].GetArray()) + { + Value arrKeyT; + arrKeyT.SetString( + memberName.c_str(), + static_cast(memberName.length()) ); + + RsGenericSerializer::SerializeContext elCtx( + nullptr, 0, ctx.mFlags, &allocator ); + elCtx.mJson.AddMember(arrKeyT, arrEl, allocator); + + el_t el; + serial_process(j, elCtx, el, ""); + ok = ok && elCtx.mOk; + ctx.mOk &= ok; + if(ok) member.insert(member.end(), el); + else break; + } + } + else ctx.mOk = false; break; + } default: fatalUnknownSerialJob(j); } } - /// std::set - template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::set& v, const std::string& memberName ) - { - switch(j) - { - case RsGenericSerializer::SIZE_ESTIMATE: - { - ctx.mOffset += 4; - for(typename std::set::iterator it(v.begin());it!=v.end();++it) - // the const cast here is a hack to avoid serial_process to - // instantiate serialise(const T&) - serial_process(j,ctx,const_cast(*it) ,memberName); - break; - } - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); - for(uint32_t i=0; i::iterator it(v.begin());it!=v.end();++it) - // the const cast here is a hack to avoid serial_process to - // instantiate serialise(const T&) - serial_process(j,ctx,const_cast(*it) ,memberName); - break; - } - case RsGenericSerializer::PRINT: - { - if(v.empty()) - std::cerr << " Empty set \"" << memberName << "\"" - << std::endl; - else - std::cerr << " Set of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; - break; - } - case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); - break; - case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(insert); - break; - default: fatalUnknownSerialJob(j); - } - } +//============================================================================// +// Strings // +//============================================================================// - /// std::list + /// Strings template - static void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - std::list& v, - const std::string& memberName ) + typename std::enable_if::value>::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& memberC, + const std::string& memberName ) { + // Avoid problems with const sneaking into template paramether + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: { - ctx.mOffset += 4; - for(typename std::list::iterator it(v.begin());it!=v.end();++it) - serial_process(j,ctx,*it ,memberName); - break; - } - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - serial_process(j,ctx,n,"temporary size"); - for(uint32_t i=0;i(member.size()); + RS_SERIAL_PROCESS(aSize); + ctx.mOffset += aSize; break; } case RsGenericSerializer::SERIALIZE: { - uint32_t n=v.size(); - serial_process(j,ctx,n,"temporary size"); - for(typename std::list::iterator it(v.begin());it!=v.end();++it) - serial_process(j,ctx,*it ,memberName); + uint32_t len = static_cast(member.length()); + RS_SERIAL_PROCESS(len); + if(len > ctx.mSize - ctx.mOffset) + { + RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space + << std::endl; + ctx.mOk = false; + } + memcpy(ctx.mData + ctx.mOffset, member.c_str(), len); + ctx.mOffset += len; break; } - case RsGenericSerializer::PRINT: + case RsGenericSerializer::DESERIALIZE: { - if(v.empty()) - std::cerr << " Empty list \"" << memberName << "\"" - << std::endl; - else - std::cerr << " List of " << v.size() << " elements: \"" - << memberName << "\"" << std::endl; + uint32_t len; + RS_SERIAL_PROCESS(len); + if(!ctx.mOk) break; + if(len > ctx.mSize - ctx.mOffset) + { + ctx.mOk = false; + RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " + << "string with apparently malformed elements count." + << " len: " << len + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset << " " + << std::errc::argument_out_of_domain + << std::endl; + print_stacktrace(); + break; + } + member.resize(len); + memcpy(&member[0], ctx.mData + ctx.mOffset, len); + ctx.mOffset += len; break; } + case RsGenericSerializer::PRINT: break; case RsGenericSerializer::TO_JSON: - RsTypeSerializer_PRIVATE_TO_JSON_ARRAY(); + ctx.mOk = ctx.mOk && to_JSON(memberName, member, ctx.mJson); break; case RsGenericSerializer::FROM_JSON: - RsTypeSerializer_PRIVATE_FROM_JSON_ARRAY(push_back); + { + bool ok = ctx.mOk || !!( ctx.mFlags.toEFT() + & RsSerializationFlags::YIELDING ); + ctx.mOk = ok && from_JSON(memberName, member, ctx.mJson) && ctx.mOk; break; + } default: fatalUnknownSerialJob(j); } } @@ -619,46 +684,10 @@ struct RsTypeSerializer RsGenericSerializer::SerializeContext& ctx, t_RsFlags32& v, const std::string& memberName ) - { - switch(j) - { - case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += 4; break; - case RsGenericSerializer::DESERIALIZE: - { - uint32_t n=0; - deserialize(ctx.mData,ctx.mSize,ctx.mOffset,n); - v = t_RsFlags32(n); - break; - } - case RsGenericSerializer::SERIALIZE: - { - uint32_t n=v.toUInt32(); - serialize(ctx.mData,ctx.mSize,ctx.mOffset,n); - break; - } - case RsGenericSerializer::PRINT: - std::cerr << " Flags of type " << std::hex << N << " : " - << v.toUInt32() << std::endl; - break; - case RsGenericSerializer::TO_JSON: - ctx.mOk = to_JSON(memberName, v.toUInt32(), ctx.mJson); - break; - case RsGenericSerializer::FROM_JSON: - { - uint32_t f = 0; - ctx.mOk &= - (ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING) - && from_JSON(memberName, f, ctx.mJson) - && (v = t_RsFlags32(f), true); - break; - } - default: fatalUnknownSerialJob(j); - } - } + { serial_process(j, ctx, v._bits, memberName); } /** * @brief serial process enum types - * * On declaration of your member of enum type you must specify the * underlying type otherwise the serialization format may differ in an * uncompatible way depending on the compiler/platform. @@ -685,11 +714,14 @@ struct RsTypeSerializer /// RsSerializable and derivatives template typename std::enable_if::value>::type - static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - T& member, - const std::string& memberName ) + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& memberC, const std::string& memberName ) { + using m_t = std::remove_const_t; + m_t& member = const_cast(memberC); + switch(j) { case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough @@ -710,11 +742,14 @@ struct RsTypeSerializer member.serial_process(j, lCtx); rapidjson::Value key; - key.SetString(memberName.c_str(), memberName.length(), allocator); + key.SetString( + memberName.c_str(), + static_cast(memberName.length()), + allocator ); - /* Because the passed allocator is reused it doesn't go out of scope and - * there is no need of deep copy and we can take advantage of the much - * faster rapidjson move semantic */ + /* Because the passed allocator is reused it doesn't go out of scope + * and there is no need of deep copy and we can take advantage of + * the much faster rapidjson move semantic */ jDoc.AddMember(key, lCtx.mJson, allocator); ctx.mOk = ctx.mOk && lCtx.mOk; @@ -768,15 +803,13 @@ struct RsTypeSerializer /// RsTlvItem derivatives only template typename std::enable_if< - std::is_base_of::value && !std::is_same::value - >::type - static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx, - T& member, - const std::string& memberName ) - { - serial_process(j, ctx, static_cast(member), memberName); - } + std::is_base_of::value && + !std::is_same::value >::type + static serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx, + T& member, const std::string& memberName ) + { serial_process(j, ctx, static_cast(member), memberName); } /** std::error_condition * supports only TO_JSON ErrConditionWrapper::serial_process will explode @@ -786,13 +819,15 @@ struct RsTypeSerializer static /*void*/ serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, - const T& cond, - const std::string& member_name ) + T& cond, const std::string& member_name ) { ErrConditionWrapper ew(cond); serial_process(j, ctx, ew, member_name); } + RS_DEPRECATED_FOR(RawMemoryWrapper) + typedef RawMemoryWrapper TlvMemBlock_proxy; + protected: //============================================================================// @@ -820,27 +855,32 @@ protected: // Generic types + type_id declarations // //============================================================================// - template static bool serialize( + template RS_DEPRECATED + static bool serialize( uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id, const T& member ); - template static bool deserialize( + template RS_DEPRECATED + static bool deserialize( const uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id, T& member ); - template static uint32_t serial_size( - uint16_t type_id,const T& member ); + template RS_DEPRECATED + static uint32_t serial_size(uint16_t type_id,const T& member); - template static void print_data( const std::string& n, - uint16_t type_id,const T& member ); + template RS_DEPRECATED + static void print_data( + const std::string& n, uint16_t type_id,const T& member ); - template static bool to_JSON( const std::string& membername, - uint16_t type_id, - const T& member, RsJson& jVal ); + template RS_DEPRECATED + static bool to_JSON( + const std::string& membername, uint16_t type_id, + const T& member, RsJson& jVal ); - template static bool from_JSON( const std::string& memberName, - uint16_t type_id, - T& member, RsJson& jDoc ); + template RS_DEPRECATED + static bool from_JSON( + const std::string& memberName, uint16_t type_id, + T& member, RsJson& jDoc ); //============================================================================// // t_RsGenericId<...> declarations // @@ -910,13 +950,190 @@ protected: t_RsTlvList& member, RsJson& jDoc ); - [[noreturn]] static void fatalUnknownSerialJob(int j) +//============================================================================// +// Integral types VLQ // +//============================================================================// + + /** + * Size calculation of unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, uint32_t> + VLQ_size(T member) { - RsFatal() << " Unknown serial job: " << j << std::endl; - print_stacktrace(); - exit(EINVAL); + std::decay_t memberBackup = member; + uint32_t ret = 1; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wbool-compare" + while(member > 127) { ++ret; member >>= 7; } +#pragma GCC diagnostic pop + RsDbg() << __PRETTY_FUNCTION__ << " memberBackup: " << memberBackup + << " return: " << ret << std::endl; + return ret; } + /** + * Serialization of unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, bool> + VLQ_serialize( + uint8_t data[], uint32_t size, uint32_t &offset, T member ) + { + std::decay_t backupMember = member; + uint32_t offsetBackup = offset; + + bool ok = true; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wbool-compare" + /* Check with < and not with <= here as we write last byte after + * the loop. Order of && operands very important here! */ + while(member > 127 && (ok = offset < size)) + { + // | 128: Set the next byte flag + data[offset++] = (static_cast(member & 127)) | 128; + // Remove the seven bits we just wrote + member >>= 7; + } +#pragma GCC diagnostic pop + + if(!(ok = ok && offset <= size)) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot serialise " + << typeid(T).name() + << " member " << member + << " size: " << size + << " offset: " << offset + << std::error_condition(std::errc::no_buffer_space) + << std::endl; + print_stacktrace(); + return false; + } + + data[offset++] = static_cast(member & 127); + + RsDbg() << __PRETTY_FUNCTION__ << " backupMember: " << backupMember + << " offsetBackup: " << offsetBackup << " offeset: " << offset + << " serialized as: "; + for(; offsetBackup < offset; ++offsetBackup) + std::cerr << " " << std::bitset<8>(data[offsetBackup]); + std::cerr << std::endl; + + return ok; + } + + /** + * Deserialization for unsigned integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + * @see https://techoverflow.net/2013/01/25/efficiently-encoding-variable-length-integers-in-cc/ + */ + template static + std::enable_if_t>::value, bool> + VLQ_deserialize( + const uint8_t data[], uint32_t size, uint32_t& offset, T& member ) + { + uint32_t backupOffset = offset; + + member = 0; + uint32_t offsetBackup = offset; + + /* In a reasonable VLQ coding representing an integer + * could take at maximum sizeof(integer) + 1 space, if it is + * not the case something fishy is happening. */ + for (size_t i = 0; offset < size && i <= sizeof(T); ++i) + { + member |= (data[offset] & 127) << (7 * i); + // If the next-byte flag is not set, ++ is after on purpose + if(!(data[offset++] & 128)) + { + RsDbg() << __PRETTY_FUNCTION__ + << " size: " << size + << " backupOffset " << backupOffset + << " offset: " << offset + << " member " << member << std::endl; + return true; + } + } + + /* If return is not triggered inside the for loop, either the buffer + * ended before we encountered the end of the number, or the number + * is VLQ encoded improperly */ + RsErr() << __PRETTY_FUNCTION__ << std::errc::illegal_byte_sequence + << " offsetBackup: " << offsetBackup + << " offset: " << offset << " bytes: "; + for(; offsetBackup < offset; ++offsetBackup) + std::cerr << " " << std::bitset<8>(data[offsetBackup]); + std::cerr << std::endl; + + print_stacktrace(); + return false; + } + + /** + * Size calculation of signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, uint32_t> + VLQ_size(T member) + { + member = (member << 1) ^ (member >> (sizeof(T)-1)); // ZigZag encoding + return VLQ_size( + static_cast::type>(member)); + } + + /** + * Serialization of signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, bool> + VLQ_serialize( + uint8_t data[], uint32_t size, uint32_t &offset, T member ) + { + member = (member << 1) ^ (member >> (sizeof(T)-1)); // ZigZag encoding + return VLQ_serialize( + data, size, offset, + static_cast::type>(member)); + } + + /** + * Deserialization for signed integers as Variable Lenght Quantity + * @see RsSerializationFlags::INTEGER_VLQ + * @see https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding + * @see https://golb.hplar.ch/2019/06/variable-length-int-java.html + */ + template static + std::enable_if_t>::value, bool> + VLQ_deserialize( + const uint8_t data[], uint32_t size, uint32_t& offset, T& member ) + { + using DT = std::decay_t; + typename std::make_unsigned
::type temp = 0; + bool ok = VLQ_deserialize(data, size, offset, temp); + // ZizZag decoding + member = (static_cast
(temp) >> 1) ^ -(static_cast
(temp) & 1); + return ok; + } + +//============================================================================// +// Error Condition Wrapper // +//============================================================================// + struct ErrConditionWrapper : RsSerializable { ErrConditionWrapper(const std::error_condition& ec): mec(ec) {} @@ -931,6 +1148,17 @@ protected: const std::error_condition& mec; }; +//============================================================================// +// Miscellanea // +//============================================================================// + + [[noreturn]] static void fatalUnknownSerialJob(int j) + { + RsFatal() << " Unknown serial job: " << j << std::endl; + print_stacktrace(); + exit(EINVAL); + } + RS_SET_CONTEXT_DEBUG_LEVEL(1) }; @@ -1079,7 +1307,6 @@ bool RsTypeSerializer::from_JSON( const std::string& memberName, return true; } - //============================================================================// // Not implemented types macros // //============================================================================// diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index d4a22d84d..9478c0f74 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -107,6 +107,10 @@ struct t_RsLogger << std::setfill('0') << std::setw(3) << msec.count() << std::setfill(tFill) << " " << val; } + + /// needed for manipulators and things like std::endl + stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) + { return std::cerr << pf; } }; #endif // def __ANDROID__ diff --git a/libretroshare/src/util/rsendian.h b/libretroshare/src/util/rsendian.h new file mode 100644 index 000000000..925542f0d --- /dev/null +++ b/libretroshare/src/util/rsendian.h @@ -0,0 +1,59 @@ +/******************************************************************************* + * * + * libretroshare base64 encoding utilities * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +/** + * @file + * This file provide convenient integer endiannes conversion utilities. + * Instead of providing them with different names for each type (a la C htonl, + * htons, ntohl, ntohs ), which make them uncomfortable to use, expose a + * templated function `rs_endian_fix` to reorder integers representation byets + * when sending or receiving from the network. */ + +/* enforce LITTLE_ENDIAN on Windows */ +#ifdef WINDOWS_SYS +# define BYTE_ORDER 1234 +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +#else +#include +#endif + +template inline INTTYPE rs_endian_fix(INTTYPE val) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + INTTYPE swapped = 0; + for (size_t i = 0; i < sizeof(INTTYPE); ++i) + swapped |= (val >> (8*(sizeof(INTTYPE)-i-1)) & 0xFF) << (8*i); + return swapped; +#else + return hostI; +#endif +}; + +#ifndef BYTE_ORDER +# error "ENDIAN determination Failed (BYTE_ORDER not defined)" +#endif + +#if !(BYTE_ORDER == BIG_ENDIAN || BYTE_ORDER == LITTLE_ENDIAN) +# error "ENDIAN determination Failed (unkown BYTE_ORDER value)" +#endif From 5610cc86004d6bc01ca3748071624eb703fb5828 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 18 Mar 2020 23:04:16 +0100 Subject: [PATCH 04/40] Fix memory management and deprecated removal from serialization Fix missing RsDiscPgpKeyItem initialization Fix inconsistent new[]/delete[] usage in RsDiscPgpKeyItem and PGPHandler::exportPublicKey which now consistently uses malloc/free Remove deprecated RsGenericSerializer::FORMAT_* Move from deprecated RsServiceSerializer::SERIALIZATION_FLAG_* to RsSerializationFlags Solve a bunch of compiler warnings Stricter checks in SerializeContext costructor --- libretroshare/src/chat/distributedchat.cc | 12 +- libretroshare/src/chat/rschatitems.cc | 4 +- libretroshare/src/chat/rschatitems.h | 5 +- libretroshare/src/file_sharing/file_tree.cc | 4 +- .../gossipdiscovery/gossipdiscoveryitems.cc | 4 +- .../gossipdiscovery/gossipdiscoveryitems.h | 20 ++-- libretroshare/src/grouter/grouteritems.cc | 11 +- libretroshare/src/grouter/grouteritems.h | 4 +- libretroshare/src/grouter/p3grouter.cc | 12 +- libretroshare/src/jsonapi/jsonapi.cpp | 4 +- libretroshare/src/pgp/pgphandler.cc | 59 ++++++---- libretroshare/src/pgp/pgphandler.h | 4 +- libretroshare/src/retroshare/rsflags.h | 4 - libretroshare/src/retroshare/rspeers.h | 2 +- libretroshare/src/rsitems/rsitem.h | 5 +- libretroshare/src/rsitems/rsmsgitems.cc | 4 +- libretroshare/src/rsitems/rsmsgitems.h | 24 ++-- libretroshare/src/rsserver/p3peers.cc | 64 +++++----- libretroshare/src/rsserver/p3peers.h | 3 +- libretroshare/src/serialiser/rsserializer.cc | 109 +++++++++--------- libretroshare/src/serialiser/rsserializer.h | 69 +++-------- .../src/serialiser/rstypeserializer.cc | 10 +- .../src/serialiser/rstypeserializer.h | 26 ++--- libretroshare/src/services/p3msgservice.cc | 11 +- 24 files changed, 230 insertions(+), 244 deletions(-) diff --git a/libretroshare/src/chat/distributedchat.cc b/libretroshare/src/chat/distributedchat.cc index 45fc4e29e..0aa16b20e 100644 --- a/libretroshare/src/chat/distributedchat.cc +++ b/libretroshare/src/chat/distributedchat.cc @@ -221,7 +221,8 @@ bool DistributedChatService::checkSignature(RsChatLobbyBouncingObject *obj,const mGixs->requestKey(obj->signature.keyId,peer_list,RsIdentityUsage(RS_SERVICE_TYPE_CHAT,RsIdentityUsage::CHAT_LOBBY_MSG_VALIDATION,RsGxsGroupId(),RsGxsMessageId(),obj->lobby_id)); - uint32_t size = RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).size(dynamic_cast(obj)) ; + uint32_t size = RsChatSerialiser(RsSerializationFlags::SIGNATURE) + .size(dynamic_cast(obj)); RsTemporaryMemory memory(size) ; #ifdef DEBUG_CHAT_LOBBIES @@ -229,7 +230,8 @@ bool DistributedChatService::checkSignature(RsChatLobbyBouncingObject *obj,const std::cerr << " signature id: " << obj->signature.keyId << std::endl; #endif - if(!RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).serialise(dynamic_cast(obj),memory,&size)) + if( !RsChatSerialiser(RsSerializationFlags::SIGNATURE) + .serialise(dynamic_cast(obj),memory,&size) ) { std::cerr << " (EE) Cannot serialise message item. " << std::endl; return false ; @@ -1003,10 +1005,12 @@ bool DistributedChatService::locked_initLobbyBouncableObject(const ChatLobbyId& // now sign the object, if the lobby expects it - uint32_t size = RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).size(dynamic_cast(&item)) ; + uint32_t size = RsChatSerialiser(RsSerializationFlags::SIGNATURE) + .size(dynamic_cast(&item)); RsTemporaryMemory memory(size) ; - if(!RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).serialise(dynamic_cast(&item),memory,&size)) + if( !RsChatSerialiser(RsSerializationFlags::SIGNATURE) + .serialise(dynamic_cast(&item),memory,&size) ) { std::cerr << "(EE) Cannot sign message item. " << std::endl; return false ; diff --git a/libretroshare/src/chat/rschatitems.cc b/libretroshare/src/chat/rschatitems.cc index d3a837b07..abe730d09 100644 --- a/libretroshare/src/chat/rschatitems.cc +++ b/libretroshare/src/chat/rschatitems.cc @@ -85,8 +85,8 @@ void RsChatLobbyBouncingObject::serial_process(RsGenericSerializer::SerializeJob RsTypeSerializer::serial_process(j,ctx,msg_id ,"msg_id") ; RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_NAME,nick,"nick") ; - if(!(ctx.mFlags & RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE)) - RsTypeSerializer::serial_process(j,ctx,signature,"signature") ; + if(!(ctx.mFlags & RsSerializationFlags::SIGNATURE)) + RS_SERIAL_PROCESS(signature); } void RsChatLobbyMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) diff --git a/libretroshare/src/chat/rschatitems.h b/libretroshare/src/chat/rschatitems.h index c08b35231..5bcfa1e0a 100644 --- a/libretroshare/src/chat/rschatitems.h +++ b/libretroshare/src/chat/rschatitems.h @@ -369,9 +369,8 @@ struct PrivateOugoingMapItem : RsChatItem struct RsChatSerialiser : RsServiceSerializer { - RsChatSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE) : - RsServiceSerializer( RS_SERVICE_TYPE_CHAT, - RsGenericSerializer::FORMAT_BINARY, flags ) {} + RsChatSerialiser(RsSerializationFlags flags = RsSerializationFlags::NONE): + RsServiceSerializer(RS_SERVICE_TYPE_CHAT, flags) {} virtual RsItem *create_item(uint16_t service_id,uint8_t item_sub_id) const; }; diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 2dbea4cc4..55de0e78b 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -61,7 +61,7 @@ RsFileTree::fromBase64(const std::string& base64) RsGenericSerializer::SerializeContext ctx( mem.data(), static_cast(mem.size()), - SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ) ); + RsSerializationFlags::INTEGER_VLQ ); std::unique_ptr ft(new RsFileTree); ft->serial_process( RsGenericSerializer::SerializeJob::DESERIALIZE, ctx); @@ -73,7 +73,7 @@ RsFileTree::fromBase64(const std::string& base64) std::string RsFileTree::toBase64() const { RsGenericSerializer::SerializeContext ctx; - ctx.mFlags = SerializationFlags::fromEFT(RsSerializationFlags::INTEGER_VLQ); + ctx.mFlags = RsSerializationFlags::INTEGER_VLQ; RsFileTree* ncThis = const_cast(this); ncThis->serial_process( RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx ); diff --git a/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.cc b/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.cc index bee4c9b59..c91d97b69 100644 --- a/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.cc +++ b/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.cc @@ -74,8 +74,8 @@ void RsDiscPgpKeyItem::clear() { pgpKeyId.clear(); free(bin_data); - bin_data = nullptr; - bin_len=0; + bin_data = nullptr; + bin_len = 0; } void RsDiscContactItem::clear() diff --git a/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.h b/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.h index f5e219b95..1ef14a138 100644 --- a/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.h +++ b/libretroshare/src/gossipdiscovery/gossipdiscoveryitems.h @@ -84,17 +84,23 @@ class RsDiscPgpKeyItem: public RsDiscItem { public: - RsDiscPgpKeyItem() : RsDiscItem(RsGossipDiscoveryItemType::PGP_CERT_BINARY) + RsDiscPgpKeyItem() : + RsDiscItem(RsGossipDiscoveryItemType::PGP_CERT_BINARY), + bin_data(nullptr), bin_len(0) { setPriorityLevel(QOS_PRIORITY_RS_DISC_PGP_CERT); } - virtual ~RsDiscPgpKeyItem() { delete[](bin_data);bin_data=nullptr;bin_len=0;} - + ~RsDiscPgpKeyItem() override { free(bin_data); } void clear() override; - void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx) override; - RsPgpId pgpKeyId; // duplicate information for practical reasons - unsigned char *bin_data; // binry key data allocated with new unsigned char[] - uint32_t bin_len; + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override; + + /// duplicate information for practical reasons + RsPgpId pgpKeyId; + + unsigned char* bin_data; + uint32_t bin_len; }; class RsDiscContactItem: public RsDiscItem diff --git a/libretroshare/src/grouter/grouteritems.cc b/libretroshare/src/grouter/grouteritems.cc index 302465414..10c91b222 100644 --- a/libretroshare/src/grouter/grouteritems.cc +++ b/libretroshare/src/grouter/grouteritems.cc @@ -99,12 +99,10 @@ void RsGRouterGenericDataItem::serial_process(RsGenericSerializer::SerializeJob RsTypeSerializer::serial_process (j,ctx,destination_key,"destination_key") ; RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; - RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ; + RsTypeSerializer::RawMemoryWrapper prox(data_bytes, data_size); + RsTypeSerializer::serial_process(j, ctx, prox, "data"); - RsTypeSerializer::serial_process(j,ctx,prox,"data") ; - - if(ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE) - return ; + if(!!(ctx.mFlags & RsSerializationFlags::SIGNATURE)) return; RsTypeSerializer::serial_process(j,ctx,signature,"signature") ; RsTypeSerializer::serial_process(j,ctx,duplication_factor,"duplication_factor") ; @@ -133,8 +131,7 @@ void RsGRouterSignedReceiptItem::serial_process(RsGenericSerializer::SerializeJo RsTypeSerializer::serial_process (j,ctx,service_id,"service_id") ; RsTypeSerializer::serial_process (j,ctx,data_hash,"data_hash") ; - if(ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE) - return ; + if(!!(ctx.mFlags & RsSerializationFlags::SIGNATURE)) return; RsTypeSerializer::serial_process(j,ctx,signature,"signature") ; } diff --git a/libretroshare/src/grouter/grouteritems.h b/libretroshare/src/grouter/grouteritems.h index 14957ab87..0a12cb3f0 100644 --- a/libretroshare/src/grouter/grouteritems.h +++ b/libretroshare/src/grouter/grouteritems.h @@ -286,7 +286,9 @@ class RsGRouterRoutingInfoItem: public RsGRouterItem, public GRouterRoutingInfo, class RsGRouterSerialiser: public RsServiceSerializer { public: - explicit RsGRouterSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE) : RsServiceSerializer(RS_SERVICE_TYPE_GROUTER,RsGenericSerializer::FORMAT_BINARY,flags) {} + explicit RsGRouterSerialiser( + RsSerializationFlags flags = RsSerializationFlags::NONE ): + RsServiceSerializer(RS_SERVICE_TYPE_GROUTER, flags) {} virtual RsItem *create_item(uint16_t service,uint8_t subtype) const ; }; diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 7fe2cc19e..d34a1349f 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -1658,7 +1658,9 @@ void p3GRouter::handleIncomingReceiptItem(const RsGRouterSignedReceiptItem *rece Sha1CheckSum p3GRouter::computeDataItemHash(const RsGRouterGenericDataItem *data_item) { - RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER); + RsGRouterSerialiser signature_serializer( + RsSerializationFlags::SIGNATURE | + RsSerializationFlags::SKIP_HEADER ); uint32_t signed_data_size = signature_serializer.size(const_cast(data_item)); uint32_t total_size = signed_data_size + data_item->signature.TlvSize() ; @@ -2034,7 +2036,9 @@ bool p3GRouter::signDataItem(RsGRouterAbstractMsgItem *item,const RsGxsId& signi std::cerr << " Key ID = " << signing_id << std::endl; std::cerr << " Getting key material..." << std::endl; //#endif - RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER) ; + RsGRouterSerialiser signature_serializer( + RsSerializationFlags::SIGNATURE | + RsSerializationFlags::SKIP_HEADER ); uint32_t data_size = signature_serializer.size(item) ; RsTemporaryMemory data(data_size) ; @@ -2092,8 +2096,8 @@ bool p3GRouter::verifySignedDataItem(const RsGRouterAbstractMsgItem *item,const } RsGRouterSerialiser signature_serializer( - RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | - RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER ); + RsSerializationFlags::SIGNATURE | + RsSerializationFlags::SKIP_HEADER ); uint32_t data_size = signature_serializer.size(const_cast(item)); // the const cast shouldn't be necessary if size() took a const. RsTemporaryMemory data(data_size); diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index f1e30881e..993165fdb 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -78,10 +78,10 @@ JsonApiServer::corsOptionsHeaders = #define INITIALIZE_API_CALL_JSON_CONTEXT \ RsGenericSerializer::SerializeContext cReq( \ nullptr, 0, \ - RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ); \ + RsSerializationFlags::YIELDING ); \ RsJson& jReq(cReq.mJson); \ if(session->get_request()->get_method() == "GET") \ - { \ + { \ const std::string jrqp(session->get_request()->get_query_parameter("jsonData")); \ jReq.Parse(jrqp.c_str(), jrqp.size()); \ } \ diff --git a/libretroshare/src/pgp/pgphandler.cc b/libretroshare/src/pgp/pgphandler.cc index 4ed342629..56b877e20 100644 --- a/libretroshare/src/pgp/pgphandler.cc +++ b/libretroshare/src/pgp/pgphandler.cc @@ -630,50 +630,61 @@ std::string PGPHandler::SaveCertificateToString(const RsPgpId& id,bool include_s return makeRadixEncodedPGPKey(key,include_signatures) ; } -bool PGPHandler::exportPublicKey(const RsPgpId& id,unsigned char *& mem_block,size_t& mem_size,bool armoured,bool include_signatures) const +bool PGPHandler::exportPublicKey( + const RsPgpId& id, + rs_view_ptr& mem_block, size_t& mem_size, + bool armoured, bool include_signatures ) const { - RsStackMutex mtx(pgphandlerMtx) ; // lock access to PGP memory structures. - const ops_keydata_t *key = locked_getPublicKey(id,false) ; - mem_block = NULL ; + mem_block = nullptr; mem_size = 0; // clear just in case if(armoured) { - std::cerr << __PRETTY_FUNCTION__ << ": should not be used with armoured=true, because there's a bug in the armoured export of OPS" << std::endl; - return false ; + RsErr() << __PRETTY_FUNCTION__ << " should not be used with " + << "armoured=true, because there's a bug in the armoured export" + << " of OPS" << std::endl; + print_stacktrace(); + return false; } - if(key == NULL) + RS_STACK_MUTEX(pgphandlerMtx); + const ops_keydata_t* key = locked_getPublicKey(id,false); + + if(!key) { - std::cerr << "Cannot output key " << id.toStdString() << ": not found in keyring." << std::endl; - return false ; + RsErr() << __PRETTY_FUNCTION__ << " key id: " << id + << " not found in keyring." << std::endl; + return false; } - ops_create_info_t* cinfo; - ops_memory_t *buf = NULL ; - ops_setup_memory_write(&cinfo, &buf, 0); + ops_create_info_t* cinfo; + ops_memory_t *buf = nullptr; + ops_setup_memory_write(&cinfo, &buf, 0); - if(ops_write_transferable_public_key_from_packet_data(key,armoured,cinfo) != ops_true) + if(ops_write_transferable_public_key_from_packet_data( + key, armoured, cinfo ) != ops_true) { - std::cerr << "ERROR: This key cannot be processed by RetroShare because\nDSA certificates are not yet handled." << std::endl; - return false ; + RsErr() << __PRETTY_FUNCTION__ << " This key id " << id + << " cannot be processed by RetroShare because DSA certificates" + << " support is not implemented yet." << std::endl; + return false; } - ops_writer_close(cinfo) ; + ops_writer_close(cinfo); - mem_block = new unsigned char[ops_memory_get_length(buf)] ; - mem_size = ops_memory_get_length(buf) ; - memcpy(mem_block,ops_memory_get_data(buf),mem_size) ; + mem_size = ops_memory_get_length(buf); + mem_block = reinterpret_cast(malloc(mem_size)); + memcpy(mem_block,ops_memory_get_data(buf),mem_size); - ops_teardown_memory_write(cinfo,buf); + ops_teardown_memory_write(cinfo,buf); if(!include_signatures) { - size_t new_size ; - PGPKeyManagement::findLengthOfMinimalKey(mem_block,mem_size,new_size) ; - mem_size = new_size ; + size_t new_size; + PGPKeyManagement::findLengthOfMinimalKey(mem_block, mem_size, new_size); + mem_size = new_size; } - return true ; + return true; } bool PGPHandler::exportGPGKeyPair(const std::string& filename,const RsPgpId& exported_key_id) const diff --git a/libretroshare/src/pgp/pgphandler.h b/libretroshare/src/pgp/pgphandler.h index 0b785285a..2c58cc736 100644 --- a/libretroshare/src/pgp/pgphandler.h +++ b/libretroshare/src/pgp/pgphandler.h @@ -107,7 +107,9 @@ public: bool LoadCertificateFromBinaryData(const unsigned char *bin_data,uint32_t bin_data_len, RsPgpId& gpg_id, std::string& error_string); std::string SaveCertificateToString(const RsPgpId& id,bool include_signatures) const ; - bool exportPublicKey(const RsPgpId& id,unsigned char *& mem,size_t& mem_size,bool armoured,bool include_signatures) const ; + bool exportPublicKey( const RsPgpId& id, + rs_view_ptr& mem,size_t& mem_size, + bool armoured, bool include_signatures) const; bool parseSignature(unsigned char *sign, unsigned int signlen,RsPgpId& issuer_id) ; bool SignDataBin(const RsPgpId& id, const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen, bool make_raw_signature=false, std::string reason = "") ; diff --git a/libretroshare/src/retroshare/rsflags.h b/libretroshare/src/retroshare/rsflags.h index 37320f08b..d92cfb33e 100644 --- a/libretroshare/src/retroshare/rsflags.h +++ b/libretroshare/src/retroshare/rsflags.h @@ -241,7 +241,3 @@ typedef t_RsFlags32 ServicePermissionFlags ; // typedef t_RsFlags32 ChatLobbyFlags ; -/// @deprecated Flags for serializer -RS_DEPRECATED_FOR(RsSerializationFlags) -typedef t_RsFlags32 SerializationFlags ; - diff --git a/libretroshare/src/retroshare/rspeers.h b/libretroshare/src/retroshare/rspeers.h index 143060bc1..5cac456f2 100644 --- a/libretroshare/src/retroshare/rspeers.h +++ b/libretroshare/src/retroshare/rspeers.h @@ -803,7 +803,7 @@ public: ServicePermissionFlags flags = RS_NODE_PERM_DEFAULT ) = 0; - /* Auth Stuff */ + RS_DEPRECATED /// This function doesn't provide meaningful error reporting virtual std::string getPGPKey(const RsPgpId& pgp_id,bool include_signatures) = 0; virtual bool GetPGPBase64StringAndCheckSum(const RsPgpId& gpg_id,std::string& gpg_base64_string,std::string& gpg_base64_checksum) = 0; diff --git a/libretroshare/src/rsitems/rsitem.h b/libretroshare/src/rsitems/rsitem.h index d8495ab58..1c8ea9789 100644 --- a/libretroshare/src/rsitems/rsitem.h +++ b/libretroshare/src/rsitems/rsitem.h @@ -45,11 +45,12 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable /// TODO: Do this make sense with the new serialization system? virtual void clear() = 0; + /// @deprecated use << ostream operator instead + RS_DEPRECATED_FOR("<< ostream operator") virtual std::ostream &print(std::ostream &out, uint16_t /* indent */ = 0) { RsGenericSerializer::SerializeContext ctx( - NULL, 0, RsGenericSerializer::FORMAT_BINARY, - RsGenericSerializer::SERIALIZATION_FLAG_NONE ); + nullptr, 0, RsSerializationFlags::NONE ); serial_process(RsGenericSerializer::PRINT,ctx); return out; } diff --git a/libretroshare/src/rsitems/rsmsgitems.cc b/libretroshare/src/rsitems/rsmsgitems.cc index 99fc342a5..fa4991b8c 100644 --- a/libretroshare/src/rsitems/rsmsgitems.cc +++ b/libretroshare/src/rsitems/rsmsgitems.cc @@ -147,8 +147,8 @@ void RsMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSeri RsTypeSerializer::serial_process(j,ctx,attachment,"attachment"); - if(ctx.mFlags & RsServiceSerializer::SERIALIZATION_FLAG_CONFIG) - RsTypeSerializer::serial_process(j,ctx,msgId,"msgId"); + if(!!(ctx.mFlags & RsSerializationFlags::CONFIG)) + RS_SERIAL_PROCESS(msgId); } void RsMsgTagType::clear() diff --git a/libretroshare/src/rsitems/rsmsgitems.h b/libretroshare/src/rsitems/rsmsgitems.h index ad84ec3ba..7757288e7 100644 --- a/libretroshare/src/rsitems/rsmsgitems.h +++ b/libretroshare/src/rsitems/rsmsgitems.h @@ -19,8 +19,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_MSG_ITEMS_H -#define RS_MSG_ITEMS_H +#pragma once #include @@ -33,10 +32,6 @@ #include "serialiser/rstlvfileitem.h" #include "grouter/grouteritems.h" -#if 0 -#include "serialiser/rstlvtypes.h" -#include "serialiser/rstlvfileitem.h" -#endif /**************************************************************************/ @@ -218,17 +213,12 @@ class RsMsgParentId : public RsMessageItem class RsMsgSerialiser: public RsServiceSerializer { - public: - RsMsgSerialiser(SerializationFlags flags = RsServiceSerializer::SERIALIZATION_FLAG_NONE) - :RsServiceSerializer(RS_SERVICE_TYPE_MSG,RsGenericSerializer::FORMAT_BINARY,flags){} +public: + RsMsgSerialiser( + RsSerializationFlags flags = RsSerializationFlags::NONE ): + RsServiceSerializer(RS_SERVICE_TYPE_MSG, flags){} - virtual ~RsMsgSerialiser() {} + RsItem* create_item(uint16_t service,uint8_t type) const override; - virtual RsItem *create_item(uint16_t service,uint8_t type) const ; + ~RsMsgSerialiser() override = default; }; - -/**************************************************************************/ - -#endif /* RS_MSG_ITEMS_H */ - - diff --git a/libretroshare/src/rsserver/p3peers.cc b/libretroshare/src/rsserver/p3peers.cc index d4c99d259..50c026c7c 100644 --- a/libretroshare/src/rsserver/p3peers.cc +++ b/libretroshare/src/rsserver/p3peers.cc @@ -3,8 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2008 by Robert Fernie * - * Copyright (C) 2015-2018 Gioacchino Mazzurco * + * Copyright (C) 2004-2008 Robert Fernie * + * Copyright (C) 2015-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -20,7 +21,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#include "util/radix64.h" + #include "pgp/pgpkeyutil.h" #include "rsserver/p3peers.h" @@ -35,7 +36,8 @@ #include "retroshare/rsinit.h" #include "retroshare/rsfiles.h" #include "util/rsurl.h" - +#include "util/radix64.h" +#include "util/rsbase64.h" #include "pgp/rscertificate.h" #include @@ -1095,51 +1097,59 @@ bool p3Peers::setProxyServer(const uint32_t type, const std::string &addr_str, c std::string p3Peers::getPGPKey(const RsPgpId& pgp_id,bool include_signatures) { - unsigned char *mem_block = NULL; + rs_owner_ptr mem_block = nullptr; size_t mem_block_size = 0; if( !AuthGPG::getAuthGPG()->exportPublicKey( RsPgpId(pgp_id), mem_block, mem_block_size, false, include_signatures ) ) { - std::cerr << "Cannot output certificate for id \"" << pgp_id - << "\". Sorry." << std::endl; - return "" ; + RsErr() << __PRETTY_FUNCTION__ + << " Failure retriving certificate for id " << pgp_id + << std::endl; + return ""; } - RsPeerDetails Detail; - if(!getGPGDetails(pgp_id,Detail)) return ""; + RsPeerDetails details; + if(!getGPGDetails(pgp_id, details)) return ""; - RsCertificate cert( Detail,mem_block,mem_block_size ); - delete[] mem_block ; + auto certPtr = + RsCertificate::fromMemoryBlock(details, mem_block, mem_block_size); - return cert.armouredPGPKey(); + free(mem_block); + + if(certPtr) return certPtr->armouredPGPKey(); + + return ""; } -bool p3Peers::GetPGPBase64StringAndCheckSum( const RsPgpId& gpg_id, - std::string& gpg_base64_string, - std::string& gpg_base64_checksum) +bool p3Peers::GetPGPBase64StringAndCheckSum( + const RsPgpId& gpg_id, + std::string& gpg_base64_string, std::string& gpg_base64_checksum ) { gpg_base64_string = "" ; gpg_base64_checksum = "" ; - unsigned char *mem_block ; - size_t mem_block_size ; + rs_owner_ptr mem_block = nullptr; + size_t mem_block_size = 0; + if(!AuthGPG::getAuthGPG()->exportPublicKey( + gpg_id,mem_block,mem_block_size,false,false )) + return false; - if(!AuthGPG::getAuthGPG()->exportPublicKey(gpg_id,mem_block,mem_block_size,false,false)) - return false ; + RsBase64::encode(mem_block, mem_block_size, gpg_base64_string, true, false); - Radix64::encode(mem_block,mem_block_size,gpg_base64_string) ; + uint32_t crc = PGPKeyManagement::compute24bitsCRC(mem_block,mem_block_size); - uint32_t crc = PGPKeyManagement::compute24bitsCRC((unsigned char *)mem_block,mem_block_size) ; + free(mem_block); - unsigned char tmp[3] = { uint8_t((crc >> 16) & 0xff), uint8_t((crc >> 8) & 0xff), uint8_t(crc & 0xff) } ; - Radix64::encode(tmp,3,gpg_base64_checksum) ; + unsigned char tmp[3] = { + uint8_t((crc >> 16) & 0xff), + uint8_t((crc >> 8) & 0xff), + uint8_t(crc & 0xff) } ; + RsBase64::encode(tmp, 3, gpg_base64_checksum, true, false); - delete[] mem_block ; - - return true ; + return true; } enum class RsShortInviteFieldType : uint8_t diff --git a/libretroshare/src/rsserver/p3peers.h b/libretroshare/src/rsserver/p3peers.h index 86e878192..e4ad390ed 100644 --- a/libretroshare/src/rsserver/p3peers.h +++ b/libretroshare/src/rsserver/p3peers.h @@ -141,7 +141,8 @@ public: virtual std::string GetRetroshareInvite( const RsPeerId& ssl_id = RsPeerId(), bool include_signatures = false, bool includeExtraLocators = true ); - virtual std::string getPGPKey(const RsPgpId& pgp_id,bool include_signatures); + RS_DEPRECATED /// @see RsPeers + std::string getPGPKey(const RsPgpId& pgp_id,bool include_signatures) override; virtual bool GetPGPBase64StringAndCheckSum(const RsPgpId& gpg_id,std::string& gpg_base64_string,std::string& gpg_base64_checksum); diff --git a/libretroshare/src/serialiser/rsserializer.cc b/libretroshare/src/serialiser/rsserializer.cc index 71d7abea3..e59b7d9fd 100644 --- a/libretroshare/src/serialiser/rsserializer.cc +++ b/libretroshare/src/serialiser/rsserializer.cc @@ -4,6 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2016 Cyril Soler * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -18,23 +20,17 @@ * You should have received a copy of the GNU Lesser General Public License * * along with this program. If not, see . * * * - *******************************************************************************/ + ******************************************************************************/ + #include #include "rsitems/rsitem.h" - #include "util/rsprint.h" #include "serialiser/rsserializer.h" #include "serialiser/rstypeserializer.h" #include "util/stacktrace.h" #include "util/rsdebug.h" -const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_NONE ( 0x0000 ); -const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_CONFIG ( 0x0001 ); -const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE ( 0x0002 ); -const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER ( 0x0004 ); -const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ( 0x0008 ); - RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size) { if(!data || !size || !*size) @@ -47,11 +43,13 @@ RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size) return nullptr; } - if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER) - { - std::cerr << "(EE) Cannot deserialise item with flags SERIALIZATION_FLAG_SKIP_HEADER. Check your code!" << std::endl; - return NULL ; - } + if(!!(mFlags & RsSerializationFlags::SKIP_HEADER)) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot deserialise item with flag " + << "SKIP_HEADER. Check your code!" << std::endl; + print_stacktrace(); + return nullptr; + } uint32_t rstype = getRsItemId(const_cast((const void*)data)) ; @@ -64,7 +62,9 @@ RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size) return NULL ; } - SerializeContext ctx(const_cast(static_cast(data)),*size,mFormat,mFlags); + SerializeContext ctx( + const_cast(static_cast(data)), *size, + mFlags ); ctx.mOffset = 8 ; item->serial_process(RsGenericSerializer::DESERIALIZE, ctx) ; @@ -85,11 +85,13 @@ RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size) } RsItem *RsConfigSerializer::deserialise(void *data, uint32_t *size) { - if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER) - { - std::cerr << "(EE) Cannot deserialise item with flags SERIALIZATION_FLAG_SKIP_HEADER. Check your code!" << std::endl; - return NULL ; - } + if(!!(mFlags & RsSerializationFlags::SKIP_HEADER)) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot deserialise item with flag " + << "SKIP_HEADER. Check your code!" << std::endl; + print_stacktrace(); + return nullptr; + } uint32_t rstype = getRsItemId(const_cast((const void*)data)) ; @@ -102,7 +104,9 @@ RsItem *RsConfigSerializer::deserialise(void *data, uint32_t *size) return NULL ; } - SerializeContext ctx(const_cast(static_cast(data)),*size,mFormat,mFlags); + SerializeContext ctx( + const_cast(static_cast(data)), *size, + mFlags ); ctx.mOffset = 8 ; item->serial_process(DESERIALIZE, ctx) ; @@ -121,50 +125,44 @@ RsItem *RsConfigSerializer::deserialise(void *data, uint32_t *size) delete item ; return NULL ; } -bool RsGenericSerializer::serialise(RsItem *item,void *data,uint32_t *size) + +bool RsGenericSerializer::serialise(RsItem* item, void* data, uint32_t* size) { - SerializeContext ctx(static_cast(data),0,mFormat,mFlags); + uint32_t tlvsize = this->size(item); - uint32_t tlvsize = this->size(item) ; + constexpr auto fName = __PRETTY_FUNCTION__; + const auto failure = [=](std::error_condition ec) + { + RsErr() << fName << " " << ec << std::endl; + print_stacktrace(); + return false; + }; - if(tlvsize > *size) - throw std::runtime_error("Cannot serialise: not enough room.") ; + if(tlvsize > *size) return failure(std::errc::no_buffer_space); + SerializeContext ctx(static_cast(data), tlvsize, mFlags); - if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER) - ctx.mOffset = 0; - else + if(!(mFlags & RsSerializationFlags::SKIP_HEADER)) { if(!setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize)) - { - std::cerr << "RsSerializer::serialise_item(): ERROR. Not enough size!" << std::endl; - return false ; - } + return failure(std::errc::no_buffer_space); ctx.mOffset = 8; } - ctx.mSize = tlvsize; + item->serial_process(RsGenericSerializer::SERIALIZE,ctx); - item->serial_process(RsGenericSerializer::SERIALIZE,ctx) ; + if(ctx.mSize != ctx.mOffset) return failure(std::errc::message_size); - if(ctx.mSize != ctx.mOffset) - { - std::cerr << "RsSerializer::serialise(): ERROR. offset does not match expected size!" << std::endl; - return false ; - } - *size = ctx.mOffset ; - - return true ; + *size = ctx.mOffset; + return true; } uint32_t RsGenericSerializer::size(RsItem *item) { - SerializeContext ctx(NULL,0,mFormat,mFlags); + SerializeContext ctx(nullptr, 0, mFlags); - if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER) - ctx.mOffset = 0; - else - ctx.mOffset = 8 ; // header size + if(!!(mFlags & RsSerializationFlags::SKIP_HEADER)) ctx.mOffset = 0; + else ctx.mOffset = 8; // header size item->serial_process(SIZE_ESTIMATE, ctx) ; return ctx.mOffset ; @@ -172,7 +170,7 @@ uint32_t RsGenericSerializer::size(RsItem *item) void RsGenericSerializer::print(RsItem *item) { - SerializeContext ctx(NULL,0,mFormat,mFlags); + SerializeContext ctx(nullptr, 0, mFlags); std::cerr << "***** RsItem class: \"" << typeid(*item).name() << "\" *****" << std::endl; item->serial_process(PRINT, ctx) ; @@ -255,7 +253,7 @@ RsItem *RsRawSerialiser::deserialise(void *data, uint32_t *pktsize) RsGenericSerializer::SerializeContext::SerializeContext( - uint8_t* data, uint32_t size, SerializationFlags flags, + uint8_t* data, uint32_t size, RsSerializationFlags flags, RsJson::AllocatorType* allocator ) : mData(data), mSize(size), mOffset(0), mOk(true), mFlags(flags), mJson(rapidjson::kObjectType, allocator) @@ -264,20 +262,23 @@ RsGenericSerializer::SerializeContext::SerializeContext( { if(size == 0) { - std::cerr << __PRETTY_FUNCTION__ << " data passed without " - << "size! This make no sense report to developers!" - << std::endl; + RsFatal() << __PRETTY_FUNCTION__ << " data passed without " + << "size! This make no sense report to developers!" + << std::endl; print_stacktrace(); + exit(-EINVAL); } - if(flags & SERIALIZATION_FLAG_YIELDING) + if(!!(flags & RsSerializationFlags::YIELDING)) { - std::cerr << __PRETTY_FUNCTION__ << " Attempt to create a " + RsFatal() << __PRETTY_FUNCTION__ + << " Attempt to create a " << "binary serialization context with " << "SERIALIZATION_FLAG_YIELDING! " << "This make no sense report to developers!" << std::endl; print_stacktrace(); + exit(-EINVAL); } } } diff --git a/libretroshare/src/serialiser/rsserializer.h b/libretroshare/src/serialiser/rsserializer.h index 60f4a21d8..1b67e2675 100644 --- a/libretroshare/src/serialiser/rsserializer.h +++ b/libretroshare/src/serialiser/rsserializer.h @@ -238,58 +238,23 @@ struct RsGenericSerializer : RsSerialType FROM_JSON } SerializeJob; - - /** @deprecated use SerializeJob instead */ - RS_DEPRECATED typedef enum - { - FORMAT_BINARY = 0x01, - FORMAT_JSON = 0x02 - } SerializationFormat; - struct SerializeContext { /** Allow shared allocator usage to avoid costly JSON deepcopy for * nested RsSerializable */ SerializeContext( uint8_t* data = nullptr, uint32_t size = 0, - SerializationFlags flags = SERIALIZATION_FLAG_NONE, + RsSerializationFlags flags = RsSerializationFlags::NONE, RsJson::AllocatorType* allocator = nullptr); - RS_DEPRECATED SerializeContext( - uint8_t *data, uint32_t size, SerializationFormat format, - SerializationFlags flags, - RsJson::AllocatorType* allocator = nullptr) : - mData(data), mSize(size), mOffset(0), mOk(true), mFormat(format), - mFlags{flags}, mJson(rapidjson::kObjectType, allocator) {} - unsigned char *mData; uint32_t mSize; uint32_t mOffset; bool mOk; - RS_DEPRECATED SerializationFormat mFormat; - SerializationFlags mFlags; + RsSerializationFlags mFlags; RsJson mJson; }; - /** These are convenience flags to be used by the items when processing the - * data. The names of the flags are not very important. What matters is that - * the serial_process() method of each item correctly deals with the data - * when it sees the flags, if the serialiser sets them. - * By default the flags are not set and shouldn't be handled. - * When deriving a new serializer, the user can set his own flags, using - * compatible values - */ - static const SerializationFlags SERIALIZATION_FLAG_NONE; // 0x0000 - static const SerializationFlags SERIALIZATION_FLAG_CONFIG; // 0x0001 - static const SerializationFlags SERIALIZATION_FLAG_SIGNATURE; // 0x0002 - static const SerializationFlags SERIALIZATION_FLAG_SKIP_HEADER; // 0x0004 - - /** Used for JSON deserialization in JSON API, it causes the deserialization - * to continue even if some field is missing (or incorrect), this way the - * API is more user friendly as some methods need just part of the structs - * they take as parameters. */ - static const SerializationFlags SERIALIZATION_FLAG_YIELDING; // 0x0008 - /** * The following functions overload RsSerialType. * They *should not* need to be further overloaded. @@ -302,18 +267,15 @@ struct RsGenericSerializer : RsSerialType protected: RsGenericSerializer( uint8_t serial_class, uint8_t serial_type, - SerializationFormat format, SerializationFlags flags ) : + RsSerializationFlags flags ): RsSerialType( RS_PKT_VERSION1, serial_class, serial_type), - mFormat(format), mFlags(flags) {} - - RsGenericSerializer( - uint16_t service, SerializationFormat format, - SerializationFlags flags ) : - RsSerialType( RS_PKT_VERSION_SERVICE, service ), mFormat(format), mFlags(flags) {} - SerializationFormat mFormat; - SerializationFlags mFlags; + RsGenericSerializer( + uint16_t service, RsSerializationFlags flags ): + RsSerialType( RS_PKT_VERSION_SERVICE, service ), mFlags(flags) {} + + RsSerializationFlags mFlags; }; @@ -323,9 +285,9 @@ protected: struct RsServiceSerializer : RsGenericSerializer { RsServiceSerializer( - uint16_t service_id, SerializationFormat format = FORMAT_BINARY, - SerializationFlags flags = SERIALIZATION_FLAG_NONE ) : - RsGenericSerializer(service_id, format, flags) {} + uint16_t service_id, + RsSerializationFlags flags = RsSerializationFlags::NONE ) : + RsGenericSerializer(service_id, flags) {} /*! should be overloaded to create the correct type of item depending on the * data */ @@ -342,11 +304,10 @@ struct RsServiceSerializer : RsGenericSerializer */ struct RsConfigSerializer : RsGenericSerializer { - RsConfigSerializer(uint8_t config_class, - uint8_t config_type, - SerializationFormat format = FORMAT_BINARY, - SerializationFlags flags = SERIALIZATION_FLAG_NONE) : - RsGenericSerializer(config_class,config_type,format,flags) {} + RsConfigSerializer( + uint8_t config_class, uint8_t config_type, + RsSerializationFlags flags = RsSerializationFlags::NONE ) : + RsGenericSerializer(config_class, config_type, flags) {} /*! should be overloaded to create the correct type of item depending on the * data */ diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index d57f3d0d4..eafc655a6 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -542,7 +542,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( uint32_t serialSize = 0; RS_SERIAL_PROCESS(serialSize); if(!ctx.mOk) break; - ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE; + ctx.mOk = serialSize <= MAX_SERIALIZED_CHUNK_SIZE; if(!ctx.mOk) { RsErr() << __PRETTY_FUNCTION__ @@ -555,6 +555,8 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( if(!serialSize) { + Dbg3() << __PRETTY_FUNCTION__ << " Deserialized empty memory chunk" + << std::endl; clear(); break; } @@ -573,7 +575,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( if(serialSize != second) { first = reinterpret_cast(realloc(first, serialSize)); - second = static_cast(serialSize); + second = serialSize; } memcpy(first, ctx.mData + ctx.mOffset, second); @@ -593,8 +595,8 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( } case RsGenericSerializer::FROM_JSON: { - const bool yelding = !!( RsSerializationFlags::YIELDING & - ctx.mFlags.toEFT() ); + const bool yelding = !!( + RsSerializationFlags::YIELDING & ctx.mFlags ); if(!(ctx.mOk || yelding)) { clear(); diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index a8d61fdeb..8c0eb9784 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -82,8 +82,7 @@ struct RsTypeSerializer INTT& member, const std::string& member_name ) { const bool VLQ_ENCODING = !!( - RsSerializationFlags::INTEGER_VLQ & - ctx.mFlags.toEFT() ); + RsSerializationFlags::INTEGER_VLQ & ctx.mFlags ); switch(j) { @@ -149,8 +148,7 @@ struct RsTypeSerializer break; case RsGenericSerializer::FROM_JSON: ctx.mOk &= ( ctx.mOk || - !!( RsSerializationFlags::YIELDING & - ctx.mFlags.toEFT() ) ) + !!(RsSerializationFlags::YIELDING & ctx.mFlags) ) && from_JSON(member_name, member, ctx.mJson); break; default: fatalUnknownSerialJob(j); @@ -198,7 +196,8 @@ struct RsTypeSerializer ctx.mOk = ctx.mOk && to_JSON(member_name, member, ctx.mJson); break; case RsGenericSerializer::FROM_JSON: - ctx.mOk &= (ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING) + ctx.mOk &= ( ctx.mOk || + !!(ctx.mFlags & RsSerializationFlags::YIELDING) ) && from_JSON(member_name, member, ctx.mJson); break; default: fatalUnknownSerialJob(j); @@ -236,7 +235,7 @@ struct RsTypeSerializer break; case RsGenericSerializer::FROM_JSON: ctx.mOk &= - (ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING) + (ctx.mOk || !!(ctx.mFlags & RsSerializationFlags::YIELDING)) && from_JSON(member_name, type_id, member, ctx.mJson); break; default: fatalUnknownSerialJob(j); @@ -324,7 +323,7 @@ struct RsTypeSerializer { using namespace rapidjson; - bool ok = ctx.mOk || ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + bool ok = ctx.mOk || !!(ctx.mFlags & RsSerializationFlags::YIELDING); Document& jDoc(ctx.mJson); Document::AllocatorType& allocator = jDoc.GetAllocator(); @@ -424,8 +423,7 @@ struct RsTypeSerializer RsJson& jDoc(ctx.mJson); const char* mName = memberName.c_str(); bool hasMember = jDoc.HasMember(mName); - bool yielding = ctx.mFlags & - RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + bool yielding = !!(ctx.mFlags & RsSerializationFlags::YIELDING); if(!hasMember) { @@ -560,8 +558,7 @@ struct RsTypeSerializer { using namespace rapidjson; - bool ok = ctx.mOk || ctx.mFlags & - RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + bool ok = ctx.mOk || !!(ctx.mFlags & RsSerializationFlags::YIELDING); Document& jDoc(ctx.mJson); Document::AllocatorType& allocator = jDoc.GetAllocator(); @@ -669,8 +666,8 @@ struct RsTypeSerializer break; case RsGenericSerializer::FROM_JSON: { - bool ok = ctx.mOk || !!( ctx.mFlags.toEFT() - & RsSerializationFlags::YIELDING ); + bool ok = ctx.mOk || !!( + ctx.mFlags & RsSerializationFlags::YIELDING ); ctx.mOk = ok && from_JSON(memberName, member, ctx.mJson) && ctx.mOk; break; } @@ -760,8 +757,7 @@ struct RsTypeSerializer RsJson& jDoc(ctx.mJson); const char* mName = memberName.c_str(); bool hasMember = jDoc.HasMember(mName); - bool yielding = ctx.mFlags & - RsGenericSerializer::SERIALIZATION_FLAG_YIELDING; + bool yielding = !!(ctx.mFlags & RsSerializationFlags::YIELDING); if(!hasMember) { diff --git a/libretroshare/src/services/p3msgservice.cc b/libretroshare/src/services/p3msgservice.cc index b1c1bf847..a04ba5fce 100644 --- a/libretroshare/src/services/p3msgservice.cc +++ b/libretroshare/src/services/p3msgservice.cc @@ -80,7 +80,9 @@ p3MsgService::p3MsgService( p3ServiceControl *sc, p3IdService *id_serv, recentlyReceivedMutex("p3MsgService recently received hash mutex"), mGxsTransServ(gxsMS) { - _serialiser = new RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_NONE); // this serialiser is used for services. It's not the same than the one returned by setupSerialiser(). We need both!! + /* this serialiser is used for services. It's not the same than the one + * returned by setupSerialiser(). We need both!! */ + _serialiser = new RsMsgSerialiser(); addSerialType(_serialiser); /* MsgIds are not transmitted, but only used locally as a storage index. @@ -509,7 +511,7 @@ RsSerialiser* p3MsgService::setupSerialiser() // this serialiser is used for con { RsSerialiser *rss = new RsSerialiser ; - rss->addSerialType(new RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_CONFIG)); + rss->addSerialType(new RsMsgSerialiser(RsSerializationFlags::CONFIG)); rss->addSerialType(new RsGeneralConfigSerialiser()); return rss; @@ -2386,10 +2388,11 @@ void p3MsgService::sendDistantMsgItem(RsMsgItem *msgitem) /* The item is serialized and turned into a generic turtle item. Use use the * explicit serialiser to make sure that the msgId is not included */ - uint32_t msg_serialized_rssize = RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_NONE).size(msgitem) ; + uint32_t msg_serialized_rssize = RsMsgSerialiser().size(msgitem); RsTemporaryMemory msg_serialized_data(msg_serialized_rssize) ; - if(!RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_NONE).serialise(msgitem,msg_serialized_data,&msg_serialized_rssize)) + if( !RsMsgSerialiser(). + serialise(msgitem,msg_serialized_data,&msg_serialized_rssize) ) { std::cerr << "(EE) p3MsgService::sendTurtleData(): Serialization error." << std::endl; return ; From a509098a7958f2b449ec13a750196bc47cbed3d6 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 18 Mar 2020 23:10:57 +0100 Subject: [PATCH 05/40] Enable C++14 support GCC with C++14 support is already shipped in all supported distributions --- RetroShare.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RetroShare.pro b/RetroShare.pro index e835e4ca2..dc590dd17 100644 --- a/RetroShare.pro +++ b/RetroShare.pro @@ -19,7 +19,7 @@ # SPDX-FileCopyrightText: Retroshare Team # SPDX-License-Identifier: LGPL-3.0-or-later -CONFIG += c++11 +CONFIG += c++14 !include("retroshare.pri"): error("Could not include file retroshare.pri") From da66719b8346328974e08f56ccd175af7ba6e3f4 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 19 Mar 2020 17:37:24 +0100 Subject: [PATCH 06/40] Fix Android compilation --- libretroshare/src/serialiser/rstypeserializer.cc | 4 ++++ libretroshare/src/serialiser/rstypeserializer.h | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index eafc655a6..0144201b4 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -503,6 +503,10 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/, // Binary blocks // //============================================================================// +/*static*/ /* without this Android compilation breaks */ +constexpr uint32_t RsTypeSerializer::RawMemoryWrapper::MAX_SERIALIZED_CHUNK_SIZE; + +/*static*/ void RsTypeSerializer::RawMemoryWrapper::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 8c0eb9784..52bf255c7 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -430,8 +430,7 @@ struct RsTypeSerializer if(!yielding) { RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName - << "\" not found in JSON:" << std::endl - << jDoc << std::endl << std::endl; + << "\" not found in JSON" << std::endl; print_stacktrace(); } ctx.mOk = false; From fc5ad40c90d30912417194ba6621937aabc00251 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 19 Mar 2020 17:48:02 +0100 Subject: [PATCH 07/40] Add C++14 template types helpers that seems missing in TravisCI MacOSX --- libretroshare/src/libretroshare.pro | 3 +- .../src/serialiser/rstypeserializer.h | 2 + libretroshare/src/util/cxx14retrocompat.h | 38 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 libretroshare/src/util/cxx14retrocompat.h diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index a923475d5..54a4989ec 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -495,7 +495,8 @@ HEADERS += util/folderiterator.h \ util/rstime.h \ util/stacktrace.h \ util/rsdeprecate.h \ - util/cxx11retrocompat.h \ + util/cxx11retrocompat.h \ + util/cxx14retrocompat.h \ util/cxx17retrocompat.h \ util/rsurl.h \ util/rserrno.h diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 52bf255c7..1290ec392 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -40,6 +40,8 @@ #include "serialiser/rsserializable.h" #include "util/rsjson.h" #include "util/rsdebug.h" +#include "util/cxx14retrocompat.h" + struct RsTypeSerializer { diff --git a/libretroshare/src/util/cxx14retrocompat.h b/libretroshare/src/util/cxx14retrocompat.h new file mode 100644 index 000000000..e7e7d9e99 --- /dev/null +++ b/libretroshare/src/util/cxx14retrocompat.h @@ -0,0 +1,38 @@ +/******************************************************************************* + * * + * libretroshare < C++14 retro-compatibility helpers * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#if __cplusplus < 201402L + +#include + +namespace std +{ +template using decay_t = typename decay::type; + +template +using enable_if_t = typename enable_if::type; + +template +using remove_const_t = typename remove_const::type; +} +#endif // __cplusplus < 201402L From d666e5840361cd78a4e5e41541c1c2fdc9db2c87 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 21 Mar 2020 16:03:22 +0100 Subject: [PATCH 08/40] Fix regression in sequence container FROM_JSON --- .../src/serialiser/rstypeserializer.h | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 1290ec392..2c3d17f3a 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -567,31 +567,29 @@ struct RsTypeSerializer arrKey.SetString( memberName.c_str(), static_cast(memberName.length()) ); - ok = ok && jDoc.IsObject(); - ok = ok && jDoc.HasMember(arrKey); + ok = ok && jDoc.IsObject() && jDoc.HasMember(arrKey) + && jDoc[arrKey].IsArray(); + if(!ok) { ctx.mOk = false; break; } - if(ok && jDoc[arrKey].IsArray()) + for (auto&& arrEl : jDoc[arrKey].GetArray()) { - for (auto&& arrEl : jDoc[arrKey].GetArray()) - { - Value arrKeyT; - arrKeyT.SetString( - memberName.c_str(), - static_cast(memberName.length()) ); + Value arrKeyT; + arrKeyT.SetString( + memberName.c_str(), + static_cast(memberName.length()) ); - RsGenericSerializer::SerializeContext elCtx( - nullptr, 0, ctx.mFlags, &allocator ); - elCtx.mJson.AddMember(arrKeyT, arrEl, allocator); + RsGenericSerializer::SerializeContext elCtx( + nullptr, 0, ctx.mFlags, &allocator ); + elCtx.mJson.AddMember(arrKeyT, arrEl, allocator); - el_t el; - serial_process(j, elCtx, el, ""); - ok = ok && elCtx.mOk; - ctx.mOk &= ok; - if(ok) member.insert(member.end(), el); - else break; - } + el_t el; + serial_process(j, elCtx, el, memberName); + ok = ok && elCtx.mOk; + ctx.mOk &= ok; + if(ok) member.insert(member.end(), el); + else break; } - else ctx.mOk = false; + break; } default: fatalUnknownSerialJob(j); From 55aab6c447695d8d4a09520889fcb843188d83f6 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 22 Mar 2020 10:49:55 +0100 Subject: [PATCH 09/40] Add RsFiles::requestFiles to de API to download whole colletions Fix filetree creation from single file RsDirUtil::moveFile now works also if parent directories doesn't exists Backport std::filesystem::create_directories from C++17 --- libretroshare/src/file_sharing/file_tree.cc | 5 ++ libretroshare/src/ft/ftcontroller.cc | 44 +++++----- libretroshare/src/ft/ftserver.cc | 92 ++++++++++++++++++++- libretroshare/src/ft/ftserver.h | 8 ++ libretroshare/src/retroshare/rsfiles.h | 29 +++++-- libretroshare/src/retroshare/rstypes.h | 2 + libretroshare/src/util/rsdir.cc | 79 ++++++++++++------ libretroshare/src/util/rsdir.h | 22 ++++- 8 files changed, 227 insertions(+), 54 deletions(-) diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 55de0e78b..3cf893402 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -185,6 +185,11 @@ std::unique_ptr RsFileTree::fromDirDetails( ft->mFiles.push_back(fd); ft->mTotalFiles = 1; ft->mTotalSize = fd.size; + + DirData dd; + dd.name = "/"; + dd.subfiles.push_back(0); + ft->mDirs.push_back(dd); } else recurs_buildFileTree(*ft, 0, dd, remote, remove_top_dirs ); return ft; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index 0af02144f..29494f189 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -38,7 +38,7 @@ #include "util/rsdiscspace.h" #include "util/rsmemory.h" #include "util/rstime.h" - +#include "util/cxx17retrocompat.h" #include "ft/ftcontroller.h" #include "ft/ftfilecreator.h" @@ -923,10 +923,15 @@ bool ftController::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) return false ; } -bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash, - uint64_t size, const std::string& dest, TransferRequestFlags flags, - const std::list &_srcIds, uint16_t state) +bool ftController::FileRequest( + const std::string& fname, const RsFileHash& hash, uint64_t size, + const std::string& dest, TransferRequestFlags flags, + const std::list &_srcIds, uint16_t state ) { + /* TODO: To support collections faithfully we need to be able to save + * the same file to multiple locations, both if already downloaded or + * incomplete */ + std::list srcIds(_srcIds) ; /* check if we have the file */ @@ -965,8 +970,9 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // create void file with the target name. FILE *f = RsDirUtil::rs_fopen(destination.c_str(),"w") ; - if(f == NULL) - std::cerr << "Could not open file " << destination << " for writting." << std::endl ; + if(!f) + RsErr() << __PRETTY_FUNCTION__ << " Could not write file " + << destination << std::endl; else fclose(f) ; @@ -979,12 +985,13 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // if(size >= 1024ull*1024ull*((1ull << 32) - 1)) { - std::cerr << "FileRequest Error: unexpected size. This is probably a bug." << std::endl; - std::cerr << " name = " << fname << std::endl ; - std::cerr << " flags = " << flags << std::endl ; - std::cerr << " dest = " << dest << std::endl ; - std::cerr << " size = " << size << std::endl ; - return false ; + RsErr() << __PRETTY_FUNCTION__ + << " unexpected size. This is probably a bug." + << " name = " << fname + << " flags = " << flags + << " dest = " << dest + << " size = " << size << std::endl; + return false; } /* If file transfer is not enabled .... @@ -1004,8 +1011,8 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash } } - // remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files. - // + /* remove the sources from the list, if they don't have clearance for direct + * transfer. This happens only for non cache files. */ for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) { bool bAllowDirectDL = false; @@ -1052,11 +1059,10 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash * This is important as some guis request duplicate files regularly. */ - { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ - - std::map::const_iterator dit = mDownloads.find(hash); + { + RS_STACK_MUTEX(ctrlMutex); + auto dit = std::as_const(mDownloads).find(hash); if (dit != mDownloads.end()) { /* we already have it! */ @@ -1110,7 +1116,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash return true; } - } /******* UNLOCKED ********/ + } // RS_STACK_MUTEX(ctrlMutex); unlocked if(mSearch && !(flags & RS_FILE_REQ_NO_SEARCH)) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 678ac4a02..7ec4251b3 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -295,7 +295,10 @@ bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info); } -bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) +bool ftServer::FileRequest( + const std::string& fname, const RsFileHash& hash, uint64_t size, + const std::string& dest, TransferRequestFlags flags, + const std::list& srcIds ) { #ifdef SERVER_DEBUG FTSERVER_DEBUG() << "Requesting " << fname << std::endl ; @@ -307,6 +310,93 @@ bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uin return true ; } +std::error_condition ftServer::requestFiles( + const RsFileTree& collection, const std::string& destPath, + const std::vector& srcIds, FileRequestFlags flags ) +{ + constexpr auto fname = __PRETTY_FUNCTION__; + const auto dirsCount = collection.mDirs.size(); + const auto filesCount = collection.mFiles.size(); + + Dbg2() << fname << " dirsCount: " << dirsCount + << " filesCount: " << filesCount << std::endl; + + if(!dirsCount) + { + RsErr() << fname << " Directories list empty in collection " + << std::endl; + return std::errc::not_a_directory; + } + + if(!filesCount) + { + RsErr() << fname << " Files list empty in collection " << std::endl; + return std::errc::invalid_argument; + } + + if(filesCount != collection.mTotalFiles) + { + RsErr() << fname << " Files count mismatch" << std::endl; + return std::errc::invalid_argument; + } + + std::string basePath = destPath.empty() ? getDownloadDirectory() : destPath; + // Track how many time a directory have been explored + std::vector dirsSeenCnt(dirsCount, 0); + // + using StackEntry = std::tuple; + std::deque dStack = { std::make_tuple(0, basePath) }; + + const auto exploreDir = [&](const StackEntry& se)-> std::error_condition + { + std::uintptr_t dirHandle; std::string parentPath; + std::tie(dirHandle, parentPath) = se; + + const auto& dirData = collection.mDirs[dirHandle]; + auto& seenTimes = dirsSeenCnt[dirHandle]; + std::string dirPath = RsDirUtil::makePath(parentPath, dirData.name); + + /* This check is not perfect but is cheap and interrupt loop exploration + * before it becomes pathological */ + if(seenTimes++ > dirsCount) + { + RsErr() << fname << " loop detected! dir: " + << dirHandle << " \"" << dirPath + << "\" explored too many times" << std::endl; + return std::errc::too_many_symbolic_link_levels; + } + + for(auto fHandle: dirData.subfiles) + { + if(fHandle >= filesCount) return std::errc::argument_out_of_domain; + + const RsFileTree::FileData& fData = collection.mFiles[fHandle]; + + bool fr = + FileRequest( fData.name, fData.hash, fData.size, + dirPath, + TransferRequestFlags::fromEFT(flags), + std::list(srcIds.begin(), srcIds.end()) ); + + Dbg2() << fname << " requested: " << fr << " " + << fData.hash << " -> " << dirPath << std::endl; + } + + for(auto dHandle: dirData.subdirs) + dStack.push_back(std::make_tuple(dHandle, dirPath)); + + return std::error_condition(); + }; + + while(!dStack.empty()) + { + if(std::error_condition ec = exploreDir(dStack.front())) return ec; + dStack.pop_front(); + } + + return std::error_condition(); +} + bool ftServer::activateTunnels(const RsFileHash& hash,uint32_t encryption_policy,TransferRequestFlags flags,bool onoff) { RsFileHash hash_of_hash ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index f06f0f5bd..7c8ca5ff2 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -206,6 +206,14 @@ public: virtual void setFilePermDirectDL(uint32_t perm) ; virtual uint32_t filePermDirectDL() ; + /// @see RsFiles + std::error_condition requestFiles( + const RsFileTree& collection, + const std::string& destPath = "", + const std::vector& srcIds = std::vector(), + FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING + ) override; + /// @see RsFiles bool turtleSearchRequest( const std::string& matchString, diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index ae5c00dea..661043a20 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -489,10 +489,10 @@ public: /** * @brief Initiate downloading of a file * @jsonapi{development} - * @param[in] fileName - * @param[in] hash - * @param[in] size - * @param[in] destPath in not empty specify a destination path + * @param[in] fileName file name + * @param[in] hash file hash + * @param[in] size file size + * @param[in] destPath optional specify the destination directory * @param[in] flags you usually want RS_FILE_REQ_ANONYMOUS_ROUTING * @param[in] srcIds eventually specify known sources * @return false if we already have the file, true otherwhise @@ -502,6 +502,25 @@ public: const std::string& destPath, TransferRequestFlags flags, const std::list& srcIds ) = 0; + /** + * @brief Initiate download of a files collection + * @jsonapi{development} + * An usually useful companion method of this is @see parseFilesLink() + * @param[in] collection collection of files to download + * @param[in] destPath optional base path on which to download the + * collection, if left empty the default download directory will be used + * @param[in] srcIds optional peers id known as direct source of the + * collection + * @param[in] flags optional flags to fine tune search and download + * algorithm + * @return success or error details. + */ + virtual std::error_condition requestFiles( + const RsFileTree& collection, + const std::string& destPath = "", + const std::vector& srcIds = std::vector(), + FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING ) = 0; + /** * @brief Cancel file downloading * @jsonapi{development} @@ -924,8 +943,6 @@ public: virtual std::error_condition parseFilesLink( const std::string& link, RsFileTree& collection ) = 0; - // virtual std::error_condition downloadFilesLink(FileRequestFlags) - /** * @brief Get list of ignored file name prefixes and suffixes * @param[out] ignoredPrefixes storage for ingored prefixes diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 1f5107fe3..978934942 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -366,6 +366,8 @@ struct DirDetails : RsSerializable RS_SERIAL_PROCESS(children); RS_SERIAL_PROCESS(parent_groups); } + + ~DirDetails() override = default; }; class FileDetail diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index 1b794f982..be327bb3d 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -265,36 +265,48 @@ bool RsDirUtil::fileExists(const std::string& filename) bool RsDirUtil::moveFile(const std::string& source,const std::string& dest) { - // Check that the destination directory exists. If not, create it. + Dbg3() << __PRETTY_FUNCTION__<< " source: " << source + << " dest: " << dest << std::endl; std::string dest_dir ; std::string dest_file ; - splitDirFromFile(dest,dest_dir,dest_file) ; + splitDirFromFile(dest, dest_dir, dest_file); - std::cerr << "Moving file " << source << " to " << dest << std::endl; - std::cerr << "Checking that directory " << dest_dir << " actually exists." << std::endl; + if(!checkDirectory(dest_dir)) + { + if(!std::filesystem::create_directories(dest_dir)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure creating directory: " + << dest_dir << std::endl; + return false; + } + } - if(!checkCreateDirectory(dest_dir)) - return false ; + // First try a rename + if(renameFile(source,dest)) + { + Dbg3() << __PRETTY_FUNCTION__ << " plain rename worked" << std::endl; + return true; + } - // First try a rename - // + /* If not, try to copy. The src and dest probably belong to different file + * systems */ + if(!copyFile(source,dest)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure copying file" << std::endl; + return false; + } - if(renameFile(source,dest)) - return true ; + // delete the original + if(!removeFile(source)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure deleting original file" + << std::endl; + return false; + } - // If not, try to copy. The src and dest probably belong to different file systems - - if(!copyFile(source,dest)) - return false ; - - // copy was successful, let's delete the original - - if(!removeFile(source)) - return false ; - - return true ; + return true; } bool RsDirUtil::removeFile(const std::string& filename) @@ -425,7 +437,7 @@ bool RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d } -bool RsDirUtil::checkDirectory(const std::string& dir) +bool RsDirUtil::checkDirectory(const std::string& dir) { int val; mode_t st_mode; @@ -461,11 +473,9 @@ bool RsDirUtil::checkDirectory(const std::string& dir) } -bool RsDirUtil::checkCreateDirectory(const std::string& dir) +bool RsDirUtil::checkCreateDirectory(const std::string& dir) { -#ifdef RSDIR_DEBUG - std::cerr << "RsDirUtil::checkCreateDirectory() dir: " << dir << std::endl; -#endif + Dbg3() << __PRETTY_FUNCTION__ << " " << dir << std::endl; #ifdef WINDOWS_SYS std::wstring wdir; @@ -516,6 +526,23 @@ bool RsDirUtil::checkCreateDirectory(const std::string& dir) return true; } +#if __cplusplus < 201703L +bool std::filesystem::create_directories(const std::string& path) +{ + for( std::string::size_type lastIndex = 0; lastIndex < std::string::npos; + lastIndex = path.find('/', lastIndex) ) + { + std::string&& curDir = path.substr(0, ++lastIndex); + if(!RsDirUtil::checkCreateDirectory(curDir)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure creating: " << curDir + << " of: " << path << std::endl; + return false; + } + } + return true; +} +#endif // __cplusplus < 201703L std::string RsDirUtil::removeSymLinks(const std::string& path) { diff --git a/libretroshare/src/util/rsdir.h b/libretroshare/src/util/rsdir.h index b6ef4891d..e8ff09943 100644 --- a/libretroshare/src/util/rsdir.h +++ b/libretroshare/src/util/rsdir.h @@ -84,7 +84,10 @@ int breakupDirList(const std::string& path, std::list &subdirs bool splitDirFromFile(const std::string& full_path,std::string& dir, std::string& file); bool copyFile(const std::string& source,const std::string& dest); -bool moveFile(const std::string& source,const std::string& dest); + +/** Move file. If destination directory doesn't exists create it. */ +bool moveFile(const std::string& source, const std::string& dest); + bool removeFile(const std::string& file); bool fileExists(const std::string& file); bool checkFile(const std::string& filename,uint64_t& file_size,bool disallow_empty_file = false); @@ -141,8 +144,23 @@ bool getWideFileHash(std::wstring filepath, RsFileHash &hash, u FILE *rs_fopen(const char* filename, const char* mode); std::string convertPathToUnix(std::string path); + +/** Concatenate two path pieces putting '/' separator between them only if + * needed */ std::string makePath(const std::string &path1, const std::string &path2); + +RS_SET_CONTEXT_DEBUG_LEVEL(1); } - +#if __cplusplus < 201703L +namespace std +{ +namespace filesystem +{ +bool create_directories(const std::string& path); +} +} +#endif // __cplusplus < 201703L + + #endif From 1fd6e7e97aedf9674c5bd15c4d37ce8fc709a33b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 22 Mar 2020 15:13:09 +0100 Subject: [PATCH 10/40] Add API call to create links to extra files --- libretroshare/src/ft/ftserver.cc | 44 ++++++++++++++++++++------ libretroshare/src/ft/ftserver.h | 17 +++++++--- libretroshare/src/retroshare/rsfiles.h | 22 +++++++++++-- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 7ec4251b3..1efbb715c 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -2217,16 +2217,14 @@ const noexcept } } -std::error_condition ftServer::exportFilesLink( - std::string& link, std::uintptr_t handle, bool fragSneak, - const std::string& baseUrl ) +std::error_condition ftServer::dirDetailsToLink( + std::string& link, + const DirDetails& dirDetails, bool fragSneak, const std::string& baseUrl ) { - DirDetails tDirDet; - if(!requestDirDetails(tDirDet, handle)) - return RsFilesErrorNum::FILES_HANDLE_NOT_FOUND; - std::unique_ptr tFileTree = - RsFileTree::fromDirDetails(tDirDet, false); + RsFileTree::fromDirDetails(dirDetails, false); + if(!tFileTree) return std::errc::invalid_argument; + link = tFileTree->toBase64(); if(!baseUrl.empty()) @@ -2234,7 +2232,7 @@ std::error_condition ftServer::exportFilesLink( RsUrl tUrl(baseUrl); tUrl.setQueryKV(FILES_URL_COUNT_FIELD, std::to_string(tFileTree->mTotalFiles) ) - .setQueryKV(FILES_URL_NAME_FIELD, tDirDet.name) + .setQueryKV(FILES_URL_NAME_FIELD, dirDetails.name) .setQueryKV( FILES_URL_SIZE_FIELD, std::to_string(tFileTree->mTotalSize) ); if(fragSneak) @@ -2247,6 +2245,34 @@ std::error_condition ftServer::exportFilesLink( return std::error_condition(); } +std::error_condition ftServer::exportCollectionLink( + std::string& link, std::uintptr_t handle, bool fragSneak, + const std::string& baseUrl ) +{ + DirDetails tDirDet; + if(!requestDirDetails(tDirDet, handle)) + return RsFilesErrorNum::FILES_HANDLE_NOT_FOUND; + + return dirDetailsToLink(link, tDirDet, fragSneak, baseUrl); +} + +/// @see RsFiles +std::error_condition ftServer::exportFileLink( + std::string& link, const RsFileHash& fileHash, uint64_t fileSize, + const std::string& fileName, bool fragSneak, const std::string& baseUrl ) +{ + if(fileHash.isNull() || !fileSize || fileName.empty()) + return std::errc::invalid_argument; + + DirDetails tDirDet; + tDirDet.type = DIR_TYPE_FILE; + tDirDet.name = fileName; + tDirDet.hash = fileHash; + tDirDet.count = fileSize; + + return dirDetailsToLink(link, tDirDet, fragSneak, baseUrl); +} + std::error_condition ftServer::parseFilesLink( const std::string& link, RsFileTree& collection ) { diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 7c8ca5ff2..789bba49d 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -224,11 +224,18 @@ public: virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; /// @see RsFiles - std::error_condition exportFilesLink( + std::error_condition exportCollectionLink( std::string& link, std::uintptr_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) override; + /// @see RsFiles + std::error_condition exportFileLink( + std::string& link, const RsFileHash& fileHash, uint64_t fileSize, + const std::string& fileName, bool fragSneak = false, + const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL + ) override; + /// @see RsFiles std::error_condition parseFilesLink( const std::string& link, RsFileTree& collection ) override; @@ -383,11 +390,11 @@ protected: bool findEncryptedHash(const RsPeerId& virtual_peer_id, RsFileHash& encrypted_hash); bool checkUploadLimit(const RsPeerId& pid,const RsFileHash& hash); -private: - /**** INTERNAL FUNCTIONS ***/ - //virtual int reScanDirs(); - //virtual int check_dBUpdate(); + std::error_condition dirDetailsToLink( + std::string& link, + const DirDetails& dirDetails, bool fragSneak, + const std::string& baseUrl ); private: diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 661043a20..ddd241e8f 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -912,10 +912,10 @@ public: static const std::string FILES_URL_SIZE_FIELD; /** - * @brief Get link to a forum + * @brief Export link to a collection of files * @jsonapi{development} * @param[out] link storage for the generated link - * @param[in] handle file/directory RetroShare handle + * @param[in] handle directory RetroShare handle * @param[in] fragSneak when true the file data is sneaked into fragment * instead of FILES_URL_DATA_FIELD query field, this way if using an * http[s] link to pass around a disguised file link a misconfigured host @@ -929,10 +929,26 @@ public: * plain base64 format. * @return error information if some error occurred, 0/SUCCESS otherwise */ - virtual std::error_condition exportFilesLink( + virtual std::error_condition exportCollectionLink( std::string& link, std::uintptr_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0; + /** + * @brief Export link to a file + * @jsonapi{development} + * @param[out] link @see exportCollectionLink + * @param[in] fileHash hash of the file + * @param[in] fileSize size of the file + * @param[in] fileName name of the file + * @param[in] fragSneak @see exportCollectionLink + * @param[in] baseUrl @see exportCollectionLink + * @return error @see exportCollectionLink + */ + virtual std::error_condition exportFileLink( + std::string& link, const RsFileHash& fileHash, uint64_t fileSize, + const std::string& fileName, bool fragSneak = false, + const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0; + /** * @brief Parse RetroShare files link * @jsonapi{development} From cdafb7e27fb7e9387d171b71f2a57b05eebf6391 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 23 Mar 2020 16:44:06 +0100 Subject: [PATCH 11/40] Fix a few compiler warnings --- .../src/async-method-wrapper-template.cpp.tmpl | 2 +- openpgpsdk/src/openpgpsdk/readerwriter.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl index 16bfe3304..29f53ad48 100644 --- a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl +++ b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl @@ -44,7 +44,7 @@ $%inputParamsDeserialization%$ const std::weak_ptr weakService(mService); const std::weak_ptr weakSession(session); - $%callbackName%$ = [this, weakService, weakSession]($%callbackParams%$) + $%callbackName%$ = [weakService, weakSession]($%callbackParams%$) { auto session = weakSession.lock(); if(!session || session->is_closed()) return; diff --git a/openpgpsdk/src/openpgpsdk/readerwriter.c b/openpgpsdk/src/openpgpsdk/readerwriter.c index f3a16c364..78b45e28c 100644 --- a/openpgpsdk/src/openpgpsdk/readerwriter.c +++ b/openpgpsdk/src/openpgpsdk/readerwriter.c @@ -462,9 +462,9 @@ char *ops_get_passphrase(void) { #ifndef __ANDROID__ return ops_malloc_passphrase(getpass("Passphrase: ")); -#else // __ANDROID__ - return ops_malloc_passphrase("getpass not supported on android"); -#warning "getpass not supported on android" +#else // __ANDROID + // We should never get here on Android, getpass not supported. + abort(); #endif // __ANDROID__ } From a0da5a3120218c090d76fd8722ab72b4431c0b1b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 25 Mar 2020 00:02:09 +0100 Subject: [PATCH 12/40] Improve debugging --- libretroshare/src/file_sharing/file_tree.cc | 2 ++ .../src/serialiser/rstypeserializer.h | 21 ++++++------ libretroshare/src/util/rsdebug.h | 34 ++++++++++++++----- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 3cf893402..ddc24c3ac 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -55,6 +55,8 @@ RsFileTree::fromBase64(const std::string& base64) const auto failure = [](std::error_condition ec) { return std::make_tuple(nullptr, ec); }; + if(base64.empty()) return failure(std::errc::invalid_argument); + std::error_condition ec; std::vector mem; if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec); diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 2c3d17f3a..ce53dce26 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -1037,8 +1037,6 @@ protected: VLQ_deserialize( const uint8_t data[], uint32_t size, uint32_t& offset, T& member ) { - uint32_t backupOffset = offset; - member = 0; uint32_t offsetBackup = offset; @@ -1048,14 +1046,14 @@ protected: for (size_t i = 0; offset < size && i <= sizeof(T); ++i) { member |= (data[offset] & 127) << (7 * i); - // If the next-byte flag is not set, ++ is after on purpose + // If the next-byte flag is not set. ++ is after on purpose if(!(data[offset++] & 128)) { - RsDbg() << __PRETTY_FUNCTION__ - << " size: " << size - << " backupOffset " << backupOffset - << " offset: " << offset - << " member " << member << std::endl; + Dbg2() << __PRETTY_FUNCTION__ + << " size: " << size + << " backupOffset " << offsetBackup + << " offset: " << offset + << " member " << member << std::endl; return true; } } @@ -1064,13 +1062,14 @@ protected: * ended before we encountered the end of the number, or the number * is VLQ encoded improperly */ RsErr() << __PRETTY_FUNCTION__ << std::errc::illegal_byte_sequence + << " size: " << size << " offsetBackup: " << offsetBackup << " offset: " << offset << " bytes: "; for(; offsetBackup < offset; ++offsetBackup) - std::cerr << " " << std::bitset<8>(data[offsetBackup]); - std::cerr << std::endl; - + RsErr().uStream() << " " << std::bitset<8>(data[offsetBackup]); + RsErr().uStream() << std::endl; print_stacktrace(); + return false; } diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 9478c0f74..95a9ee955 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -46,7 +46,10 @@ struct t_RsLogger { inline t_RsLogger() = default; - typedef t_RsLogger stream_type; + /** On other platforms expose the type of underlying stream. + * On Android it cannot work like that so return the class type itself + * just for code compatibility with other platforms */ + using stream_type = t_RsLogger; template inline stream_type& operator<<(const T& val) @@ -68,6 +71,11 @@ struct t_RsLogger return *this; } + /** On other platforms return underlying stream to write avoiding additional + * prefixes. On Android it cannot work like that so return the object itself + * just for code compatibility with other platforms */ + inline stream_type& uStream() { return *this; } + private: std::ostringstream ostr; }; @@ -92,7 +100,8 @@ struct t_RsLogger { inline t_RsLogger() = default; - typedef decltype(std::cerr) stream_type; + /// Expose the type of underlying stream + using stream_type = decltype(std::cerr); template inline stream_type& operator<<(const T& val) @@ -111,6 +120,9 @@ struct t_RsLogger /// needed for manipulators and things like std::endl stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) { return std::cerr << pf; } + + /// Return underlying stream to write avoiding additional prefixes + inline stream_type& uStream() const { return std::cerr; } }; #endif // def __ANDROID__ @@ -176,16 +188,22 @@ struct RsNoDbg { inline RsNoDbg() = default; - /** - * This match most of the types, but might be not enough for templated - * types - */ + /** Defined as the type itself just for code compatibility with other + * logging classes */ + using stream_type = RsNoDbg; + + /** This match most of the types, but might be not enough for templated + * types */ template - inline RsNoDbg& operator<<(const T&) { return *this; } + inline stream_type& operator<<(const T&) { return *this; } /// needed for manipulators and things like std::endl - inline RsNoDbg& operator<<(std::ostream& (*/*pf*/)(std::ostream&)) + inline stream_type& operator<<(std::ostream& (*/*pf*/)(std::ostream&)) { return *this; } + + /** Return the object itself just for code compatibility with other + * logging classes */ + inline stream_type& uStream() { return *this; } }; /** From 8b4fcbe9e005d9b3429fb8ecbd4db7a204171ec2 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 25 Mar 2020 00:35:19 +0100 Subject: [PATCH 13/40] Remove unused constant from rsfiles.h --- libretroshare/src/retroshare/rsfiles.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index ddd241e8f..a8b65fcbe 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -178,10 +178,6 @@ const TransferRequestFlags RS_FILE_REQ_NO_SEARCH ( 0x02000000 ); // di // const uint32_t RS_FILE_HINTS_SHARE_FLAGS_MASK = RS_FILE_HINTS_NETWORK_WIDE_OTHERS | RS_FILE_HINTS_BROWSABLE_OTHERS // | RS_FILE_HINTS_NETWORK_WIDE_GROUPS | RS_FILE_HINTS_BROWSABLE_GROUPS ; -/* Callback Codes */ - -const uint32_t RS_FILE_EXTRA_DELETE = 0x0010; - enum class RsSharedDirectoriesEventCode: uint8_t { UNKNOWN = 0x00, STARTING_DIRECTORY_SWEEP = 0x01, // (void) From 198fa7d8bc0178d339e1a7e6034249826df10f94 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 25 Mar 2020 01:31:58 +0100 Subject: [PATCH 14/40] RsFiles::parseFilesLink support old link formats --- libretroshare/src/ft/ftserver.cc | 57 ++++++++++++++++++++++++++ libretroshare/src/retroshare/rsfiles.h | 5 ++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 1efbb715c..9f5c1d1bd 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -2277,6 +2277,63 @@ std::error_condition ftServer::parseFilesLink( const std::string& link, RsFileTree& collection ) { RsUrl tUrl(link); + + { + /* Handle retrocompatibility with old stile single file links + * retroshare://file?name=$FILE_NAME&size=$FILE_SIZE&hash=$FILE_HASH + */ + if( tUrl.scheme() == "retroshare" && tUrl.host() == "file" + && tUrl.hasQueryK("name") && !tUrl.getQueryV("name")->empty() + && tUrl.hasQueryK("size") && !tUrl.getQueryV("size")->empty() + && tUrl.hasQueryK("hash") && !tUrl.getQueryV("hash")->empty() ) + { + DirDetails dt; + dt.type = DIR_TYPE_FILE; + dt.hash = RsFileHash(*tUrl.getQueryV("hash")); + dt.name = *tUrl.getQueryV("name"); + try + { + dt.count = std::stoull(*tUrl.getQueryV("size")); + std::unique_ptr ft; + if( !dt.hash.isNull() && + (ft = RsFileTree::fromDirDetails(dt, true)) ) + { + collection = *ft; + return std::error_condition(); + } + } + catch (...) {} + } + } + + { + /* Handle retrocompatibility with old stile collection links + * retroshare://collection?name=$NAME&size=$SIZE&radix=$RADIX&files=$COUNT + */ + if( tUrl.scheme() == "retroshare" && tUrl.host() == "collection" + && tUrl.hasQueryK("name") && !tUrl.getQueryV("name")->empty() + && tUrl.hasQueryK("size") && !tUrl.getQueryV("size")->empty() + && tUrl.hasQueryK("radix") && !tUrl.getQueryV("radix")->empty() + && tUrl.hasQueryK("files") && !tUrl.getQueryV("files")->empty() ) + { + try + { + // Don't need the values just check they are valid numbers + std::stoull(*tUrl.getQueryV("size")); + std::stoull(*tUrl.getQueryV("files")); + + auto ft = RsFileTree::fromRadix64(*tUrl.getQueryV("radix")); + if(ft) + { + collection = *ft; + return std::error_condition(); + } + } + catch (...) {} + } + } + + // Finaly handle the new files link format rs_view_ptr radixPtr = tUrl.getQueryV(FILES_URL_DATA_FIELD); if(!radixPtr) radixPtr = &link; diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index a8b65fcbe..8a60444c7 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -948,8 +948,9 @@ public: /** * @brief Parse RetroShare files link * @jsonapi{development} - * @param[in] link files link either in radix or URL format - * @param[out] collection optional storage for parsed files link + * Support also old RetroShare-gui file and collections links format. + * @param[in] link files link either in base64 or URL format + * @param[out] collection storage for parsed files link * @return error information if some error occurred, 0/SUCCESS otherwise */ virtual std::error_condition parseFilesLink( From d7f23ce21c5a2e903b4dd9a67af9452817d9ddf9 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 25 Mar 2020 01:57:25 +0100 Subject: [PATCH 15/40] Deprecate unreliable Channels features Fix licensing years --- libretroshare/src/retroshare/rsgxschannels.h | 101 +++++++++++-------- libretroshare/src/util/rsdir.cc | 4 +- libretroshare/src/util/rsdir.h | 10 +- 3 files changed, 65 insertions(+), 50 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 9cc4bd33e..901d4c4ef 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -283,26 +284,6 @@ public: */ virtual bool ExtraFileRemove(const RsFileHash& hash) = 0; - /** - * @brief Get auto-download option value for given channel - * @jsonapi{development} - * @param[in] channelId channel id - * @param[out] enabled storage for the auto-download option value - * @return false if something failed, true otherwhise - */ - virtual bool getChannelAutoDownload( - const RsGxsGroupId& channelId, bool& enabled ) = 0; - - /** - * @brief Get download directory for the given channel - * @jsonapi{development} - * @param[in] channelId id of the channel - * @param[out] directory reference to string where to store the path - * @return false on error, true otherwise - */ - virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId, - std::string& directory ) = 0; - /** * @brief Get channels summaries list. Blocking API. * @jsonapi{development} @@ -369,16 +350,6 @@ public: */ virtual bool markRead(const RsGxsGrpMsgIdPair& postId, bool read) = 0; - /** - * @brief Enable or disable auto-download for given channel. Blocking API - * @jsonapi{development} - * @param[in] channelId channel id - * @param[in] enable true to enable, false to disable - * @return false if something failed, true otherwhise - */ - virtual bool setChannelAutoDownload( - const RsGxsGroupId& channelId, bool enable ) = 0; - /** * @brief Share channel publishing key * This can be used to authorize other peers to post on the channel @@ -390,16 +361,6 @@ public: virtual bool shareChannelKeys( const RsGxsGroupId& channelId, const std::set& peers ) = 0; - /** - * @brief Set download directory for the given channel. Blocking API. - * @jsonapi{development} - * @param[in] channelId id of the channel - * @param[in] directory path - * @return false on error, true otherwise - */ - virtual bool setChannelDownloadDirectory( - const RsGxsGroupId& channelId, const std::string& directory) = 0; - /** * @brief Subscrbe to a channel. Blocking API * @jsonapi{development} @@ -525,8 +486,62 @@ public: std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; - /* Following functions are deprecated as they expose internal functioning - * semantic, instead of a safe to use API */ + /* Following functions are deprecated and should not be considered a safe to + * use API */ + + /** + * @brief Get auto-download option value for given channel + * @jsonapi{development} + * @deprecated This feature rely on very buggy code, the returned value is + * not reliable @see setChannelAutoDownload(). + * @param[in] channelId channel id + * @param[out] enabled storage for the auto-download option value + * @return false if something failed, true otherwhise + */ + RS_DEPRECATED + virtual bool getChannelAutoDownload( + const RsGxsGroupId& channelId, bool& enabled ) = 0; + + /** + * @brief Enable or disable auto-download for given channel. Blocking API + * @jsonapi{development} + * @deprecated This feature rely on very buggy code, when enabled the + * channel service start flooding erratically log with error messages, + * apparently without more dangerous consequences. Still those messages + * hints that something out of control is happening under the hood, use at + * your own risk. A safe alternative to this method can easly implemented + * at API client level instead. + * @param[in] channelId channel id + * @param[in] enable true to enable, false to disable + * @return false if something failed, true otherwhise + */ + RS_DEPRECATED + virtual bool setChannelAutoDownload( + const RsGxsGroupId& channelId, bool enable ) = 0; + + /** + * @brief Get download directory for the given channel + * @jsonapi{development} + * @deprecated @see setChannelAutoDownload() + * @param[in] channelId id of the channel + * @param[out] directory reference to string where to store the path + * @return false on error, true otherwise + */ + RS_DEPRECATED + virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId, + std::string& directory ) = 0; + + /** + * @brief Set download directory for the given channel. Blocking API. + * @jsonapi{development} + * @deprecated @see setChannelAutoDownload() + * @param[in] channelId id of the channel + * @param[in] directory path + * @return false on error, true otherwise + */ + RS_DEPRECATED + virtual bool setChannelDownloadDirectory( + const RsGxsGroupId& channelId, const std::string& directory) = 0; /** * @brief Create channel. Blocking API. diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index be327bb3d..fe9907d2d 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2007 by Robert Fernie * + * Copyright (C) 2004-2007 Robert Fernie * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * diff --git a/libretroshare/src/util/rsdir.h b/libretroshare/src/util/rsdir.h index e8ff09943..50e636541 100644 --- a/libretroshare/src/util/rsdir.h +++ b/libretroshare/src/util/rsdir.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2007 by Robert Fernie * + * Copyright (C) 2004-2007 Robert Fernie * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -20,8 +22,7 @@ * * *******************************************************************************/ -#ifndef RSUTIL_DIRFNS_H -#define RSUTIL_DIRFNS_H +#pragma once #include #include @@ -161,6 +162,3 @@ bool create_directories(const std::string& path); } } #endif // __cplusplus < 201703L - - -#endif From 0187bf2f2f310834d20d919291fcdc59ee72f6b8 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 25 Mar 2020 22:21:05 +0100 Subject: [PATCH 16/40] Suppress eccessive VLQ debugging --- libretroshare/src/serialiser/rstypeserializer.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index ce53dce26..adf885c31 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -966,8 +966,8 @@ protected: #pragma GCC diagnostic ignored "-Wbool-compare" while(member > 127) { ++ret; member >>= 7; } #pragma GCC diagnostic pop - RsDbg() << __PRETTY_FUNCTION__ << " memberBackup: " << memberBackup - << " return: " << ret << std::endl; + Dbg2() << __PRETTY_FUNCTION__ << " memberBackup: " << memberBackup + << " return: " << ret << std::endl; return ret; } @@ -1007,6 +1007,7 @@ protected: << " member " << member << " size: " << size << " offset: " << offset + << " backupMember: " << backupMember << std::error_condition(std::errc::no_buffer_space) << std::endl; print_stacktrace(); @@ -1015,12 +1016,12 @@ protected: data[offset++] = static_cast(member & 127); - RsDbg() << __PRETTY_FUNCTION__ << " backupMember: " << backupMember - << " offsetBackup: " << offsetBackup << " offeset: " << offset - << " serialized as: "; + Dbg3() << __PRETTY_FUNCTION__ << " backupMember: " << backupMember + << " offsetBackup: " << offsetBackup << " offeset: " << offset + << " serialized as: "; for(; offsetBackup < offset; ++offsetBackup) - std::cerr << " " << std::bitset<8>(data[offsetBackup]); - std::cerr << std::endl; + Dbg3().uStream() << " " << std::bitset<8>(data[offsetBackup]); + Dbg3().uStream() << std::endl; return ok; } From 0d1524aa12c30ed96dd218132fd7681202a58bf2 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 31 Mar 2020 23:20:08 +0200 Subject: [PATCH 17/40] Fix crash due to RsChatAvatarItem uninitialized pointer Caught signal 11 (SIGSEGV) stack trace: retroshare( 0xbbd2da)[0x558ccf46a2da] retroshare(CrashStackTrace::abortHandler(int) 0xeb)[0x558ccf471c5c] /lib64/libc.so.6( 0x38f80)[0x7f6ef6facf80] /lib64/libc.so.6(realloc 0x54)[0x7f6ef6fffef4] retroshare(RsTypeSerializer::RawMemoryWrapper::serial_process(RsGenericSerializer::SerializeJob, RsGenericSerializer::SerializeContext&) 0x5d4)[0x558ccfcddad0] retroshare(std::enable_if::value, void>::type RsTypeSerializer::serial_process(RsGenericSerializer::SerializeJob, RsGenericSerializer::SerializeContext&, RsTypeSerializer::RawMemoryWrapper&, std::__cxx11::basic_string, std::allocator > const&) 0x9d)[0x558ccfb3da66] retroshare(RsChatAvatarItem::serial_process(RsGenericSerializer::SerializeJob, RsGenericSerializer::SerializeContext&) 0x84)[0x558ccff90ba6] retroshare(RsServiceSerializer::deserialise(void*, unsigned int*) 0x403)[0x558ccfcd96e9] retroshare(RsSerialiser::deserialise(void*, unsigned int*) 0x38e)[0x558ccfb5a1a8] retroshare(pqiSSLstore::readPkt(RsItem**) 0x3f9)[0x558ccfad329f] retroshare(pqiSSLstore::GetItem() 0xbb)[0x558ccfad2d9f] retroshare(pqiSSLstore::getEncryptedItems(std::__cxx11::list >&) 0x3e)[0x558ccfad2c26] retroshare(p3Config::loadAttempt(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::list >&) 0xf8)[0x558ccfab20d0] retroshare(p3Config::loadConfig() 0xeb)[0x558ccfab1d69] retroshare(p3Config::loadConfiguration(t_RsGenericIdType<20u, false, (RsGenericIdType)2>&) 0x1c)[0x558ccfab1c7c] retroshare(p3ConfigMgr::loadConfig() 0x97)[0x558ccfab17f3] retroshare(p3ConfigMgr::loadConfiguration() 0x18)[0x558ccfab1758] retroshare(RsServer::StartupRetroShare() 0x4407)[0x558ccfb02fab] retroshare(main 0x17e2)[0x558ccf46d291] /lib64/libc.so.6(__libc_start_main 0xeb)[0x7f6ef6f97e9b] retroshare(_start 0x2a)[0x558ccf438c9a] D 1585687209 void RetroDb::closeDb() sqlite3_close return: 0 D 1585687209 void RetroDb::closeDb() sqlite3_close return: 0 Memory still in use at end of program: 24776 bytes. --- libretroshare/src/chat/rschatitems.cc | 9 --------- libretroshare/src/chat/rschatitems.h | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/libretroshare/src/chat/rschatitems.cc b/libretroshare/src/chat/rschatitems.cc index abe730d09..13d272c4e 100644 --- a/libretroshare/src/chat/rschatitems.cc +++ b/libretroshare/src/chat/rschatitems.cc @@ -70,15 +70,6 @@ void RsChatMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGeneric /*************************************************************************/ -RsChatAvatarItem::~RsChatAvatarItem() -{ - if(image_data != NULL) - { - free(image_data) ; - image_data = NULL ; - } -} - void RsChatLobbyBouncingObject::serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,lobby_id,"lobby_id") ; diff --git a/libretroshare/src/chat/rschatitems.h b/libretroshare/src/chat/rschatitems.h index 5bcfa1e0a..aca3a411a 100644 --- a/libretroshare/src/chat/rschatitems.h +++ b/libretroshare/src/chat/rschatitems.h @@ -346,14 +346,20 @@ class RsChatStatusItem: public RsChatItem // class RsChatAvatarItem: public RsChatItem { - public: - RsChatAvatarItem() :RsChatItem(RS_PKT_SUBTYPE_CHAT_AVATAR) {setPriorityLevel(QOS_PRIORITY_RS_CHAT_AVATAR_ITEM) ;} +public: + RsChatAvatarItem(): + RsChatItem(RS_PKT_SUBTYPE_CHAT_AVATAR), + image_size(0), image_data(nullptr) + { setPriorityLevel(QOS_PRIORITY_RS_CHAT_AVATAR_ITEM); } - virtual ~RsChatAvatarItem() ; - void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); + ~RsChatAvatarItem() override { free(image_data); } - uint32_t image_size ; // size of data in bytes - unsigned char *image_data ; // image + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx) override; + + uint32_t image_size; /// size of data in bytes + unsigned char* image_data ; /// image data }; From 9d325e3d177738efb976b9dc10bbe12a5755479e Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 31 Mar 2020 23:21:09 +0200 Subject: [PATCH 18/40] Fix mail in rsjsonapi.h copyright header --- libretroshare/src/retroshare/rsjsonapi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/retroshare/rsjsonapi.h b/libretroshare/src/retroshare/rsjsonapi.h index 98b052d58..147011787 100644 --- a/libretroshare/src/retroshare/rsjsonapi.h +++ b/libretroshare/src/retroshare/rsjsonapi.h @@ -1,7 +1,7 @@ /* * RetroShare JSON API public header * - * Copyright (C) 2018-2020 Gioacchino Mazzurco + * Copyright (C) 2018-2020 Gioacchino Mazzurco * Copyright (C) 2019 Cyril Soler * Copyright (C) 2020 Asociación Civil Altermundi * From b42c0410f1f5aa6ea8b86fb206d2ea6c96f2c344 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 31 Mar 2020 23:21:59 +0200 Subject: [PATCH 19/40] Expose pgpIdFromFingerprint to JSON API + missing explicit in one paramether costructor --- libretroshare/src/retroshare/rspeers.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/retroshare/rspeers.h b/libretroshare/src/retroshare/rspeers.h index 5cac456f2..124dba357 100644 --- a/libretroshare/src/retroshare/rspeers.h +++ b/libretroshare/src/retroshare/rspeers.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2008 by Robert Fernie * + * Copyright (C) 2004-2008 by Robert Fernie * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -468,7 +470,7 @@ struct RsGroupInfo : RsSerializable struct RsPeerStateChangedEvent : RsEvent { /// @param[in] sslId is of the peer which changed state - RsPeerStateChangedEvent(RsPeerId sslId); + explicit RsPeerStateChangedEvent(RsPeerId sslId); /// Storage fot the id of the peer that changed state RsPeerId mSslId; @@ -605,7 +607,15 @@ public: virtual bool getAssociatedSSLIds(const RsPgpId& gpg_id, std::list& ids) = 0; virtual bool gpgSignData(const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen, std::string reason = "") = 0; - virtual RsPgpId pgpIdFromFingerprint(const RsPgpFingerprint& fpr) = 0; + /** + * @brief Convert PGP fingerprint to PGP id + * @jsonapi{development} + * Helper method useful while we port the whole RetroShare codebase from + * RsPgpId to RsPgpFingerprint + * @param[in] fpr PGP fingerprint to convert + * @return PGP id corresponding to the fingerprint + */ + virtual RsPgpId pgpIdFromFingerprint(const RsPgpFingerprint& fpr) = 0; // Note: the two methods below could be unified. The fact that one of them can take an optional RsPeerDetails struct as parameter // seems quite inconsistent. From ce5f5faa972a3b16e07fe437984125093fe7a318 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 1 Apr 2020 01:30:57 +0200 Subject: [PATCH 20/40] Fix Android compilation after rebase on master --- .../src/retroshare/rsgxsifacehelper.h | 57 +++++++++-------- .../src/retroshare/rsgxsifacetypes.h | 63 +++++++------------ libretroshare/src/retroshare/rsidentity.h | 2 +- 3 files changed, 56 insertions(+), 66 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxsifacehelper.h b/libretroshare/src/retroshare/rsgxsifacehelper.h index e2d04f887..1ea2ad061 100644 --- a/libretroshare/src/retroshare/rsgxsifacehelper.h +++ b/libretroshare/src/retroshare/rsgxsifacehelper.h @@ -30,6 +30,7 @@ #include "retroshare/rsreputations.h" #include "rsgxsflags.h" #include "util/rsdeprecate.h" +#include "util/rsdebug.h" /*! * This class only make method of internal members visible tu upper level to @@ -46,7 +47,7 @@ enum class TokenRequestType: uint8_t { - UNDEFINED = 0x00, + __NONE = 0x00, /// Used to detect uninitialized GROUP_DATA = 0x01, GROUP_META = 0x02, POSTS = 0x03, @@ -55,6 +56,7 @@ enum class TokenRequestType: uint8_t GROUP_STATISTICS = 0x06, SERVICE_STATISTICS = 0x07, NO_KILL_TYPE = 0x08, + __MAX /// Used to detect out of range }; class RsGxsIfaceHelper @@ -444,9 +446,9 @@ protected: uint32_t token, std::chrono::milliseconds maxWait = std::chrono::milliseconds(20000), std::chrono::milliseconds checkEvery = std::chrono::milliseconds(100), - bool auto_delete_if_unsuccessful=true) + bool auto_delete_if_unsuccessful=true) { - #if defined(__ANDROID__) && (__ANDROID_API__ < 24) +#if defined(__ANDROID__) && (__ANDROID_API__ < 24) auto wkStartime = std::chrono::steady_clock::now(); int maxWorkAroundCnt = 10; LLwaitTokenBeginLabel: @@ -454,13 +456,14 @@ LLwaitTokenBeginLabel: auto timeout = std::chrono::steady_clock::now() + maxWait; auto st = requestStatus(token); - while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE) && std::chrono::steady_clock::now() < timeout ) + while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE) + && std::chrono::steady_clock::now() < timeout ) { std::this_thread::sleep_for(checkEvery); st = requestStatus(token); } - if(st != RsTokenService::COMPLETE && auto_delete_if_unsuccessful) - cancelRequest(token); + if(st != RsTokenService::COMPLETE && auto_delete_if_unsuccessful) + cancelRequest(token); #if defined(__ANDROID__) && (__ANDROID_API__ < 24) /* Work around for very slow/old android devices, we don't expect this @@ -487,35 +490,39 @@ LLwaitTokenBeginLabel: #endif { - RS_STACK_MUTEX(mMtx); + RS_STACK_MUTEX(mMtx); mActiveTokens.erase(token); - } + } - return st; - } + return st; + } private: RsGxsIface& mGxs; RsTokenService& mTokenService; - RsMutex mMtx; + RsMutex mMtx; - std::map mActiveTokens; + std::map mActiveTokens; - void locked_dumpTokens() - { - uint16_t service_id = mGxs.serviceType(); + void locked_dumpTokens() + { + const uint16_t service_id = mGxs.serviceType(); + const auto countSize = static_cast(TokenRequestType::__MAX); + uint32_t count[countSize] = {0}; - uint32_t count[7] = {0}; + RsDbg() << __PRETTY_FUNCTION__ << "Service 0x" << std::hex << service_id + << " (" << rsServiceControl->getServiceName( + RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(service_id) ) + << ") this=0x" << static_cast(this) + << ") Active tokens (per type): "; - RsDbg() << "Service " << std::hex << service_id << std::dec - << " (" << rsServiceControl->getServiceName(RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(service_id)) - << ") this=" << std::hex << (void*)this << std::dec << ") Active tokens (per type): " ; + // let's count how many token of each type we've got. + for(auto& it: mActiveTokens) ++count[static_cast(it.second)]; - for(auto& it: mActiveTokens) // let's count how many token of each type we've got. - ++count[static_cast(it.second)]; + for(uint32_t i=0; i < countSize; ++i) + RsDbg().uStream() /* << i << ":" */ << count[i] << " "; + RsDbg().uStream() << std::endl; + } - for(uint32_t i=0;i<7;++i) - std::cerr << std::dec /* << i << ":" */ << count[i] << " "; - std::cerr << std::endl; - } + RS_SET_CONTEXT_DEBUG_LEVEL(1) }; diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index 90715dfcb..13230156a 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -191,37 +191,35 @@ struct RsMsgMetaData : RsSerializable } }; -struct RsGxsGenericMsgData +struct GxsGroupStatistic : RsSerializable { - virtual ~RsGxsGenericMsgData() = default; // making the type polymorphic + GxsGroupStatistic() : + mNumMsgs(0), mTotalSizeOfMsgs(0), mNumThreadMsgsNew(0), + mNumThreadMsgsUnread(0), mNumChildMsgsNew(0), mNumChildMsgsUnread(0) {} - RsMsgMetaData mMeta; -}; - - -class GxsGroupStatistic -{ -public: - GxsGroupStatistic() + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx) override { - mNumMsgs = 0; - mTotalSizeOfMsgs = 0; - mNumThreadMsgsNew = 0; - mNumThreadMsgsUnread = 0; - mNumChildMsgsNew = 0; - mNumChildMsgsUnread = 0; + RS_SERIAL_PROCESS(mGrpId); + RS_SERIAL_PROCESS(mNumMsgs); + RS_SERIAL_PROCESS(mTotalSizeOfMsgs); + RS_SERIAL_PROCESS(mNumThreadMsgsNew); + RS_SERIAL_PROCESS(mNumThreadMsgsUnread); + RS_SERIAL_PROCESS(mNumChildMsgsNew); + RS_SERIAL_PROCESS(mNumChildMsgsUnread); + } -public: - /// number of message - RsGxsGroupId mGrpId; - - uint32_t mNumMsgs; // from the database + RsGxsGroupId mGrpId; + uint32_t mNumMsgs; /// number of message, from the database uint32_t mTotalSizeOfMsgs; uint32_t mNumThreadMsgsNew; uint32_t mNumThreadMsgsUnread; uint32_t mNumChildMsgsNew; - uint32_t mNumChildMsgsUnread; + uint32_t mNumChildMsgsUnread; + + ~GxsGroupStatistic() override = default; }; class GxsServiceStatistic @@ -254,30 +252,15 @@ public: uint32_t mSizeStore; }; -class UpdateItem -{ -public: - virtual ~UpdateItem() { } -}; - -class StringUpdateItem : public UpdateItem -{ -public: - StringUpdateItem(const std::string update) : mUpdate(update) {} - const std::string& getUpdate() const { return mUpdate; } - -private: - std::string mUpdate; -}; - -class RsGxsGroupUpdateMeta +class RS_DEPRECATED RsGxsGroupUpdateMeta { public: // expand as support is added for other utypes enum UpdateType { DESCRIPTION, NAME }; - RsGxsGroupUpdateMeta(const RsGxsGroupId& groupId) : mGroupId(groupId) {} + explicit RsGxsGroupUpdateMeta(const RsGxsGroupId& groupId): + mGroupId(groupId) {} typedef std::map GxsMetaUpdate; diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index afd8f5544..d1345461e 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -37,7 +37,7 @@ #include "serialiser/rstypeserializer.h" #include "util/rsdeprecate.h" -struct RsIdentity; +class RsIdentity; /** * Pointer to global instance of RsIdentity service implementation From 4c0baa1ec3611099016a394dc6e033f2b8b64bff Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 1 Apr 2020 19:34:49 +0200 Subject: [PATCH 21/40] Fix JSON API event handler registering behavior As a bonus the behaviour is now homogeneous between C++ API and JSON API Fix a bunch of compiler warning RsEvents implementation is now safer mHandlerMaps size is known at compile time, so use an std::array instead of vector --- .../src/gossipdiscovery/p3gossipdiscovery.cc | 13 --- libretroshare/src/jsonapi/jsonapi.cpp | 5 +- libretroshare/src/retroshare/rsevents.h | 22 ++--- libretroshare/src/services/rseventsservice.cc | 81 ++++++++++--------- libretroshare/src/services/rseventsservice.h | 20 ++--- .../src/gui/FileTransfer/TransfersDialog.cpp | 40 +++++---- retroshare-gui/src/gui/Identity/IdDialog.cpp | 14 +++- retroshare-gui/src/gui/NewsFeed.cpp | 17 ++-- .../src/gui/Posted/PostedDialog.cpp | 13 +-- .../src/gui/Posted/PostedListWidget.cpp | 8 +- .../src/gui/common/NewFriendList.cpp | 6 +- .../src/gui/gxschannels/GxsChannelDialog.cpp | 12 +-- .../gui/gxschannels/GxsChannelPostsWidget.cpp | 8 +- .../gui/gxsforums/GxsForumThreadWidget.cpp | 10 ++- .../src/gui/gxsforums/GxsForumsDialog.cpp | 11 +-- .../src/gui/statusbar/hashingstatus.cpp | 6 +- .../src/util/RsGxsUpdateBroadcast.cpp | 18 ++--- 17 files changed, 165 insertions(+), 139 deletions(-) diff --git a/libretroshare/src/gossipdiscovery/p3gossipdiscovery.cc b/libretroshare/src/gossipdiscovery/p3gossipdiscovery.cc index 81dd097e0..3b20527e1 100644 --- a/libretroshare/src/gossipdiscovery/p3gossipdiscovery.cc +++ b/libretroshare/src/gossipdiscovery/p3gossipdiscovery.cc @@ -108,14 +108,6 @@ p3discovery2::p3discovery2( // Add self into PGP FriendList. mFriendList[AuthGPG::getAuthGPG()->getGPGOwnId()] = DiscPgpInfo(); - - if(rsEvents) - rsEvents->registerEventsHandler( - RsEventType::GOSSIP_DISCOVERY, - [this](std::shared_ptr event) - { - rsEventsHandler(*event); - }, mRsEventsHandle ); // mRsEventsHandle is zeroed in initializer list } @@ -1280,11 +1272,6 @@ bool p3discovery2::setPeerVersion(const RsPeerId &peerId, const std::string &ver return true; } -void p3discovery2::rsEventsHandler(const RsEvent& event) -{ - Dbg3() << __PRETTY_FUNCTION__ << " " << static_cast(event.mType) << std::endl; -} - /*************************************************************************************/ /* AuthGPGService */ diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index 993165fdb..a24d73c4b 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -352,7 +352,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), rsEvents, "rsEvents", cAns, session ) ) return; - RsEventType eventType = RsEventType::NONE; + RsEventType eventType = RsEventType::__NONE; // deserialize input parameters from JSON { @@ -395,7 +395,8 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), } ); }; - bool retval = rsEvents->registerEventsHandler(eventType,multiCallback, hId); + bool retval = rsEvents->registerEventsHandler( + multiCallback, hId, eventType ); { RsGenericSerializer::SerializeContext& ctx(cAns); diff --git a/libretroshare/src/retroshare/rsevents.h b/libretroshare/src/retroshare/rsevents.h index cd23cacd9..c960a16a2 100644 --- a/libretroshare/src/retroshare/rsevents.h +++ b/libretroshare/src/retroshare/rsevents.h @@ -50,7 +50,7 @@ extern RsEvents* rsEvents; */ enum class RsEventType : uint32_t { - NONE = 0, /// Used to detect uninitialized event + __NONE = 0, /// Used internally to detect invalid event type passed /// @see RsBroadcastDiscovery BROADCAST_DISCOVERY = 1, @@ -64,7 +64,7 @@ enum class RsEventType : uint32_t /// @see pqissl PEER_CONNECTION = 4, - /// @see RsGxsChanges // this one is used in RsGxsBroadcast + /// @see RsGxsChanges, used also in @see RsGxsBroadcast GXS_CHANGES = 5, /// Emitted when a peer state changes, @see RsPeers @@ -95,9 +95,9 @@ enum class RsEventType : uint32_t FILE_TRANSFER = 14, /// @see RsMsgs - CHAT_MESSAGE = 15, + CHAT_MESSAGE = 15, - MAX /// Used to detect invalid event type passed + __MAX /// Used internally to detect invalid event type passed }; /** @@ -107,7 +107,7 @@ enum class RsEventType : uint32_t struct RsEvent : RsSerializable { protected: - RsEvent(RsEventType type) : + explicit RsEvent(RsEventType type) : mType(type), mTimePoint(std::chrono::system_clock::now()) {} RsEvent() = delete; @@ -174,8 +174,9 @@ public: * @brief Register events handler * Every time an event is dispatced the registered events handlers will get * their method handleEvent called with the event passed as paramether. + * @attention Callbacks must not fiddle internally with methods of this + * class otherwise a deadlock will happen. * @jsonapi{development,manualwrapper} - * @param eventType Type of event for which the callback is called * @param multiCallback Function that will be called each time an event * is dispatched. * @param[inout] hId Optional storage for handler id, useful to @@ -183,13 +184,15 @@ public: * value may be provided to the function call but * must habe been generated with * @see generateUniqueHandlerId() + * @param eventType Optional type of event for which the callback is + * called, if NONE is passed multiCallback is + * called for every events without filtering. * @return False on error, true otherwise. */ virtual bool registerEventsHandler( - RsEventType eventType, std::function)> multiCallback, - RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0) - ) = 0; + RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0), + RsEventType eventType = RsEventType::__NONE ) = 0; /** * @brief Unregister event handler @@ -200,4 +203,3 @@ public: virtual ~RsEvents(); }; - diff --git a/libretroshare/src/services/rseventsservice.cc b/libretroshare/src/services/rseventsservice.cc index c1aa0958c..f0db8bc3f 100644 --- a/libretroshare/src/services/rseventsservice.cc +++ b/libretroshare/src/services/rseventsservice.cc @@ -39,7 +39,7 @@ bool isEventValid( return false; } - if(event->mType <= RsEventType::NONE) + if(event->mType <= RsEventType::__NONE) { errorMessage = "Event has type NONE: " + std::to_string( @@ -48,9 +48,9 @@ bool isEventValid( return false; } - if(event->mType >= RsEventType::MAX) + if(event->mType >= RsEventType::__MAX) { - errorMessage = "Event has type >= RsEventType::MAX: " + + errorMessage = "Event has type >= RsEventType::__MAX: " + std::to_string( static_cast::type >( event->mType ) ); @@ -100,25 +100,30 @@ RsEventsHandlerId_t RsEventsService::generateUniqueHandlerId_unlocked() } bool RsEventsService::registerEventsHandler( - RsEventType eventType, std::function)> multiCallback, - RsEventsHandlerId_t& hId ) + RsEventsHandlerId_t& hId, RsEventType eventType ) { RS_STACK_MUTEX(mHandlerMapMtx); - if( (int)eventType > mHandlerMaps.size() + 10) - { - RsErr() << "Cannot register an event handler for an event type larger than 10 plus the max pre-defined event (value passed was " << (int)eventType << " whereas max is " << (int)RsEventType::MAX << ")" << std::endl; - return false; - } + if( eventType >= RsEventType::__MAX) + { + RsErr() << __PRETTY_FUNCTION__ << " Invalid event type: " + << static_cast(eventType) << " >= RsEventType::__MAX:" + << static_cast(RsEventType::__MAX) << std::endl; + print_stacktrace(); + return false; + } - if( (int)eventType >= mHandlerMaps.size()) - mHandlerMaps.resize( (int)eventType +1 ); + if(!hId) hId = generateUniqueHandlerId_unlocked(); + else if (hId > mLastHandlerId) + { + RsErr() << __PRETTY_FUNCTION__ << " Invalid handler id: " << hId + << " how did you generate it? " << std::endl; + print_stacktrace(); + return false; + } - if(!hId) - hId = generateUniqueHandlerId_unlocked(); - - mHandlerMaps[(int)eventType][hId] = multiCallback; + mHandlerMaps[static_cast(eventType)][hId] = multiCallback; return true; } @@ -126,14 +131,14 @@ bool RsEventsService::unregisterEventsHandler(RsEventsHandlerId_t hId) { RS_STACK_MUTEX(mHandlerMapMtx); - for(uint32_t i=0;i event) { - std::function)> mCallback; + uint32_t event_type_index = static_cast(event->mType); - uint32_t event_type_index = static_cast(event->mType); + if(RsEventType::__NONE >= event->mType || event->mType >= RsEventType::__MAX ) + { + RsErr() << __PRETTY_FUNCTION__ << " Invalid event type: " + << event_type_index << std::endl; + print_stacktrace(); + return; + } { - RS_STACK_MUTEX(mHandlerMapMtx); /* LOCKED AREA */ + RS_STACK_MUTEX(mHandlerMapMtx); + /* It is important to also call the callback under mutex protection to + * ensure they are not unregistered in the meanwhile. + * If a callback try to fiddle with registering/unregistering it will + * deadlock */ - if(event_type_index >= mHandlerMaps.size() || event_type_index < 1) - { - RsErr() << "Cannot handle an event of type " << event_type_index << ": out of scope!" << std::endl; - return; - } + // Call all clients that registered a callback for this event type + for(auto cbit: mHandlerMaps[event_type_index]) cbit.second(event); - // Call all clients that registered a callback for this event type - - for(auto cbit: mHandlerMaps[event_type_index]) - cbit.second(event); - - // Also call all clients that registered with NONE, meaning that they expect all events - - for(auto cbit: mHandlerMaps[static_cast(RsEventType::NONE)]) - cbit.second(event); + /* Also call all clients that registered with NONE, meaning that they + * expect all events */ + for(auto cbit: mHandlerMaps[static_cast(RsEventType::__NONE)]) + cbit.second(event); } } diff --git a/libretroshare/src/services/rseventsservice.h b/libretroshare/src/services/rseventsservice.h index 805a173a0..3a61cdd79 100644 --- a/libretroshare/src/services/rseventsservice.h +++ b/libretroshare/src/services/rseventsservice.h @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2019 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -24,6 +25,7 @@ #include #include #include +#include #include "retroshare/rsevents.h" #include "util/rsthreads.h" @@ -35,7 +37,6 @@ class RsEventsService : public: RsEventsService(): mHandlerMapMtx("RsEventsService::mHandlerMapMtx"), mLastHandlerId(1), - mHandlerMaps(static_cast(RsEventType::MAX)), mEventQueueMtx("RsEventsService::mEventQueueMtx") {} /// @see RsEvents @@ -55,10 +56,9 @@ public: /// @see RsEvents bool registerEventsHandler( - RsEventType eventType, std::function)> multiCallback, - RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0) - ) override; + RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0), + RsEventType eventType = RsEventType::__NONE ) override; /// @see RsEvents bool unregisterEventsHandler(RsEventsHandlerId_t hId) override; @@ -67,10 +67,12 @@ protected: RsMutex mHandlerMapMtx; RsEventsHandlerId_t mLastHandlerId; - std::vector< - std::map< - RsEventsHandlerId_t, - std::function)> > > mHandlerMaps; + std::array< + std::map< + RsEventsHandlerId_t, + std::function)> >, + static_cast(RsEventType::__MAX) + > mHandlerMaps; RsMutex mEventQueueMtx; std::deque< std::shared_ptr > mEventQueue; diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index 3c00f2122..ab5a6d9fd 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -1076,30 +1076,36 @@ TransfersDialog::TransfersDialog(QWidget *parent) // load settings processSettings(true); - int S = QFontMetricsF(font()).height(); - QString help_str = tr( - "

  File Transfer

\ -

Retroshare brings two ways of transferring files: direct transfers from your friends, and \ - distant anonymous tunnelled transfers. In addition, file transfer is multi-source and allows swarming \ - (you can be a source while downloading)

\ -

You can share files using the icon from the left side bar. \ - These files will be listed in the My Files tab. You can decide for each friend group whether they can or not see these files \ - in their Friends Files tab

\ -

The search tab reports files from your friends' file lists, and distant files that can be reached \ - anonymously using the multi-hop tunnelling system.

\ - ").arg(QString::number(2*S)).arg(QString::number(S)) ; + int S = static_cast(QFontMetricsF(font()).height()); + QString help_str = tr( + "

  " + "File Transfer

" + "

Retroshare brings two ways of transferring files: direct " + "transfers from your friends, and distant anonymous tunnelled " + "transfers. In addition, file transfer is multi-source and " + "allows swarming (you can be a source while downloading)

" + "

You can share files using the " + "" + " icon from the left side bar. These files will be listed in " + "the My Files tab. You can decide for each friend group whether" + " they can or not see these files in their Friends Files tab

" + "

The search tab reports files from your friends' file lists," + " and distant files that can be reached anonymously using the " + "multi-hop tunnelling system.

") + .arg(QString::number(2*S)).arg(QString::number(S)) ; - registerHelpButton(ui.helpButton,help_str,"TransfersDialog") ; + registerHelpButton(ui.helpButton,help_str,"TransfersDialog") ; - mEventHandlerId=0; - rsEvents->registerEventsHandler(RsEventType::FILE_TRANSFER, [this](std::shared_ptr event) { handleEvent(event); }, mEventHandlerId ); + mEventHandlerId=0; + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) { handleEvent(event); }, + mEventHandlerId, RsEventType::FILE_TRANSFER ); } void TransfersDialog::handleEvent(std::shared_ptr event) { - if(event->mType != RsEventType::FILE_TRANSFER) - return; + if(event->mType != RsEventType::FILE_TRANSFER) return; const RsFileTransferEvent *fe = dynamic_cast(event.get()); if(!fe) diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index d02de1742..d646227f8 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -149,11 +149,17 @@ IdDialog::IdDialog(QWidget *parent) : MainPage(parent), ui(new Ui::IdDialog) { ui->setupUi(this); - mEventHandlerId_identity = 0; - rsEvents->registerEventsHandler(RsEventType::GXS_IDENTITY, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId_identity ); + mEventHandlerId_identity = 0; + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject([=](){ handleEvent_main_thread(event); }, this); }, + mEventHandlerId_identity, RsEventType::GXS_IDENTITY ); - mEventHandlerId_circles = 0; - rsEvents->registerEventsHandler(RsEventType::GXS_CIRCLES, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId_circles ); + mEventHandlerId_circles = 0; + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject([=](){ handleEvent_main_thread(event); }, this); }, + mEventHandlerId_circles, RsEventType::GXS_CIRCLES ); // This is used to grab the broadcast of changes from p3GxsCircles, which is discarded by the current dialog, since it expects data for p3Identity only. //mCirclesBroadcastBase = new RsGxsUpdateBroadcastBase(rsGxsCircles, this); diff --git a/retroshare-gui/src/gui/NewsFeed.cpp b/retroshare-gui/src/gui/NewsFeed.cpp index 111bc1fe8..9b85a32d1 100644 --- a/retroshare-gui/src/gui/NewsFeed.cpp +++ b/retroshare-gui/src/gui/NewsFeed.cpp @@ -67,12 +67,11 @@ * #define NEWS_DEBUG 1 ****/ -static NewsFeed *instance = NULL; +static NewsFeed* instance = nullptr; /** Constructor */ -NewsFeed::NewsFeed(QWidget *parent) : MainPage(parent), ui(new Ui::NewsFeed) -{ - mEventTypes = { +NewsFeed::NewsFeed(QWidget *parent) : MainPage(parent), ui(new Ui::NewsFeed), + mEventTypes({ RsEventType::AUTHSSL_CONNECTION_AUTENTICATION, RsEventType::PEER_CONNECTION , RsEventType::GXS_CIRCLES , @@ -80,12 +79,14 @@ NewsFeed::NewsFeed(QWidget *parent) : MainPage(parent), ui(new Ui::NewsFeed) RsEventType::GXS_FORUMS , RsEventType::GXS_POSTED , RsEventType::MAIL_STATUS - }; - - for(uint32_t i=0;iregisterEventsHandler(mEventTypes[i], [this](std::shared_ptr event) { handleEvent(event); }, mEventHandlerIds.back() ); + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) { handleEvent(event); }, + mEventHandlerIds.back(), mEventTypes[i] ); } /* Invoke the Qt Designer generated object setup routine */ diff --git a/retroshare-gui/src/gui/Posted/PostedDialog.cpp b/retroshare-gui/src/gui/Posted/PostedDialog.cpp index a6fdda0bd..6c2772276 100644 --- a/retroshare-gui/src/gui/Posted/PostedDialog.cpp +++ b/retroshare-gui/src/gui/Posted/PostedDialog.cpp @@ -41,13 +41,14 @@ public: }; /** Constructor */ -PostedDialog::PostedDialog(QWidget *parent) - : GxsGroupFrameDialog(rsPosted, parent) +PostedDialog::PostedDialog(QWidget *parent): + GxsGroupFrameDialog(rsPosted, parent), mEventHandlerId(0) { - mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - - rsEvents->registerEventsHandler(RsEventType::GXS_POSTED, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + // Needs to be asynced because this function is likely to be called by another thread! + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_POSTED ); } void PostedDialog::handleEvent_main_thread(std::shared_ptr event) diff --git a/retroshare-gui/src/gui/Posted/PostedListWidget.cpp b/retroshare-gui/src/gui/Posted/PostedListWidget.cpp index babf214e3..6c6e04158 100644 --- a/retroshare-gui/src/gui/Posted/PostedListWidget.cpp +++ b/retroshare-gui/src/gui/Posted/PostedListWidget.cpp @@ -107,9 +107,11 @@ PostedListWidget::PostedListWidget(const RsGxsGroupId &postedId, QWidget *parent processSettings(true); mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - - rsEvents->registerEventsHandler(RsEventType::GXS_POSTED, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + // Needs to be asynced because this function is called by another thread! + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_POSTED ); /* Initialize GUI */ setGroupId(postedId); diff --git a/retroshare-gui/src/gui/common/NewFriendList.cpp b/retroshare-gui/src/gui/common/NewFriendList.cpp index 428479d84..131281285 100644 --- a/retroshare-gui/src/gui/common/NewFriendList.cpp +++ b/retroshare-gui/src/gui/common/NewFriendList.cpp @@ -177,8 +177,10 @@ NewFriendList::NewFriendList(QWidget *parent) : /* RsAutoUpdatePage(5000,parent) ui->filterLineEdit->setPlaceholderText(tr("Search")) ; ui->filterLineEdit->showFilterIcon(); - mEventHandlerId=0; // forces initialization - rsEvents->registerEventsHandler( RsEventType::PEER_CONNECTION, [this](std::shared_ptr e) { handleEvent(e); }, mEventHandlerId ); + mEventHandlerId=0; // forces initialization + rsEvents->registerEventsHandler( + [this](std::shared_ptr e) { handleEvent(e); }, + mEventHandlerId, RsEventType::PEER_CONNECTION ); mModel = new RsFriendListModel(); mProxyModel = new FriendListSortFilterProxyModel(ui->peerTreeWidget->header(),this); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 70f3de04a..8e25e4ea0 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -47,12 +47,14 @@ // }; /** Constructor */ -GxsChannelDialog::GxsChannelDialog(QWidget *parent) - : GxsGroupFrameDialog(rsGxsChannels, parent,true) +GxsChannelDialog::GxsChannelDialog(QWidget *parent): + GxsGroupFrameDialog(rsGxsChannels, parent, true), mEventHandlerId(0) { - mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - rsEvents->registerEventsHandler(RsEventType::GXS_CHANNELS, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + // Needs to be asynced because this function is called by another thread! + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject([=]() { handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_CHANNELS ); } void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr event) diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp index 2421756b8..8ca863954 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp @@ -130,9 +130,11 @@ GxsChannelPostsWidget::GxsChannelPostsWidget(const RsGxsGroupId &channelId, QWid setGroupId(channelId); mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - - rsEvents->registerEventsHandler(RsEventType::GXS_CHANNELS, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + // Needs to be asynced because this function is called by another thread! + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject([=](){ handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_CHANNELS ); } void GxsChannelPostsWidget::handleEvent_main_thread(std::shared_ptr event) diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 37b479e6a..b3e98f7b1 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -345,10 +345,12 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget ui->threadTreeWidget->enableColumnCustomize(true); #endif - mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - - rsEvents->registerEventsHandler(RsEventType::GXS_FORUMS, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + mEventHandlerId = 0; + // Needs to be asynced because this function is called by another thread! + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject([=](){ handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_FORUMS ); } void GxsForumThreadWidget::handleEvent_main_thread(std::shared_ptr event) diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumsDialog.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumsDialog.cpp index 1b0a36c6f..b0dcc0402 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumsDialog.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumsDialog.cpp @@ -38,14 +38,15 @@ public: }; /** Constructor */ -GxsForumsDialog::GxsForumsDialog(QWidget *parent) - : GxsGroupFrameDialog(rsGxsForums, parent) +GxsForumsDialog::GxsForumsDialog(QWidget *parent) : + GxsGroupFrameDialog(rsGxsForums, parent), mEventHandlerId(0) { mCountChildMsgs = true; - mEventHandlerId = 0; - // Needs to be asynced because this function is likely to be called by another thread! - rsEvents->registerEventsHandler(RsEventType::GXS_FORUMS, [this](std::shared_ptr event) { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, mEventHandlerId ); + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { RsQThreadUtils::postToObject( [=]() { handleEvent_main_thread(event); }, this ); }, + mEventHandlerId, RsEventType::GXS_FORUMS ); } void GxsForumsDialog::handleEvent_main_thread(std::shared_ptr event) diff --git a/retroshare-gui/src/gui/statusbar/hashingstatus.cpp b/retroshare-gui/src/gui/statusbar/hashingstatus.cpp index 997ca5f8f..563c8385a 100644 --- a/retroshare-gui/src/gui/statusbar/hashingstatus.cpp +++ b/retroshare-gui/src/gui/statusbar/hashingstatus.cpp @@ -52,8 +52,10 @@ HashingStatus::HashingStatus(QWidget *parent) hashloader->hide(); statusHashing->hide(); - mEventHandlerId=0; - rsEvents->registerEventsHandler(RsEventType::SHARED_DIRECTORIES, [this](std::shared_ptr event) { handleEvent(event); }, mEventHandlerId ); + mEventHandlerId=0; + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) { handleEvent(event); }, + mEventHandlerId, RsEventType::SHARED_DIRECTORIES ); } void HashingStatus::handleEvent(std::shared_ptr event) diff --git a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp index efd54a8e7..5128f069d 100644 --- a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp +++ b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp @@ -1,7 +1,7 @@ /******************************************************************************* * util/RsGxsUpdateBroadcast.cpp * * * - * Copyright (c) 2014 Retroshare Team * + * Copyright (C) 2014-2020 Retroshare Team * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License as * @@ -32,17 +32,17 @@ // now the update notify works through rsnotify and notifyqt // so the single instance per service is not really needed anymore -QMap updateBroadcastMap; +static QMap updateBroadcastMap; RsGxsUpdateBroadcast::RsGxsUpdateBroadcast(RsGxsIfaceHelper *ifaceImpl) : - QObject(NULL), mIfaceImpl(ifaceImpl) + QObject(nullptr), mIfaceImpl(ifaceImpl), mEventHandlerId(0) { - mEventHandlerId = 0; // forces initialization in registerEventsHandler() - - rsEvents->registerEventsHandler(RsEventType::GXS_CHANGES, [this](std::shared_ptr event) - { - onChangesReceived(*dynamic_cast(event.get())); - }, mEventHandlerId ); + /* No need of postToObject here as onChangesReceived just emit signals + * internally */ + rsEvents->registerEventsHandler( + [this](std::shared_ptr event) + { onChangesReceived(*dynamic_cast(event.get())); }, + mEventHandlerId, RsEventType::GXS_CHANGES ); } RsGxsUpdateBroadcast::~RsGxsUpdateBroadcast() From dc9ba74bdd72a4607ec7e7ababc0150ed97e1337 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 1 Apr 2020 23:56:37 +0200 Subject: [PATCH 22/40] Add missing postToObjcted in RsGxsUpdateBroadcast Deprecate RsGxsUpdateBroadcast each service should handle its own event --- retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp | 10 +++++++--- retroshare-gui/src/util/RsGxsUpdateBroadcast.h | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp index 5128f069d..c228923e8 100644 --- a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp +++ b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp @@ -22,6 +22,7 @@ #include "RsGxsUpdateBroadcast.h" #include "gui/notifyqt.h" +#include "util/qtthreadsutils.h" #include @@ -37,11 +38,14 @@ static QMap updateBroadcastMap; RsGxsUpdateBroadcast::RsGxsUpdateBroadcast(RsGxsIfaceHelper *ifaceImpl) : QObject(nullptr), mIfaceImpl(ifaceImpl), mEventHandlerId(0) { - /* No need of postToObject here as onChangesReceived just emit signals - * internally */ rsEvents->registerEventsHandler( [this](std::shared_ptr event) - { onChangesReceived(*dynamic_cast(event.get())); }, + { + RsQThreadUtils::postToObject( + [=]() + { onChangesReceived(*dynamic_cast(event.get())); }, + this ); + }, mEventHandlerId, RsEventType::GXS_CHANGES ); } diff --git a/retroshare-gui/src/util/RsGxsUpdateBroadcast.h b/retroshare-gui/src/util/RsGxsUpdateBroadcast.h index 1ec733bfa..1b700900d 100644 --- a/retroshare-gui/src/util/RsGxsUpdateBroadcast.h +++ b/retroshare-gui/src/util/RsGxsUpdateBroadcast.h @@ -31,7 +31,7 @@ struct RsGxsChanges; typedef uint32_t TurtleRequestId ; -class RsGxsUpdateBroadcast : public QObject +class RS_DEPRECATED RsGxsUpdateBroadcast : public QObject { Q_OBJECT From 7d1efa7ed915582e0d695bc65cb87c862385813d Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 2 Apr 2020 23:58:09 +0200 Subject: [PATCH 23/40] Fix compiler warnings --- libretroshare/src/retroshare/rsfiles.h | 5 +++-- libretroshare/src/retroshare/rstypes.h | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 8a60444c7..78de40262 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -402,12 +402,12 @@ struct TurtleFileInfoV2 : RsSerializable { TurtleFileInfoV2() : fSize(0), fWeight(0) {} - TurtleFileInfoV2(const TurtleFileInfo& oldInfo) : + explicit TurtleFileInfoV2(const TurtleFileInfo& oldInfo) : fSize(oldInfo.size), fHash(oldInfo.hash), fName(oldInfo.name), fWeight(0) {} #ifdef RS_DEEP_FILES_INDEX - TurtleFileInfoV2(const DeepFilesSearchResult& dRes); + explicit TurtleFileInfoV2(const DeepFilesSearchResult& dRes); #endif // def RS_DEEP_FILES_INDEX uint64_t fSize; /// File size @@ -746,6 +746,7 @@ public: * interaction with Qt. As soon as you can, you should prefer to use the * version of this methodn which take `std::uintptr_t handle` as paramether. */ + RS_DEPRECATED_FOR(requestDirDetails) virtual int RequestDirDetails( void* handle, DirDetails& details, FileSearchFlags flags ) = 0; diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 978934942..60902b7ab 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -201,7 +201,8 @@ struct FileInfo : RsSerializable std::string path; std::string fname; RsFileHash hash; - std::string ext; + + RS_DEPRECATED std::string ext; /// @deprecated unused uint64_t size; uint64_t avail; /// how much we have From 8cb46e59376447d41013e6cf96bc7d9a4a02f691 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 3 Apr 2020 13:00:44 +0200 Subject: [PATCH 24/40] Fix compilation on Android arm64 --- libretroshare/src/ft/ftserver.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 9f5c1d1bd..4c6c5b6c3 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -20,6 +20,9 @@ * * *******************************************************************************/ +#include +#include + #include "crypto/chacha20.h" //const int ftserverzone = 29539; @@ -50,8 +53,6 @@ #include "util/rsdebug.h" #include "util/rsdir.h" #include "util/rsprint.h" - -#include #include "util/rstime.h" #ifdef RS_DEEP_FILES_INDEX @@ -1974,15 +1975,17 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item) hasCallback = true; std::vector cRes; - for( const auto& tfiold : item->result) - cRes.push_back(tfiold); + std::transform( item->result.begin(), item->result.end(), + std::back_inserter(cRes), + [](const auto& it){ return TurtleFileInfoV2(it); } ); cbpt->second.first(cRes); } } // end RS_STACK_MUTEX(mSearchCallbacksMapMutex); if(!hasCallback) - RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result ); + RsServer::notify()->notifyTurtleSearchResult( + item->PeerId(), item->request_id, item->result ); } bool ftServer::receiveSearchRequest( From 612d47908dbe4feafba8749da0ff78da6183a0b4 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 01:00:29 +0200 Subject: [PATCH 25/40] Fix JSON API breakage introduced in cf7a77e51250b66052799b4453f0be916afb3e8e --- libretroshare/src/retroshare/rsgxschannels.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 901d4c4ef..c31eb011f 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -312,13 +312,15 @@ public: * @param[out] comments storage for the comments * @return false if something failed, true otherwhise */ - virtual bool getChannelAllContent(const RsGxsGroupId& channelId, + virtual bool getChannelAllContent( const RsGxsGroupId& channelId, std::vector& posts, std::vector& comments ) = 0; /** - * @brief Get channel messages and comments corresponding to the given message ID list. If the list is empty, nothing is returned. Since - * comments are themselves messages, it is possible to get comments only by only supplying their message IDs. + * @brief Get channel messages and comments corresponding to the given IDs. + * If the set is empty, nothing is returned. + * @note Since comments are internally themselves messages, it is possible + * to get comments only by supplying their IDs. * @jsonapi{development} * @param[in] channelId id of the channel of which the content is requested * @param[in] contentsIds ids of requested contents @@ -326,8 +328,8 @@ public: * @param[out] comments storage for the comments * @return false if something failed, true otherwhise */ - virtual bool getChannelContent(const RsGxsGroupId& channelId, - const std::set& contentIds, + virtual bool getChannelContent( const RsGxsGroupId& channelId, + const std::set& contentsIds, std::vector& posts, std::vector& comments ) = 0; From 13e01798120f64560aa08e3d9e1a7c582010824f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 01:09:18 +0200 Subject: [PATCH 26/40] Readd struct disappeared in rebase + fix warning --- libretroshare/src/gxs/rsgxsdataaccess.cc | 1 + libretroshare/src/retroshare/rsgxsifacetypes.h | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index e5f26dbb3..0b23594c2 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1969,3 +1969,4 @@ bool RsGxsDataAccess::checkMsgFilter( return true; } +GxsGroupStatistic::~GxsGroupStatistic() = default; diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index 13230156a..ad4979067 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -191,6 +191,14 @@ struct RsMsgMetaData : RsSerializable } }; +struct RsGxsGenericMsgData +{ + virtual ~RsGxsGenericMsgData() = default; // making the type polymorphic + + RsMsgMetaData mMeta; +}; + + struct GxsGroupStatistic : RsSerializable { GxsGroupStatistic() : @@ -199,7 +207,7 @@ struct GxsGroupStatistic : RsSerializable /// @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx) override + RsGenericSerializer::SerializeContext& ctx ) override { RS_SERIAL_PROCESS(mGrpId); RS_SERIAL_PROCESS(mNumMsgs); @@ -219,7 +227,7 @@ struct GxsGroupStatistic : RsSerializable uint32_t mNumChildMsgsNew; uint32_t mNumChildMsgsUnread; - ~GxsGroupStatistic() override = default; + ~GxsGroupStatistic() override; }; class GxsServiceStatistic From 05f65e9211733082c3b0e91cfc3153acb018518d Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 11:17:33 +0200 Subject: [PATCH 27/40] Fix missing paramether direction in Doxygen doc --- libretroshare/src/retroshare/rsevents.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/retroshare/rsevents.h b/libretroshare/src/retroshare/rsevents.h index c960a16a2..667ab4a58 100644 --- a/libretroshare/src/retroshare/rsevents.h +++ b/libretroshare/src/retroshare/rsevents.h @@ -184,8 +184,8 @@ public: * value may be provided to the function call but * must habe been generated with * @see generateUniqueHandlerId() - * @param eventType Optional type of event for which the callback is - * called, if NONE is passed multiCallback is + * @param[in] eventType Optional type of event for which the callback is + * called, if __NONE is passed multiCallback is * called for every events without filtering. * @return False on error, true otherwise. */ From 76d492b4aa0f762fcf39dcb315412756fcc7f452 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 17:44:16 +0200 Subject: [PATCH 28/40] Add forgot authorship in rsbase64* --- libretroshare/src/util/rsbase64.cc | 1 + libretroshare/src/util/rsbase64.h | 1 + 2 files changed, 2 insertions(+) diff --git a/libretroshare/src/util/rsbase64.cc b/libretroshare/src/util/rsbase64.cc index 80e0f20d1..6b856888e 100644 --- a/libretroshare/src/util/rsbase64.cc +++ b/libretroshare/src/util/rsbase64.cc @@ -2,6 +2,7 @@ * * * libretroshare base64 encoding utilities * * * + * Copyright (C) 2015 Retroshare Team * * Copyright (C) 2020 Gioacchino Mazzurco * * Copyright (C) 2020 Asociación Civil Altermundi * * * diff --git a/libretroshare/src/util/rsbase64.h b/libretroshare/src/util/rsbase64.h index b11ec08cc..0715b15ae 100644 --- a/libretroshare/src/util/rsbase64.h +++ b/libretroshare/src/util/rsbase64.h @@ -2,6 +2,7 @@ * * * libretroshare base64 encoding utilities * * * + * Copyright (C) 2015 Retroshare Team * * Copyright (C) 2020 Gioacchino Mazzurco * * Copyright (C) 2020 Asociación Civil Altermundi * * * From b701ca8da38dc7ab446c8cc4a7f42d8ed22de504 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 6 Apr 2020 17:10:16 +0200 Subject: [PATCH 29/40] Keep 10 extra unassigned slots for event types Proper error reporting + cruft removal --- libretroshare/src/jsonapi/jsonapi.cpp | 2 +- libretroshare/src/jsonapi/jsonapi.h | 2 - libretroshare/src/retroshare/rsevents.h | 82 ++++++++-- libretroshare/src/services/rseventsservice.cc | 148 ++++++++---------- libretroshare/src/services/rseventsservice.h | 24 +-- 5 files changed, 149 insertions(+), 109 deletions(-) diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index a24d73c4b..295d2974a 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -395,7 +395,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), } ); }; - bool retval = rsEvents->registerEventsHandler( + auto retval = rsEvents->registerEventsHandler( multiCallback, hId, eventType ); { diff --git a/libretroshare/src/jsonapi/jsonapi.h b/libretroshare/src/jsonapi/jsonapi.h index 70cc5e309..317d4066f 100644 --- a/libretroshare/src/jsonapi/jsonapi.h +++ b/libretroshare/src/jsonapi/jsonapi.h @@ -151,8 +151,6 @@ protected: /// @see RsThread void onStopRequested() override; - static const RsJsonApiErrorCategory sErrorCategory; - static std::error_condition badApiCredientalsFormat( const std::string& user, const std::string& passwd ); diff --git a/libretroshare/src/retroshare/rsevents.h b/libretroshare/src/retroshare/rsevents.h index 667ab4a58..47a499497 100644 --- a/libretroshare/src/retroshare/rsevents.h +++ b/libretroshare/src/retroshare/rsevents.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2019 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Retroshare Team * + * Copyright (C) 2019-2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -97,9 +99,60 @@ enum class RsEventType : uint32_t /// @see RsMsgs CHAT_MESSAGE = 15, - __MAX /// Used internally to detect invalid event type passed + __MAX /// Used internally, keep last }; +enum class RsEventsErrorNum : int32_t +{ + EVENT_TYPE_UNDEFINED = 3004, + EVENT_TYPE_OUT_OF_RANGE = 3005, + INVALID_HANDLER_ID = 3006, + NULL_EVENT_POINTER = 3007 +}; + +struct RsEventsErrorCategory: std::error_category +{ + const char* name() const noexcept override + { return "RetroShare Events"; } + + std::string message(int ev) const override + { + switch (static_cast(ev)) + { + case RsEventsErrorNum::EVENT_TYPE_UNDEFINED: + return "Undefined event type"; + case RsEventsErrorNum::EVENT_TYPE_OUT_OF_RANGE: + return "Event type out of range"; + case RsEventsErrorNum::INVALID_HANDLER_ID: + return "Invalid handler id"; + default: + return "Error message for error: " + std::to_string(ev) + + " not available in category: " + name(); + } + } + + std::error_condition default_error_condition(int ev) const noexcept override; + + const static RsEventsErrorCategory instance; +}; + + +namespace std +{ +/** Register RsJsonApiErrorNum as an error condition enum, must be in std + * namespace */ +template<> struct is_error_condition_enum : true_type {}; +} + +/** Provide RsEventsErrorNum conversion to std::error_condition, must be in + * same namespace of RsJsonApiErrorNum */ +inline std::error_condition make_error_condition(RsEventsErrorNum e) noexcept +{ + return std::error_condition( + static_cast(e), RsEventsErrorCategory::instance ); +}; + + /** * This struct is not meant to be used directly, you should create events type * deriving from it. @@ -144,12 +197,10 @@ public: * @param[in] event * @param[out] errorMessage Optional storage for error messsage, meaningful * only on failure. - * @return False on error, true otherwise. + * @return Success or error details. */ - virtual bool postEvent( - std::shared_ptr event, - std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string) - ) = 0; + virtual std::error_condition postEvent( + std::shared_ptr event ) = 0; /** * @brief Send event directly to handlers. Blocking API @@ -157,12 +208,10 @@ public: * @param[in] event * @param[out] errorMessage Optional storage for error messsage, meaningful * only on failure. - * @return False on error, true otherwise. + * @return Success or error details. */ - virtual bool sendEvent( - std::shared_ptr event, - std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string) - ) = 0; + virtual std::error_condition sendEvent( + std::shared_ptr event ) = 0; /** * @brief Generate unique handler identifier @@ -187,9 +236,9 @@ public: * @param[in] eventType Optional type of event for which the callback is * called, if __NONE is passed multiCallback is * called for every events without filtering. - * @return False on error, true otherwise. + * @return Success or error details. */ - virtual bool registerEventsHandler( + virtual std::error_condition registerEventsHandler( std::function)> multiCallback, RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0), RsEventType eventType = RsEventType::__NONE ) = 0; @@ -197,9 +246,10 @@ public: /** * @brief Unregister event handler * @param[in] hId Id of the event handler to unregister - * @return True if the handler id has been found, false otherwise. + * @return Success or error details. */ - virtual bool unregisterEventsHandler(RsEventsHandlerId_t hId) = 0; + virtual std::error_condition unregisterEventsHandler( + RsEventsHandlerId_t hId ) = 0; virtual ~RsEvents(); }; diff --git a/libretroshare/src/services/rseventsservice.cc b/libretroshare/src/services/rseventsservice.cc index f0db8bc3f..3bbb852be 100644 --- a/libretroshare/src/services/rseventsservice.cc +++ b/libretroshare/src/services/rseventsservice.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2019 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Retroshare Team * + * Copyright (C) 2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -27,64 +29,62 @@ /*extern*/ RsEvents* rsEvents = nullptr; -RsEvent::~RsEvent() {}; -RsEvents::~RsEvents() {}; -bool isEventValid( - std::shared_ptr event, std::string& errorMessage ) +RsEvent::~RsEvent() = default; +RsEvents::~RsEvents() = default; + +/*static*/ const RsEventsErrorCategory RsEventsErrorCategory::instance; + +std::error_condition RsEventsErrorCategory::default_error_condition(int ev) +const noexcept { - if(!event) + switch(static_cast(ev)) { - errorMessage = "Event is null!"; - return false; + case RsEventsErrorNum::INVALID_HANDLER_ID: // [[fallthrough]]; + case RsEventsErrorNum::NULL_EVENT_POINTER: // [[fallthrough]]; + case RsEventsErrorNum::EVENT_TYPE_UNDEFINED: // [[fallthrough]]; + case RsEventsErrorNum::EVENT_TYPE_OUT_OF_RANGE: + return std::errc::invalid_argument; + default: + return std::error_condition(ev, *this); } - - if(event->mType <= RsEventType::__NONE) - { - errorMessage = "Event has type NONE: " + - std::to_string( - static_cast::type >( - event->mType ) ); - return false; - } - - if(event->mType >= RsEventType::__MAX) - { - errorMessage = "Event has type >= RsEventType::__MAX: " + - std::to_string( - static_cast::type >( - event->mType ) ); - } - - return true; } -bool RsEventsService::postEvent( std::shared_ptr event, - std::string& errorMessage ) +std::error_condition RsEventsService::isEventTypeInvalid(RsEventType eventType) { - if(!isEventValid(event, errorMessage)) - { - std::cerr << __PRETTY_FUNCTION__ << " Error: "<< errorMessage - << std::endl; - return false; - } + if(eventType == RsEventType::__NONE) + return RsEventsErrorNum::EVENT_TYPE_UNDEFINED; + + if( eventType < RsEventType::__NONE || + eventType >= static_cast(mHandlerMaps.size()) ) + return RsEventsErrorNum::EVENT_TYPE_OUT_OF_RANGE; + + return std::error_condition(); +} + +std::error_condition RsEventsService::isEventInvalid( + std::shared_ptr event) +{ + if(!event) return RsEventsErrorNum::NULL_EVENT_POINTER; + return isEventTypeInvalid(event->mType); +} + +std::error_condition RsEventsService::postEvent( + std::shared_ptr event ) +{ + if(std::error_condition ec = isEventInvalid(event)) return ec; RS_STACK_MUTEX(mEventQueueMtx); mEventQueue.push_back(event); - return true; + return std::error_condition(); } -bool RsEventsService::sendEvent( std::shared_ptr event, - std::string& errorMessage ) +std::error_condition RsEventsService::sendEvent( + std::shared_ptr event ) { - if(!isEventValid(event, errorMessage)) - { - RsErr() << __PRETTY_FUNCTION__ << " "<< errorMessage << std::endl; - return false; - } - + if(std::error_condition ec = isEventInvalid(event)) return ec; handleEvent(event); - return true; + return std::error_condition(); } RsEventsHandlerId_t RsEventsService::generateUniqueHandlerId() @@ -99,35 +99,29 @@ RsEventsHandlerId_t RsEventsService::generateUniqueHandlerId_unlocked() return 1; } -bool RsEventsService::registerEventsHandler( +std::error_condition RsEventsService::registerEventsHandler( std::function)> multiCallback, RsEventsHandlerId_t& hId, RsEventType eventType ) { RS_STACK_MUTEX(mHandlerMapMtx); - if( eventType >= RsEventType::__MAX) - { - RsErr() << __PRETTY_FUNCTION__ << " Invalid event type: " - << static_cast(eventType) << " >= RsEventType::__MAX:" - << static_cast(RsEventType::__MAX) << std::endl; - print_stacktrace(); - return false; - } + if(eventType != RsEventType::__NONE) + if(std::error_condition ec = isEventTypeInvalid(eventType)) + return ec; if(!hId) hId = generateUniqueHandlerId_unlocked(); else if (hId > mLastHandlerId) { - RsErr() << __PRETTY_FUNCTION__ << " Invalid handler id: " << hId - << " how did you generate it? " << std::endl; print_stacktrace(); - return false; + return RsEventsErrorNum::INVALID_HANDLER_ID; } mHandlerMaps[static_cast(eventType)][hId] = multiCallback; - return true; + return std::error_condition(); } -bool RsEventsService::unregisterEventsHandler(RsEventsHandlerId_t hId) +std::error_condition RsEventsService::unregisterEventsHandler( + RsEventsHandlerId_t hId ) { RS_STACK_MUTEX(mHandlerMapMtx); @@ -137,10 +131,10 @@ bool RsEventsService::unregisterEventsHandler(RsEventsHandlerId_t hId) if(it != mHandlerMaps[i].end()) { mHandlerMaps[i].erase(it); - return true; + return std::error_condition(); } } - return false; + return RsEventsErrorNum::INVALID_HANDLER_ID; } void RsEventsService::threadTick() @@ -179,29 +173,25 @@ dispatchEventFromQueueLock: void RsEventsService::handleEvent(std::shared_ptr event) { - uint32_t event_type_index = static_cast(event->mType); - - if(RsEventType::__NONE >= event->mType || event->mType >= RsEventType::__MAX ) + if(std::error_condition ec = isEventInvalid(event)) { - RsErr() << __PRETTY_FUNCTION__ << " Invalid event type: " - << event_type_index << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl; print_stacktrace(); return; } - { - RS_STACK_MUTEX(mHandlerMapMtx); - /* It is important to also call the callback under mutex protection to - * ensure they are not unregistered in the meanwhile. - * If a callback try to fiddle with registering/unregistering it will - * deadlock */ + RS_STACK_MUTEX(mHandlerMapMtx); + /* It is important to also call the callback under mutex protection to + * ensure they are not unregistered in the meanwhile. + * If a callback try to fiddle with registering/unregistering it will + * deadlock */ - // Call all clients that registered a callback for this event type - for(auto cbit: mHandlerMaps[event_type_index]) cbit.second(event); + // Call all clients that registered a callback for this event type + for(auto cbit: mHandlerMaps[static_cast(event->mType)]) + cbit.second(event); - /* Also call all clients that registered with NONE, meaning that they - * expect all events */ - for(auto cbit: mHandlerMaps[static_cast(RsEventType::__NONE)]) - cbit.second(event); - } + /* Also call all clients that registered with NONE, meaning that they + * expect all events */ + for(auto cbit: mHandlerMaps[static_cast(RsEventType::__NONE)]) + cbit.second(event); } diff --git a/libretroshare/src/services/rseventsservice.h b/libretroshare/src/services/rseventsservice.h index 3a61cdd79..4f6ce98eb 100644 --- a/libretroshare/src/services/rseventsservice.h +++ b/libretroshare/src/services/rseventsservice.h @@ -40,38 +40,40 @@ public: mEventQueueMtx("RsEventsService::mEventQueueMtx") {} /// @see RsEvents - bool postEvent( - std::shared_ptr event, - std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string) - ) override; + std::error_condition postEvent( + std::shared_ptr event ) override; /// @see RsEvents - bool sendEvent( - std::shared_ptr event, - std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string) - ) override; + std::error_condition sendEvent( + std::shared_ptr event ) override; /// @see RsEvents RsEventsHandlerId_t generateUniqueHandlerId() override; /// @see RsEvents - bool registerEventsHandler( + std::error_condition registerEventsHandler( std::function)> multiCallback, RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0), RsEventType eventType = RsEventType::__NONE ) override; /// @see RsEvents - bool unregisterEventsHandler(RsEventsHandlerId_t hId) override; + std::error_condition unregisterEventsHandler( + RsEventsHandlerId_t hId ) override; protected: + std::error_condition isEventTypeInvalid(RsEventType eventType); + std::error_condition isEventInvalid(std::shared_ptr event); + RsMutex mHandlerMapMtx; RsEventsHandlerId_t mLastHandlerId; + /** Storage for event handlers, keep 10 extra types for plugins that might + * be released indipendently */ std::array< std::map< RsEventsHandlerId_t, std::function)> >, - static_cast(RsEventType::__MAX) + static_cast(RsEventType::__MAX) + 10 > mHandlerMaps; RsMutex mEventQueueMtx; From 0c3fd6f27ce467f741d07bc135850526a0461e9c Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 6 Apr 2020 17:37:28 +0200 Subject: [PATCH 30/40] Explicit type as requested by Cyril --- libretroshare/src/jsonapi/jsonapi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index 295d2974a..338782a01 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -395,7 +395,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), } ); }; - auto retval = rsEvents->registerEventsHandler( + std::error_condition retval = rsEvents->registerEventsHandler( multiCallback, hId, eventType ); { From a4950aca66ba2ce8fb62119536b3179c4f44cb7c Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 8 Apr 2020 17:51:36 +0200 Subject: [PATCH 31/40] Safer checks in type serializer --- libretroshare/src/serialiser/rsserial.cc | 21 ++---------- libretroshare/src/serialiser/rsserial.h | 24 ++++++++------ .../src/serialiser/rstypeserializer.h | 33 ++++++++++++++----- 3 files changed, 41 insertions(+), 37 deletions(-) diff --git a/libretroshare/src/serialiser/rsserial.cc b/libretroshare/src/serialiser/rsserial.cc index dda560867..52123abf1 100644 --- a/libretroshare/src/serialiser/rsserial.cc +++ b/libretroshare/src/serialiser/rsserial.cc @@ -125,9 +125,7 @@ RsItem::RsItem(uint8_t ver, uint8_t cls, uint8_t t, uint8_t subtype) type = (ver << 24) + (cls << 16) + (t << 8) + subtype; } -RsItem::~RsItem() -{ -} +RsItem::~RsItem() = default; void RsItem::print_string(std::string &out, uint16_t indent) { @@ -243,10 +241,7 @@ uint32_t RsSerialType::PacketId() const -RsSerialiser::RsSerialiser() -{ - return; -} +RsSerialiser::RsSerialiser() = default; RsSerialiser::~RsSerialiser() @@ -559,17 +554,7 @@ std::ostream &RsRawItem::print(std::ostream &out, uint16_t indent) return out; } - -uint32_t getRsPktMaxSize() -{ - //return 65535; /* 2^16 (old artifical low size) */ - //return 1048575; /* 2^20 -1 (Too Big! - must remove fixed static buffers first) */ - /* Remember that every pqistreamer allocates an input buffer of this size! - * So don't make it too big! - */ - return 262143; /* 2^18 -1 */ -} - +uint32_t getRsPktMaxSize() { return RsSerialiser::MAX_SERIAL_SIZE; } uint32_t getRsPktBaseSize() { diff --git a/libretroshare/src/serialiser/rsserial.h b/libretroshare/src/serialiser/rsserial.h index 3ce56a424..168db8d75 100644 --- a/libretroshare/src/serialiser/rsserial.h +++ b/libretroshare/src/serialiser/rsserial.h @@ -19,16 +19,16 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_BASE_SERIALISER_H -#define RS_BASE_SERIALISER_H +#pragma once -#include -#include +#include #include #include #include -#include -#include +#include +#include + +#include "util/rsdeprecate.h" /******************************************************************* * This is the Top-Level serialiser/deserialise, @@ -66,7 +66,11 @@ class RsSerialType ; class RsSerialiser { - public: +public: + /** Remember that every pqistreamer allocates an input buffer of this size! + * So don't make it too big! */ + static constexpr uint32_t MAX_SERIAL_SIZE = 262143; /* 2^18 -1 */ + RsSerialiser(); ~RsSerialiser(); bool addSerialType(RsSerialType *type); @@ -76,7 +80,7 @@ class RsSerialiser RsItem * deserialise(void *data, uint32_t *size); - private: +private: std::map serialisers; }; @@ -95,6 +99,8 @@ uint16_t getRsItemService(uint32_t type); /* size constants */ uint32_t getRsPktBaseSize(); + +RS_DEPRECATED_FOR(RsSerialiser::MAX_SERIAL_SIZE) uint32_t getRsPktMaxSize(); @@ -106,5 +112,3 @@ std::ostream &printRsItemEnd(std::ostream &o, std::string n, uint16_t i); /* defined in rstlvtypes.cc - redeclared here for ease */ std::ostream &printIndent(std::ostream &out, uint16_t indent); /* Wrapper class for data that is serialised somewhere else */ - -#endif /* RS_BASE_SERIALISER_H */ diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index adf885c31..d72280bb3 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -480,7 +480,7 @@ struct RsTypeSerializer switch(j) { - case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]] + case RsGenericSerializer::SIZE_ESTIMATE: // [[falltrough]]; case RsGenericSerializer::SERIALIZE: { uint32_t aSize = member.size(); @@ -499,10 +499,11 @@ struct RsTypeSerializer if(!ctx.mOk) break; /* This check is not perfect but will catch most pathological cases. - * Avoid multiplying by sizeof(el_t) as it is not a good exitimation + * Avoid multiplying by sizeof(el_t) as it is not a good estimation * of the actual serialized size, depending on the elements - * structure and on the so it would raises many false positives. */ - if(elCount > ctx.mSize - ctx.mOffset) + * structure and on the so it would raises many false positives. + * Arithmetic operations on elCount are also at risk of overflow */ + if(elCount > RsSerialiser::MAX_SERIAL_SIZE) { ctx.mOk = false; RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " @@ -626,7 +627,7 @@ struct RsTypeSerializer { uint32_t len = static_cast(member.length()); RS_SERIAL_PROCESS(len); - if(len > ctx.mSize - ctx.mOffset) + if(len + ctx.mOffset > ctx.mSize) { RsErr() << __PRETTY_FUNCTION__ << std::errc::no_buffer_space << std::endl; @@ -641,19 +642,33 @@ struct RsTypeSerializer uint32_t len; RS_SERIAL_PROCESS(len); if(!ctx.mOk) break; - if(len > ctx.mSize - ctx.mOffset) + + if(len > RsSerialiser::MAX_SERIAL_SIZE) { ctx.mOk = false; RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " - << "string with apparently malformed elements count." + << "string with apparently malformed length." << " len: " << len << " ctx.mSize: " << ctx.mSize << " ctx.mOffset: " << ctx.mOffset << " " - << std::errc::argument_out_of_domain - << std::endl; + << std::errc::argument_out_of_domain << std::endl; print_stacktrace(); break; } + + if(len + ctx.mOffset > ctx.mSize) + { + ctx.mOk = false; + RsErr() << __PRETTY_FUNCTION__ << " attempt to deserialize a " + << "string with a length bigger available data." + << " len: " << len + << " ctx.mSize: " << ctx.mSize + << " ctx.mOffset: " << ctx.mOffset << " " + << std::errc::no_buffer_space << std::endl; + print_stacktrace(); + break; + } + member.resize(len); memcpy(&member[0], ctx.mData + ctx.mOffset, len); ctx.mOffset += len; From 2a369cb785b9859aca4dea77b680b8d2d8121c51 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 8 Apr 2020 18:54:22 +0200 Subject: [PATCH 32/40] Fix noisy compiler warning --- libretroshare/src/retroshare/rsgxsifacehelper.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxsifacehelper.h b/libretroshare/src/retroshare/rsgxsifacehelper.h index 1ea2ad061..0addbf6e5 100644 --- a/libretroshare/src/retroshare/rsgxsifacehelper.h +++ b/libretroshare/src/retroshare/rsgxsifacehelper.h @@ -3,8 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2011 by Christopher Evi-Parker * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2011 Christopher Evi-Parker * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -66,10 +66,11 @@ public: * @param gxs handle to RsGenExchange instance of service (Usually the * service class itself) */ - RsGxsIfaceHelper(RsGxsIface& gxs) : - mGxs(gxs), mTokenService(*gxs.getTokenService()),mMtx("GxsIfaceHelper") {} + explicit RsGxsIfaceHelper(RsGxsIface& gxs) : + mGxs(gxs), mTokenService(*gxs.getTokenService()), mMtx("GxsIfaceHelper") + {} - ~RsGxsIfaceHelper(){} + ~RsGxsIfaceHelper() = default; /*! * Gxs services should call this for automatic handling of @@ -507,7 +508,7 @@ private: void locked_dumpTokens() { const uint16_t service_id = mGxs.serviceType(); - const auto countSize = static_cast(TokenRequestType::__MAX); + const auto countSize = static_cast(TokenRequestType::__MAX); uint32_t count[countSize] = {0}; RsDbg() << __PRETTY_FUNCTION__ << "Service 0x" << std::hex << service_id @@ -517,7 +518,7 @@ private: << ") Active tokens (per type): "; // let's count how many token of each type we've got. - for(auto& it: mActiveTokens) ++count[static_cast(it.second)]; + for(auto& it: mActiveTokens) ++count[static_cast(it.second)]; for(uint32_t i=0; i < countSize; ++i) RsDbg().uStream() /* << i << ":" */ << count[i] << " "; From 7d388c3090f1e523b40b42f8634d22738fc75314 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 16:33:19 +0200 Subject: [PATCH 33/40] Solve non compatible 32/64bit file handle API --- libretroshare/src/file_sharing/file_tree.cc | 17 ++++---- libretroshare/src/ft/ftserver.cc | 8 ++-- libretroshare/src/ft/ftserver.h | 4 +- libretroshare/src/retroshare/rsfiles.h | 14 +++---- libretroshare/src/retroshare/rstypes.h | 44 ++++++++------------- 5 files changed, 39 insertions(+), 48 deletions(-) diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index ddc24c3ac..ad34501e3 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -160,20 +160,23 @@ void RsFileTree::recurs_buildFileTree( } bool RsFileTree::getDirectoryContent( - std::string& name, std::vector& subdirs, - std::vector& subfiles, std::uintptr_t index ) const + std::string& name, std::vector& subdirs, + std::vector& subfiles, uint64_t index_p ) const { - if(index >= mDirs.size()) - return false ; + // Avoid warnings on Android armv7 + using sz_t = std::vector::size_type; + sz_t index = static_cast(index_p); + + if(index >= mDirs.size()) return false; name = mDirs[index].name; subdirs = mDirs[index].subdirs ; subfiles.clear() ; - for(uint32_t i=0;i(mDirs[index].subfiles[i])]); - return true ; + return true; } std::unique_ptr RsFileTree::fromDirDetails( diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 4c6c5b6c3..d2c2f9f0a 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -345,12 +345,12 @@ std::error_condition ftServer::requestFiles( // Track how many time a directory have been explored std::vector dirsSeenCnt(dirsCount, 0); // - using StackEntry = std::tuple; + using StackEntry = std::tuple; std::deque dStack = { std::make_tuple(0, basePath) }; const auto exploreDir = [&](const StackEntry& se)-> std::error_condition { - std::uintptr_t dirHandle; std::string parentPath; + uint64_t dirHandle; std::string parentPath; std::tie(dirHandle, parentPath) = se; const auto& dirData = collection.mDirs[dirHandle]; @@ -843,7 +843,7 @@ bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFl } bool ftServer::requestDirDetails( - DirDetails &details, std::uintptr_t handle, FileSearchFlags flags ) + DirDetails &details, uint64_t handle, FileSearchFlags flags ) { return RequestDirDetails(reinterpret_cast(handle), details, flags); } int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) @@ -2249,7 +2249,7 @@ std::error_condition ftServer::dirDetailsToLink( } std::error_condition ftServer::exportCollectionLink( - std::string& link, std::uintptr_t handle, bool fragSneak, + std::string& link, uint64_t handle, bool fragSneak, const std::string& baseUrl ) { DirDetails tDirDet; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 789bba49d..f6cc4b44c 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -225,7 +225,7 @@ public: /// @see RsFiles std::error_condition exportCollectionLink( - std::string& link, std::uintptr_t handle, bool fragSneak = false, + std::string& link, uint64_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) override; @@ -279,7 +279,7 @@ public: /// @see RsFiles::RequestDirDetails virtual bool requestDirDetails( - DirDetails &details, std::uintptr_t handle = 0, + DirDetails &details, uint64_t handle = 0, FileSearchFlags flags = RS_FILE_HINTS_LOCAL ); virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ; diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 78de40262..64563f621 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -324,14 +324,14 @@ public: /// Allow browsing the hierarchy bool getDirectoryContent( - std::string& name, std::vector& subdirs, - std::vector& subfiles, std::uintptr_t handle = 0 ) const; + std::string& name, std::vector& subdirs, + std::vector& subfiles, uint64_t handle = 0 ) const; struct DirData: RsSerializable { std::string name; - std::vector subdirs; - std::vector subfiles; + std::vector subdirs; + std::vector subfiles; void serial_process( RsGenericSerializer::SerializeJob j, @@ -735,7 +735,7 @@ public: * @return false if error occurred, true otherwise */ virtual bool requestDirDetails( - DirDetails &details, std::uintptr_t handle = 0, + DirDetails &details, uint64_t handle = 0, FileSearchFlags flags = RS_FILE_HINTS_LOCAL ) = 0; /*** @@ -744,7 +744,7 @@ public: /** * Kept for retrocompatibility, it was originally written for easier * interaction with Qt. As soon as you can, you should prefer to use the - * version of this methodn which take `std::uintptr_t handle` as paramether. + * version of this method which take `uint64_t handle` as paramether. */ RS_DEPRECATED_FOR(requestDirDetails) virtual int RequestDirDetails( @@ -927,7 +927,7 @@ public: * @return error information if some error occurred, 0/SUCCESS otherwise */ virtual std::error_condition exportCollectionLink( - std::string& link, std::uintptr_t handle, bool fragSneak = false, + std::string& link, uint64_t handle, bool fragSneak = false, const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0; /** diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 60902b7ab..2daa25947 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -304,10 +304,16 @@ struct DirDetails : RsSerializable type(DIR_TYPE_UNKNOWN), count(0), mtime(0), max_mtime(0) {} + /* G10h4ck do we still need to keep this as void* instead of uint64_t for + * retroshare-gui sake? */ void* parent; + int prow; /* parent row */ + /* G10h4ck do we still need to keep this as void* instead of uint64_t for + * retroshare-gui sake? */ void* ref; + uint8_t type; RsPeerId id; std::string name; @@ -323,36 +329,18 @@ struct DirDetails : RsSerializable /// @see RsSerializable void serial_process(RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx) + RsGenericSerializer::SerializeContext& ctx) override { -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif // defined(__GNUC__) && !defined(__clang__) + /* Enforce serialization as uint64_t because void* changes size + * depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs) + */ + uint64_t handle = reinterpret_cast(ref); + RS_SERIAL_PROCESS(handle); + ref = reinterpret_cast(handle); - // (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always - // preferable to force type serialization to the correct size rather than letting the compiler choose for us. - // /!\ This structure cannot be sent over the network. The serialization would be inconsistent. - - if(sizeof(ref) == 4) - { - std::uint32_t& handle(reinterpret_cast(ref)); - RS_SERIAL_PROCESS(handle); - std::uint32_t& parentHandle(reinterpret_cast(parent)); - RS_SERIAL_PROCESS(parentHandle); - } - else if(sizeof(ref) == 8) - { - std::uint64_t& handle(reinterpret_cast(ref)); - RS_SERIAL_PROCESS(handle); - std::uint64_t& parentHandle(reinterpret_cast(parent)); - RS_SERIAL_PROCESS(parentHandle); - } - else - std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl; - -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif // defined(__GNUC__) && !defined(__clang__) + uint64_t parentHandle = reinterpret_cast(parent); + RS_SERIAL_PROCESS(parentHandle); + parent = reinterpret_cast(parentHandle); RS_SERIAL_PROCESS(prow); RS_SERIAL_PROCESS(type); From 37854c8f2325c1abe13d2beaacf81eefd720dbdd Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 4 Apr 2020 19:06:23 +0200 Subject: [PATCH 34/40] Fix compiler error on 32bit platforms --- retroshare-gui/src/gui/common/RsCollection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/common/RsCollection.cpp b/retroshare-gui/src/gui/common/RsCollection.cpp index ec9fec59b..6a78a1cf8 100644 --- a/retroshare-gui/src/gui/common/RsCollection.cpp +++ b/retroshare-gui/src/gui/common/RsCollection.cpp @@ -277,7 +277,7 @@ void RsCollection::recursAddElements( QDomDocument& doc, const RsFileTree& ft, uint32_t index, QDomElement& e ) const { - std::vector subdirs; + std::vector subdirs; std::vector subfiles ; std::string name; if(!ft.getDirectoryContent(name, subdirs, subfiles, index)) return; From f54fb046e48694840384e42b97e601a6d2a66977 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 12 Apr 2020 22:46:58 +0200 Subject: [PATCH 35/40] Use a a loop instead of std::trasform as suggested by Cyril --- libretroshare/src/ft/ftserver.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index d2c2f9f0a..3ae1af491 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -43,7 +43,7 @@ #include "retroshare/rstypes.h" #include "retroshare/rspeers.h" #include "retroshare/rsinit.h" - +#include "util/cxx17retrocompat.h" #include "rsitems/rsfiletransferitems.h" #include "rsitems/rsserviceids.h" #include "util/rsmemory.h" @@ -1975,9 +1975,8 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item) hasCallback = true; std::vector cRes; - std::transform( item->result.begin(), item->result.end(), - std::back_inserter(cRes), - [](const auto& it){ return TurtleFileInfoV2(it); } ); + for(auto& res: std::as_const(item->result)) + cRes.push_back(TurtleFileInfoV2(res)); cbpt->second.first(cRes); } From e406a1fb8bd53ea65d8acc5ef4ecfca93385186a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 20 Apr 2020 11:55:20 +0200 Subject: [PATCH 36/40] Fix JSON API compilation after rebasing on master --- libretroshare/src/gxs/rsgxsdataaccess.cc | 1 + .../src/retroshare/rsgxsifacetypes.h | 39 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index 0b23594c2..17c61f956 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1970,3 +1970,4 @@ bool RsGxsDataAccess::checkMsgFilter( } GxsGroupStatistic::~GxsGroupStatistic() = default; +GxsServiceStatistic::~GxsServiceStatistic() = default; diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index ad4979067..038906317 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -230,24 +230,13 @@ struct GxsGroupStatistic : RsSerializable ~GxsGroupStatistic() override; }; -class GxsServiceStatistic +struct GxsServiceStatistic : RsSerializable { -public: - GxsServiceStatistic() - { - mNumMsgs = 0; - mNumGrps = 0; - mSizeOfMsgs = 0; - mSizeOfGrps = 0; - mNumGrpsSubscribed = 0; - mNumThreadMsgsNew = 0; - mNumThreadMsgsUnread = 0; - mNumChildMsgsNew = 0; - mNumChildMsgsUnread = 0; - mSizeStore = 0; - } + GxsServiceStatistic() : + mNumMsgs(0), mNumGrps(0), mSizeOfMsgs(0), mSizeOfGrps(0), + mNumGrpsSubscribed(0), mNumThreadMsgsNew(0), mNumThreadMsgsUnread(0), + mNumChildMsgsNew(0), mNumChildMsgsUnread(0), mSizeStore(0) {} -public: uint32_t mNumMsgs; uint32_t mNumGrps; uint32_t mSizeOfMsgs; @@ -258,6 +247,24 @@ public: uint32_t mNumChildMsgsNew; uint32_t mNumChildMsgsUnread; uint32_t mSizeStore; + + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mNumMsgs); + RS_SERIAL_PROCESS(mNumGrps); + RS_SERIAL_PROCESS(mSizeOfMsgs); + RS_SERIAL_PROCESS(mSizeOfGrps); + RS_SERIAL_PROCESS(mNumGrpsSubscribed); + RS_SERIAL_PROCESS(mNumThreadMsgsNew); + RS_SERIAL_PROCESS(mNumThreadMsgsUnread); + RS_SERIAL_PROCESS(mNumChildMsgsNew); + RS_SERIAL_PROCESS(mNumChildMsgsUnread); + RS_SERIAL_PROCESS(mSizeStore); + } + + ~GxsServiceStatistic() override; }; class RS_DEPRECATED RsGxsGroupUpdateMeta From 3f38d0468008a41f17a550ed10f51fad35776e3e Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 20 Apr 2020 14:46:46 +0200 Subject: [PATCH 37/40] Clarify pointer ownership decorators documentation Use old char* in PGP functionas asked by Cyril --- libretroshare/src/pgp/pgphandler.cc | 2 +- libretroshare/src/pgp/pgphandler.h | 4 +++- libretroshare/src/util/rsmemory.h | 34 +++++++++++++++-------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/pgp/pgphandler.cc b/libretroshare/src/pgp/pgphandler.cc index 56b877e20..0638136b6 100644 --- a/libretroshare/src/pgp/pgphandler.cc +++ b/libretroshare/src/pgp/pgphandler.cc @@ -632,7 +632,7 @@ std::string PGPHandler::SaveCertificateToString(const RsPgpId& id,bool include_s bool PGPHandler::exportPublicKey( const RsPgpId& id, - rs_view_ptr& mem_block, size_t& mem_size, + unsigned char*& mem_block, size_t& mem_size, bool armoured, bool include_signatures ) const { mem_block = nullptr; mem_size = 0; // clear just in case diff --git a/libretroshare/src/pgp/pgphandler.h b/libretroshare/src/pgp/pgphandler.h index 2c58cc736..5d7eb82f8 100644 --- a/libretroshare/src/pgp/pgphandler.h +++ b/libretroshare/src/pgp/pgphandler.h @@ -107,8 +107,10 @@ public: bool LoadCertificateFromBinaryData(const unsigned char *bin_data,uint32_t bin_data_len, RsPgpId& gpg_id, std::string& error_string); std::string SaveCertificateToString(const RsPgpId& id,bool include_signatures) const ; + + /** The caller is in charge of freeing `mem` once finished */ bool exportPublicKey( const RsPgpId& id, - rs_view_ptr& mem,size_t& mem_size, + unsigned char*& mem, size_t& mem_size, bool armoured, bool include_signatures) const; bool parseSignature(unsigned char *sign, unsigned int signlen,RsPgpId& issuer_id) ; diff --git a/libretroshare/src/util/rsmemory.h b/libretroshare/src/util/rsmemory.h index 769163ca5..eb2889a6f 100644 --- a/libretroshare/src/util/rsmemory.h +++ b/libretroshare/src/util/rsmemory.h @@ -4,7 +4,7 @@ * libretroshare: retroshare core library * * * * Copyright 2012 Cyril Soler * - * Copyright 2019 Gioacchino Mazzurco * + * Copyright 2019-2020 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -89,19 +89,28 @@ bool rs_unique_cast( return false; } -/** Mark a pointer as non-owned aka you are not in charge of deleting it and - * must not delete it. +/** Mark a pointer as non-owned aka it must not be deleted/freed in that context. + * If a function take an `rs_view_ptr` as paramether it means that she will not + * own (aka free/delete) the passed memory, instead the caller is in charge of + * managing it. If a function return an `rs_view_ptr` it means the memory is + * managed internally and the caller should not call free/delete on it. * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ template using rs_view_ptr = T*; -/** Mark a pointer as owned aka you are in charge of deletingonce finished - * dealing with it. +/** Mark a pointer as owned aka the receiving context is in charge of dealing + * with it by free/delete once finished. + * If a function take an `rs_owner_ptr` as paramether it means that she will own + * (aka free/delete when finished using it) the passed memory, instead the + * caller is NOT in charge of managing it. + * If a function return an `rs_owner_ptr` it means the memory is NOT managed + * internally and the caller should call free/delete on it once finished using + * it. * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ template using rs_owner_ptr = T*; - void *rs_malloc(size_t size) ; +/** @deprecated use std::unique_ptr instead // This is a scope guard to release the memory block when going of of the current scope. // Can be very useful to auto-delete some memory on quit without the need to call free each time. // @@ -118,11 +127,11 @@ void *rs_malloc(size_t size) ; // [do something] // // } // mem gets freed automatically -// +*/ class RsTemporaryMemory { public: - RsTemporaryMemory(size_t s) + explicit RsTemporaryMemory(size_t s) { _mem = (unsigned char *)rs_malloc(s) ; @@ -136,14 +145,7 @@ public: size_t size() const { return _size ; } - ~RsTemporaryMemory() - { - if(_mem != NULL) - { - free(_mem) ; - _mem = NULL ; - } - } + ~RsTemporaryMemory() { free(_mem); } private: unsigned char *_mem ; From bd76917cac554d934df3aa385d5f6f8e2d7ba7e2 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 20 Apr 2020 15:16:20 +0200 Subject: [PATCH 38/40] GitlabCI print git information of checked out version --- build_scripts/GitlabCI/gitlabCI.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_scripts/GitlabCI/gitlabCI.Dockerfile b/build_scripts/GitlabCI/gitlabCI.Dockerfile index 87d064de1..aa9c6cc87 100644 --- a/build_scripts/GitlabCI/gitlabCI.Dockerfile +++ b/build_scripts/GitlabCI/gitlabCI.Dockerfile @@ -7,7 +7,8 @@ ARG REPO_BRANCH=master RUN \ cd RetroShare && git remote add testing $REPO_URL && \ git fetch --tags testing $REPO_BRANCH && \ - git reset --hard testing/$REPO_BRANCH + git reset --hard testing/$REPO_BRANCH && \ + git --no-pager log --max-count 1 RUN \ mkdir RetroShare-build && cd RetroShare-build && \ qmake ../RetroShare CONFIG+=no_retroshare_gui \ From 75082d7ca80a7c02948313541f4bd759256ffb83 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 20 Apr 2020 15:22:38 +0200 Subject: [PATCH 39/40] GitlabCI print merge requests arguments --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7a51f9c5..f19f15413 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ build-and-test: MR_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ; MR_ARGS="$MR_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ; export MR_ARGS ; + echo MR_ARGS=$MR_ARGS ; fi - mkdir Dockercontext - > From e2491b5f198ee6161117aba511f90454fe46e2a7 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 20 Apr 2020 15:46:11 +0200 Subject: [PATCH 40/40] GitlabCI specify repo url and branch also without merge request --- .gitlab-ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f19f15413..d620b3719 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,14 +12,17 @@ build-and-test: script: - > if [ -n "$CI_MERGE_REQUEST_ID" ]; then - MR_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ; - MR_ARGS="$MR_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ; - export MR_ARGS ; - echo MR_ARGS=$MR_ARGS ; - fi + REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ; + REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ; + else + REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ; + REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ; + fi ; + export REPO_ARGS ; + echo REPO_ARGS=$REPO_ARGS ; - mkdir Dockercontext - > - docker build -t retroshare:testing $MR_ARGS + docker build -t retroshare:testing $REPO_ARGS --file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile Dockercontext - >