From f98edd400e9c3092bcb04f785c2683cc5da4488b Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 18 Oct 2017 00:04:04 +0200 Subject: [PATCH] added FileTree class to implement a compact representation of file hierarchies --- .../src/file_sharing/dir_hierarchy.cc | 18 +- libretroshare/src/file_sharing/file_tree.cc | 244 ++++++++++++++++++ libretroshare/src/file_sharing/file_tree.h | 25 ++ libretroshare/src/file_sharing/filelist_io.cc | 10 + libretroshare/src/file_sharing/filelist_io.h | 16 +- libretroshare/src/libretroshare.pro | 2 + libretroshare/src/retroshare/rsfiles.h | 30 +++ 7 files changed, 328 insertions(+), 17 deletions(-) create mode 100644 libretroshare/src/file_sharing/file_tree.cc create mode 100644 libretroshare/src/file_sharing/file_tree.h diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index bc8faf0e9..9ec6d2088 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -35,6 +35,8 @@ //#define DEBUG_DIRECTORY_STORAGE 1 +typedef FileListIO::read_error read_error; + /******************************************************************************************************************/ /* Internal File Hierarchy Storage */ /******************************************************************************************************************/ @@ -1041,22 +1043,6 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) } } -class read_error -{ -public: - read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag) - { - std::ostringstream s ; - s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl; - err_string = s.str(); - } - read_error(const std::string& s) : err_string(s) {} - - const std::string& what() const { return err_string ; } -private: - std::string err_string ; -}; - bool InternalFileHierarchyStorage::load(const std::string& fname) { unsigned char *buffer = NULL ; diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc new file mode 100644 index 000000000..59987553c --- /dev/null +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -0,0 +1,244 @@ +#include +#include + +#include "file_sharing_defaults.h" +#include "filelist_io.h" +#include "file_tree.h" + +std::string FileTreeImpl::toRadix64() const +{ + unsigned char *buff = NULL ; + uint32_t size = 0 ; + + serialise(buff,size) ; + + std::string res ; + + Radix64::encode(buff,size,res) ; + + free(buff) ; + return res ; +} + +FileTree *FileTree::create(const std::string& radix64_string) +{ + FileTreeImpl *ft = new FileTreeImpl ; + + std::vector mem = Radix64::decode(radix64_string); + ft->deserialise(mem.data(),mem.size()) ; + + return ft ; +} + +static void recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,void *ref) +{ +} + +FileTree *FileTree::create(void *ref) +{ + FileTreeImpl *ft = new FileTreeImpl ; + + recurs_buildFileTree(*ft,0,ref) ; + + return ft ; +} + +typedef FileListIO::read_error read_error ; + +bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size) +{ + uint32_t buffer_offset = 0 ; + + mTotalFiles = 0; + mTotalSize = 0; + + try + { + // Read some header + + uint32_t version,n_dirs,n_files ; + + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION) ; + if(version != (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001) throw std::runtime_error("Wrong version number") ; + + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_files)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_dirs)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; + + // Write all file/dir entries + + mFiles.resize(n_files) ; + mDirs.resize(n_dirs) ; + + unsigned char *node_section_data = NULL ; + uint32_t node_section_size = 0 ; + + for(uint32_t i=0;i= mDirs.size()) + { + std::cerr << "(EE) inconsistent FileTree structure" << std::endl; + return; + } + std::cerr << indent << mDirs[index].name << std::endl; + + for(uint32_t i=0;i& subdirs,std::vector& subfiles) const ; + virtual void print() const ; + + bool serialise(unsigned char *& data,uint32_t& data_size) const ; + bool deserialise(unsigned char* data, uint32_t data_size) ; + +protected: + void recurs_print(uint32_t index,const std::string& indent) const; + + struct DirData { + std::string name; + std::vector subdirs ; + std::vector subfiles ; + }; + std::vector mFiles ; + std::vector mDirs ; +}; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 618d6c6bb..15d4d1b53 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -22,12 +22,22 @@ * Please report all bugs and problems to "retroshare.project@gmail.com". * */ + +#include #include "retroshare/rsids.h" #include "pqi/authssl.h" #include "util/rsdir.h" +#include "util/rsprint.h" #include "serialiser/rsbaseserial.h" #include "filelist_io.h" +FileListIO::read_error::read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag) +{ + std::ostringstream s ; + s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl; + err_string = s.str(); +} + template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; } diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 0fcdb00fb..8eccd6b27 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -36,6 +36,7 @@ // WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ; +static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001 = 0x00010001 ; static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ; @@ -65,6 +66,7 @@ static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) + class FileListIO { public: @@ -93,7 +95,19 @@ public: return deserialise(buff,buff_size,offset,val); } - static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; + class read_error + { + public: + read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag); + read_error(const std::string& s) : err_string(s) {} + + const std::string& what() const { return err_string ; } + private: + std::string err_string ; + }; + + + static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ; template static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index d96e7dc7f..0cb982ee6 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -47,6 +47,7 @@ file_lists { file_sharing/directory_updater.h \ file_sharing/rsfilelistitems.h \ file_sharing/dir_hierarchy.h \ + file_sharing/file_tree.h \ file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ @@ -55,6 +56,7 @@ file_lists { file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ file_sharing/dir_hierarchy.cc \ + file_sharing/file_tree.cc \ file_sharing/rsfilelistitems.cc } diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 313ca26eb..22a7dd8bf 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -134,6 +134,36 @@ struct SharedDirStats uint64_t total_shared_size ; }; +// This class represents a tree of directories and files, only with their names size and hash. It is used to create collection links in the GUI +// and to transmit directory information between services. This class is independent from the existing FileHierarchy classes used in storage because +// we need a very copact serialization and storage size since we create links with it. Besides, we cannot afford to risk the leak of other local information +// by using the orignal classes. + +class FileTree +{ +public: + virtual ~FileTree() {} + + static FileTree *create(void *ref) ; + static FileTree *create(const std::string& radix64_string) ; + + virtual std::string toRadix64() const =0 ; + + // These methods allow the user to browse the hierarchy + + struct FileData { + std::string name ; + uint64_t size ; + RsFileHash hash ; + }; + + virtual uint32_t root() const { return 0;} + virtual bool getDirectoryContent(uint32_t index,std::vector& subdirs,std::vector& subfiles) const = 0; + + uint32_t mTotalFiles ; + uint64_t mTotalSize ; +}; + class RsFiles { public: