mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-02-04 17:15:31 -05:00
added FileTree class to implement a compact representation of file hierarchies
This commit is contained in:
parent
847c1b2bce
commit
f98edd400e
@ -35,6 +35,8 @@
|
||||
|
||||
//#define DEBUG_DIRECTORY_STORAGE 1
|
||||
|
||||
typedef FileListIO::read_error read_error;
|
||||
|
||||
/******************************************************************************************************************/
|
||||
/* Internal File Hierarchy Storage */
|
||||
/******************************************************************************************************************/
|
||||
@ -1041,22 +1043,6 @@ bool InternalFileHierarchyStorage::save(const std::string& fname)
|
||||
}
|
||||
}
|
||||
|
||||
class read_error
|
||||
{
|
||||
public:
|
||||
read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag)
|
||||
{
|
||||
std::ostringstream s ;
|
||||
s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl;
|
||||
err_string = s.str();
|
||||
}
|
||||
read_error(const std::string& s) : err_string(s) {}
|
||||
|
||||
const std::string& what() const { return err_string ; }
|
||||
private:
|
||||
std::string err_string ;
|
||||
};
|
||||
|
||||
bool InternalFileHierarchyStorage::load(const std::string& fname)
|
||||
{
|
||||
unsigned char *buffer = NULL ;
|
||||
|
244
libretroshare/src/file_sharing/file_tree.cc
Normal file
244
libretroshare/src/file_sharing/file_tree.cc
Normal file
@ -0,0 +1,244 @@
|
||||
#include <iomanip>
|
||||
#include <util/radix64.h>
|
||||
|
||||
#include "file_sharing_defaults.h"
|
||||
#include "filelist_io.h"
|
||||
#include "file_tree.h"
|
||||
|
||||
std::string FileTreeImpl::toRadix64() const
|
||||
{
|
||||
unsigned char *buff = NULL ;
|
||||
uint32_t size = 0 ;
|
||||
|
||||
serialise(buff,size) ;
|
||||
|
||||
std::string res ;
|
||||
|
||||
Radix64::encode(buff,size,res) ;
|
||||
|
||||
free(buff) ;
|
||||
return res ;
|
||||
}
|
||||
|
||||
FileTree *FileTree::create(const std::string& radix64_string)
|
||||
{
|
||||
FileTreeImpl *ft = new FileTreeImpl ;
|
||||
|
||||
std::vector<uint8_t> mem = Radix64::decode(radix64_string);
|
||||
ft->deserialise(mem.data(),mem.size()) ;
|
||||
|
||||
return ft ;
|
||||
}
|
||||
|
||||
static void recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,void *ref)
|
||||
{
|
||||
}
|
||||
|
||||
FileTree *FileTree::create(void *ref)
|
||||
{
|
||||
FileTreeImpl *ft = new FileTreeImpl ;
|
||||
|
||||
recurs_buildFileTree(*ft,0,ref) ;
|
||||
|
||||
return ft ;
|
||||
}
|
||||
|
||||
typedef FileListIO::read_error read_error ;
|
||||
|
||||
bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size)
|
||||
{
|
||||
uint32_t buffer_offset = 0 ;
|
||||
|
||||
mTotalFiles = 0;
|
||||
mTotalSize = 0;
|
||||
|
||||
try
|
||||
{
|
||||
// Read some header
|
||||
|
||||
uint32_t version,n_dirs,n_files ;
|
||||
|
||||
if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION) ;
|
||||
if(version != (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001) throw std::runtime_error("Wrong version number") ;
|
||||
|
||||
if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_files)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_dirs)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
|
||||
// Write all file/dir entries
|
||||
|
||||
mFiles.resize(n_files) ;
|
||||
mDirs.resize(n_dirs) ;
|
||||
|
||||
unsigned char *node_section_data = NULL ;
|
||||
uint32_t node_section_size = 0 ;
|
||||
|
||||
for(uint32_t i=0;i<mFiles.size() && buffer_offset < buffer_size;++i) // only the 2nd condition really is needed. The first one ensures that the loop wont go forever.
|
||||
{
|
||||
uint32_t node_section_offset = 0 ;
|
||||
#ifdef DEBUG_DIRECTORY_STORAGE
|
||||
std::cerr << "reading node " << i << ", offset " << buffer_offset << " : " << RsUtil::BinToHex(&buffer[buffer_offset],std::min((int)buffer_size - (int)buffer_offset,100)) << "..." << std::endl;
|
||||
#endif
|
||||
|
||||
if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,node_section_data,node_section_size))
|
||||
{
|
||||
std::string file_name ;
|
||||
uint64_t file_size ;
|
||||
RsFileHash file_hash ;
|
||||
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,mFiles[i].name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ;
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,mFiles[i].size )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ) ;
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,mFiles[i].hash )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH) ;
|
||||
|
||||
mTotalFiles++ ;
|
||||
mTotalSize += file_size ;
|
||||
}
|
||||
else
|
||||
throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY) ;
|
||||
}
|
||||
|
||||
for(uint32_t i=0;i<mDirs.size() && buffer_offset < buffer_size;++i)
|
||||
{
|
||||
uint32_t node_section_offset = 0 ;
|
||||
|
||||
if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size))
|
||||
{
|
||||
DirData& de(mDirs[i]) ;
|
||||
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME,de.name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ;
|
||||
|
||||
uint32_t n_subdirs = 0 ;
|
||||
uint32_t n_subfiles = 0 ;
|
||||
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
|
||||
for(uint32_t j=0;j<n_subdirs;++j)
|
||||
{
|
||||
uint32_t di = 0 ;
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,di)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
de.subdirs.push_back(di) ;
|
||||
}
|
||||
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
|
||||
for(uint32_t j=0;j<n_subfiles;++j)
|
||||
{
|
||||
uint32_t fi = 0 ;
|
||||
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,fi)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ;
|
||||
de.subfiles.push_back(fi) ;
|
||||
}
|
||||
}
|
||||
else
|
||||
throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY) ;
|
||||
}
|
||||
free(node_section_data) ;
|
||||
|
||||
return true ;
|
||||
}
|
||||
catch(read_error& e)
|
||||
{
|
||||
#ifdef DEBUG_DIRECTORY_STORAGE
|
||||
std::cerr << "Error while reading: " << e.what() << std::endl;
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
return true ;
|
||||
}
|
||||
|
||||
bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) const
|
||||
{
|
||||
buffer = 0 ;
|
||||
uint32_t buffer_offset = 0 ;
|
||||
|
||||
unsigned char *tmp_section_data = (unsigned char*)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
|
||||
|
||||
if(!tmp_section_data)
|
||||
return false;
|
||||
|
||||
uint32_t tmp_section_size = FL_BASE_TMP_SECTION_SIZE ;
|
||||
|
||||
try
|
||||
{
|
||||
// Write some header
|
||||
|
||||
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,(uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001)) throw std::runtime_error("Write error") ;
|
||||
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mFiles.size())) throw std::runtime_error("Write error") ;
|
||||
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mDirs.size())) throw std::runtime_error("Write error") ;
|
||||
|
||||
// Write all file/dir entries
|
||||
|
||||
for(uint32_t i=0;i<mFiles.size();++i)
|
||||
{
|
||||
const FileData& fe(mFiles[i]) ;
|
||||
|
||||
uint32_t file_section_offset = 0 ;
|
||||
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.name )) throw std::runtime_error("Write error") ;
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.size )) throw std::runtime_error("Write error") ;
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,fe.hash )) throw std::runtime_error("Write error") ;
|
||||
|
||||
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,tmp_section_data,file_section_offset)) throw std::runtime_error("Write error") ;
|
||||
}
|
||||
|
||||
for(uint32_t i=0;i<mDirs.size();++i)
|
||||
{
|
||||
const DirData& de(mDirs[i]) ;
|
||||
|
||||
uint32_t dir_section_offset = 0 ;
|
||||
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.name )) throw std::runtime_error("Write error") ;
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs.size())) throw std::runtime_error("Write error") ;
|
||||
|
||||
for(uint32_t j=0;j<de.subdirs.size();++j)
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs[j])) throw std::runtime_error("Write error") ;
|
||||
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subfiles.size())) throw std::runtime_error("Write error") ;
|
||||
|
||||
for(uint32_t j=0;j<de.subfiles.size();++j)
|
||||
if(!FileListIO::writeField(tmp_section_data,tmp_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subfiles[j])) throw std::runtime_error("Write error") ;
|
||||
|
||||
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,tmp_section_data,dir_section_offset)) throw std::runtime_error("Write error") ;
|
||||
}
|
||||
|
||||
free(tmp_section_data) ;
|
||||
buffer_size = buffer_offset;
|
||||
|
||||
return true ;
|
||||
}
|
||||
catch(std::exception& e)
|
||||
{
|
||||
std::cerr << "Error while writing: " << e.what() << std::endl;
|
||||
|
||||
if(buffer != NULL)
|
||||
free(buffer) ;
|
||||
|
||||
if(tmp_section_data != NULL)
|
||||
free(tmp_section_data) ;
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void FileTreeImpl::print() const
|
||||
{
|
||||
recurs_print(0,"") ;
|
||||
}
|
||||
|
||||
void FileTreeImpl::recurs_print(uint32_t index,const std::string& indent) const
|
||||
{
|
||||
if(index >= mDirs.size())
|
||||
{
|
||||
std::cerr << "(EE) inconsistent FileTree structure" << std::endl;
|
||||
return;
|
||||
}
|
||||
std::cerr << indent << mDirs[index].name << std::endl;
|
||||
|
||||
for(uint32_t i=0;i<mDirs[index].subdirs.size();++i)
|
||||
recurs_print(mDirs[index].subdirs[i],indent+" ") ;
|
||||
|
||||
for(uint32_t i=0;i<mDirs[index].subfiles.size();++i)
|
||||
{
|
||||
const FileData& fd(mFiles[mDirs[index].subfiles[i]]) ;
|
||||
|
||||
std::cerr << indent << " " << fd.hash << " " << std::setprecision(8) << fd.size << " " << fd.name << std::endl;
|
||||
}
|
||||
}
|
25
libretroshare/src/file_sharing/file_tree.h
Normal file
25
libretroshare/src/file_sharing/file_tree.h
Normal file
@ -0,0 +1,25 @@
|
||||
#include "retroshare/rsfiles.h"
|
||||
|
||||
class FileTreeImpl: public FileTree
|
||||
{
|
||||
public:
|
||||
FileTreeImpl() {}
|
||||
|
||||
virtual std::string toRadix64() const ;
|
||||
virtual bool getDirectoryContent(uint32_t index,std::vector<uint32_t>& subdirs,std::vector<FileData>& subfiles) const ;
|
||||
virtual void print() const ;
|
||||
|
||||
bool serialise(unsigned char *& data,uint32_t& data_size) const ;
|
||||
bool deserialise(unsigned char* data, uint32_t data_size) ;
|
||||
|
||||
protected:
|
||||
void recurs_print(uint32_t index,const std::string& indent) const;
|
||||
|
||||
struct DirData {
|
||||
std::string name;
|
||||
std::vector<uint32_t> subdirs ;
|
||||
std::vector<uint32_t> subfiles ;
|
||||
};
|
||||
std::vector<FileData> mFiles ;
|
||||
std::vector<DirData> mDirs ;
|
||||
};
|
@ -22,12 +22,22 @@
|
||||
* Please report all bugs and problems to "retroshare.project@gmail.com".
|
||||
*
|
||||
*/
|
||||
|
||||
#include <sstream>
|
||||
#include "retroshare/rsids.h"
|
||||
#include "pqi/authssl.h"
|
||||
#include "util/rsdir.h"
|
||||
#include "util/rsprint.h"
|
||||
#include "serialiser/rsbaseserial.h"
|
||||
#include "filelist_io.h"
|
||||
|
||||
FileListIO::read_error::read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag)
|
||||
{
|
||||
std::ostringstream s ;
|
||||
s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl;
|
||||
err_string = s.str();
|
||||
}
|
||||
|
||||
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; }
|
||||
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; }
|
||||
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; }
|
||||
|
@ -36,6 +36,7 @@
|
||||
// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers.
|
||||
|
||||
static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ;
|
||||
static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001 = 0x00010001 ;
|
||||
|
||||
static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ;
|
||||
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ;
|
||||
@ -65,6 +66,7 @@ static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ;
|
||||
|
||||
static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes)
|
||||
|
||||
|
||||
class FileListIO
|
||||
{
|
||||
public:
|
||||
@ -93,7 +95,19 @@ public:
|
||||
return deserialise(buff,buff_size,offset,val);
|
||||
}
|
||||
|
||||
static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ;
|
||||
class read_error
|
||||
{
|
||||
public:
|
||||
read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag);
|
||||
read_error(const std::string& s) : err_string(s) {}
|
||||
|
||||
const std::string& what() const { return err_string ; }
|
||||
private:
|
||||
std::string err_string ;
|
||||
};
|
||||
|
||||
|
||||
static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ;
|
||||
static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ;
|
||||
|
||||
template<class T> static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ;
|
||||
|
@ -47,6 +47,7 @@ file_lists {
|
||||
file_sharing/directory_updater.h \
|
||||
file_sharing/rsfilelistitems.h \
|
||||
file_sharing/dir_hierarchy.h \
|
||||
file_sharing/file_tree.h \
|
||||
file_sharing/file_sharing_defaults.h
|
||||
|
||||
SOURCES *= file_sharing/p3filelists.cc \
|
||||
@ -55,6 +56,7 @@ file_lists {
|
||||
file_sharing/directory_storage.cc \
|
||||
file_sharing/directory_updater.cc \
|
||||
file_sharing/dir_hierarchy.cc \
|
||||
file_sharing/file_tree.cc \
|
||||
file_sharing/rsfilelistitems.cc
|
||||
}
|
||||
|
||||
|
@ -134,6 +134,36 @@ struct SharedDirStats
|
||||
uint64_t total_shared_size ;
|
||||
};
|
||||
|
||||
// This class represents a tree of directories and files, only with their names size and hash. It is used to create collection links in the GUI
|
||||
// and to transmit directory information between services. This class is independent from the existing FileHierarchy classes used in storage because
|
||||
// we need a very copact serialization and storage size since we create links with it. Besides, we cannot afford to risk the leak of other local information
|
||||
// by using the orignal classes.
|
||||
|
||||
class FileTree
|
||||
{
|
||||
public:
|
||||
virtual ~FileTree() {}
|
||||
|
||||
static FileTree *create(void *ref) ;
|
||||
static FileTree *create(const std::string& radix64_string) ;
|
||||
|
||||
virtual std::string toRadix64() const =0 ;
|
||||
|
||||
// These methods allow the user to browse the hierarchy
|
||||
|
||||
struct FileData {
|
||||
std::string name ;
|
||||
uint64_t size ;
|
||||
RsFileHash hash ;
|
||||
};
|
||||
|
||||
virtual uint32_t root() const { return 0;}
|
||||
virtual bool getDirectoryContent(uint32_t index,std::vector<uint32_t>& subdirs,std::vector<FileData>& subfiles) const = 0;
|
||||
|
||||
uint32_t mTotalFiles ;
|
||||
uint64_t mTotalSize ;
|
||||
};
|
||||
|
||||
class RsFiles
|
||||
{
|
||||
public:
|
||||
|
Loading…
x
Reference in New Issue
Block a user