Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-27 00:19:25 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

added load/save of directory hierarchies. Fixed bug in loading of hash cache (caused y non initialisation of dest string in GetRawString())

Browse Source
This commit is contained in:
mr-alice 2016-09-05 21:22:11 +02:00
parent 5b221c56db
commit 679c53b0b0
6 changed files with 229 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
libretroshare/src/file_sharing/dir_hierarchy.cc
View File

@ -1,4 +1,5 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#define DEBUG_DIRECTORY_STORAGE 1
@ -235,7 +236,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
d.most_recent_time = most_recent_time;
d.dir_most_recent_time = most_recent_time;
d.dir_modtime = dir_modtime;
d.dir_update_time = time(NULL);
d.dir_name = dir_name;
@ -297,7 +298,7 @@ bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryI
DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
recurs_max_modf_TS = d.most_recent_time ;
recurs_max_modf_TS = d.dir_most_recent_time ;
local_update_TS = d.dir_update_time ;
return true;
@ -312,7 +313,7 @@ bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryI
DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
d.most_recent_time = recurs_max_modf_TS ;
d.dir_most_recent_time = recurs_max_modf_TS ;
d.dir_update_time = local_update_TS ;
return true;
@ -332,7 +333,7 @@ time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectorySto
for(uint32_t i=0;i<d.subdirs.size();++i)
largest_modf_time = std::max(largest_modf_time,recursUpdateLastModfTime(d.subdirs[i])) ;
d.most_recent_time = largest_modf_time ;
d.dir_most_recent_time = largest_modf_time ;
return largest_modf_time ;
}
@ -502,7 +503,7 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[node]));
std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ;
std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ;
for(uint32_t i=0;i<d.subdirs.size();++i)
std::cerr << d.subdirs[i] << " " ;
@ -547,13 +548,199 @@ bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::Entry
bool InternalFileHierarchyStorage::save(const std::string& fname)
{
NOT_IMPLEMENTED();
return true;
unsigned char *buffer = NULL ;
uint32_t buffer_size = 0 ;
uint32_t buffer_offset = 0 ;
try
{
// Write some header
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,(uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001)) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mNodes.size())) throw std::runtime_error("Write error") ;
// Write all file/dir entries
for(uint32_t i=0;i<mNodes.size();++i)
if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_FILE)
{
const FileEntry& fe(*static_cast<const FileEntry*>(mNodes[i])) ;
unsigned char *file_section_data = NULL ;
uint32_t file_section_offset = 0 ;
uint32_t file_section_size = 0;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.file_name )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.file_size )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,fe.file_hash )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)fe.file_modtime)) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,file_section_data,file_section_offset)) throw std::runtime_error("Write error") ;
free(file_section_data) ;
}
else if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR)
{
const DirEntry& de(*static_cast<const DirEntry*>(mNodes[i])) ;
unsigned char *dir_section_data = NULL ;
uint32_t dir_section_offset = 0 ;
uint32_t dir_section_size = 0;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.dir_name )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,de.dir_parent_path )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)de.dir_modtime )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,(uint32_t)de.dir_update_time )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)de.dir_most_recent_time )) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs.size())) throw std::runtime_error("Write error") ;
for(uint32_t j=0;j<de.subdirs.size();++j)
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs[j])) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subfiles.size())) throw std::runtime_error("Write error") ;
for(uint32_t j=0;j<de.subfiles.size();++j)
if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subfiles[j])) throw std::runtime_error("Write error") ;
if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,dir_section_data,dir_section_offset)) throw std::runtime_error("Write error") ;
free(dir_section_data) ;
}
bool res = FileListIO::saveEncryptedDataToFile(fname,buffer,buffer_offset) ;
free(buffer) ;
return res ;
}
catch(std::exception& e)
{
std::cerr << "Error while writing: " << e.what() << std::endl;
if(buffer != NULL)
free(buffer) ;
return false;
}
}
bool InternalFileHierarchyStorage::load(const std::string& fname)
{
NOT_IMPLEMENTED();
return true;
unsigned char *buffer = NULL ;
uint32_t buffer_size = 0 ;
uint32_t buffer_offset = 0 ;
try
{
if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_offset) )
throw std::runtime_error("Read error") ;
// Read some header
uint32_t version, n_nodes ;
if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw std::runtime_error("Read error") ;
if(version != (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001) throw std::runtime_error("Wrong version number") ;
if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_nodes)) throw std::runtime_error("Read error") ;
// Write all file/dir entries
for(uint32_t i=0;i<mNodes.size();++i)
if(mNodes[i])
delete mNodes[i] ;
mNodes.clear();
mNodes.resize(n_nodes,NULL) ;
for(uint32_t i=0;i<mNodes.size();++i)
{
unsigned char *node_section_data = NULL ;
uint32_t node_section_size = 0 ;
uint32_t node_section_offset = 0 ;
if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,node_section_data,node_section_size))
{
uint32_t node_index ;
std::string file_name ;
uint64_t file_size ;
RsFileHash file_hash ;
uint32_t file_modtime ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Write error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Write error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Write error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Write error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Write error") ;
if(node_index >= mNodes.size())
mNodes.resize(node_index+1,NULL) ;
mNodes[node_index] = new FileEntry(file_name,file_size,file_modtime,file_hash);
}
else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size))
{
uint32_t node_index ;
std::string dir_name ;
std::string dir_parent_path ;
uint32_t dir_modtime ;
uint32_t dir_update_time ;
uint64_t dir_most_recent_time ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw std::runtime_error("Read error") ;
if(node_index >= mNodes.size())
mNodes.resize(node_index+1,NULL) ;
DirEntry *de = new DirEntry(dir_name) ;
de->dir_name = dir_name ;
de->dir_parent_path = dir_parent_path ;
de->dir_modtime = dir_modtime ;
de->dir_update_time = dir_update_time ;
de->dir_most_recent_time = dir_most_recent_time ;
uint32_t n_subdirs = 0 ;
uint32_t n_subfiles = 0 ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw std::runtime_error("Read error") ;
for(uint32_t j=0;j<n_subdirs;++j)
{
uint32_t di = 0 ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,di)) throw std::runtime_error("Read error") ;
de->subdirs.push_back(di) ;
}
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw std::runtime_error("Read error") ;
for(uint32_t j=0;j<n_subfiles;++j)
{
uint32_t fi = 0 ;
if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,fi)) throw std::runtime_error("Read error") ;
de->subfiles.push_back(fi) ;
}
}
else
throw std::runtime_error("Unknown node section.") ;
free(node_section_data) ;
}
free(buffer) ;
return true ;
}
catch(std::exception& e)
{
std::cerr << "Error while writing: " << e.what() << std::endl;
if(buffer != NULL)
free(buffer) ;
return false;
}
}

6
libretroshare/src/file_sharing/dir_hierarchy.h
View File

@ -26,6 +26,8 @@ public:
{
public:
FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {}
FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {}
virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
virtual ~FileEntry() {}
@ -39,7 +41,7 @@ public:
class DirEntry: public FileStorageNode
{
public:
DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {}
DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {}
virtual ~DirEntry() {}
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
@ -52,7 +54,7 @@ public:
std::vector<DirectoryStorage::EntryIndex> subfiles ;
time_t dir_modtime;
time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below.
time_t dir_most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below.
time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info.
};

4
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -229,7 +229,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
d.type = DIR_TYPE_DIR;
d.hash.clear() ;
d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size();
d.min_age = now - dir_entry->most_recent_time ;
d.min_age = now - dir_entry->dir_most_recent_time ;
d.name = dir_entry->dir_name;
d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ;
d.parent = (void*)(intptr_t)dir_entry->parent_index ;
@ -487,7 +487,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
//
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->most_recent_time)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ;
// serialise number of subdirs and number of subfiles

35
libretroshare/src/file_sharing/filelist_io.h
View File

@ -9,20 +9,25 @@
// This file implements load/save of various fields used for file lists and directory content.
// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers.
static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ;
static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ;
static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ;
static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ;
static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ;
static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ;
static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ;
static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ;
static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ;
static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ;
static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ;
static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ;
static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ;
static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ;
static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ;
static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ;
static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ;
static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ;
static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ;
static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes)
@ -94,11 +99,13 @@ private:
if(offset + 1 > buff_size)
return false ;
uint8_t section_tag = buff[offset++] ;
uint8_t section_tag = buff[offset] ; // we do the offset++ after, only if the header can be read. Doing so, we can make multiple read attempts.
if(section_tag != check_section_tag)
return false;
offset++ ;
return read125Size(buff,buff_size,offset,S) ;
}
};

3
libretroshare/src/file_sharing/hash_cache.cc
View File

@ -1,4 +1,5 @@
#include "util/rsdir.h"
#include "util/rsprint.h"
#include "rsserver/p3face.h"
#include "pqi/authssl.h"
#include "hash_cache.h"
@ -299,6 +300,8 @@ bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_si
if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ;
std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl;
std::cerr << "Info.filename = " << info.filename << std::endl;
free(section_data) ;
return true;

3
libretroshare/src/serialiser/rsbaseserial.cc
View File

@ -231,6 +231,9 @@ uint32_t getRawStringSize(const std::string &outStr)
bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr)
{
#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect.
outStr.clear();
uint32_t len = 0;
if (!getRawUInt32(data, size, offset, &len))
{