Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #573 from csoler/v0.6-FileListsOptim

Browse Source 
V0.6 file lists optim
This commit is contained in:
Cyril Soler 2016-11-17 22:10:57 +01:00 committed by GitHub
commit 6d0293cc45
5 changed files with 81 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -82,11 +82,14 @@ std::string DirectoryStorage::DirIterator::name() const { const InternalFil
/* Directory Storage */
/******************************************************************************************************************/
DirectoryStorage::DirectoryStorage(const RsPeerId &pid)
: mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString())
DirectoryStorage::DirectoryStorage(const RsPeerId &pid,const std::string& fname)
: mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString()),mLastSavedTime(0),mChanged(false),mFileName(fname)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy = new InternalFileHierarchyStorage();
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy = new InternalFileHierarchyStorage();
}
load(fname) ;
}
DirectoryStorage::EntryIndex DirectoryStorage::root() const
@ -131,22 +134,22 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs,hash_salt) ;
locked_check() ;
mChanged = true ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
locked_check() ;
mChanged = true ;
return res ;
}
bool DirectoryStorage::removeDirectory(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->removeDirectory(indx);
mChanged = true ;
locked_check();
return res ;
}
@ -160,11 +163,13 @@ void DirectoryStorage::locked_check()
bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mChanged = true ;
return mFileHierarchy->updateFile(index,hash,fname,size,modf_time);
}
bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mChanged = true ;
return mFileHierarchy->updateHash(index,hash);
}
@ -177,6 +182,7 @@ void DirectoryStorage::getStatistics(SharedDirStats& stats)
bool DirectoryStorage::load(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mChanged = false ;
return mFileHierarchy->load(local_file_name);
}
void DirectoryStorage::save(const std::string& local_file_name)
@ -289,6 +295,19 @@ bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& in
return mFileHierarchy->getIndexFromDirHash(hash,index) ;
}
void DirectoryStorage::checkSave()
{
time_t now = time(NULL);
if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
{
locked_check();
save(mFileName);
mLastSavedTime = now ;
mChanged = false ;
}
}
/******************************************************************************************************************/
/* Local Directory Storage */
/******************************************************************************************************************/
@ -753,15 +772,14 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
return true ;
}
/******************************************************************************************************************/
/* Remote Directory Storage */
/******************************************************************************************************************/
RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname)
: DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname)
: DirectoryStorage(pid,fname)
{
load(fname) ;
mLastSweepTime = time(NULL) - (RSRandom::random_u32() % DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP) ;
std::cerr << "Loaded remote directory for peer " << pid << ", inited last sweep time to " << time(NULL) - mLastSweepTime << " secs ago." << std::endl;
@ -770,18 +788,6 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st
#endif
}
void RemoteDirectoryStorage::checkSave()
{
time_t now = time(NULL);
if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
{
save(mFileName);
mLastSavedTime = now ;
mChanged = false ;
}
}
bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata)
{
const unsigned char *section_data = (unsigned char*)bindata.bin_data ;

46
libretroshare/src/file_sharing/directory_storage.h
View File

@ -41,7 +41,7 @@ class RsTlvBinaryData ;
class DirectoryStorage
{
public:
DirectoryStorage(const RsPeerId& pid) ;
DirectoryStorage(const RsPeerId& pid, const std::string& fname) ;
virtual ~DirectoryStorage() {}
typedef uint32_t EntryIndex ;
@ -156,6 +156,14 @@ class DirectoryStorage
void print();
void cleanup();
/*!
* \brief checkSave
* Checks the time of last saving, last modification time, and saves if needed.
*/
void checkSave() ;
const std::string& filename() const { return mFileName ; }
protected:
bool load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;
@ -173,6 +181,10 @@ class DirectoryStorage
mutable RsMutex mDirStorageMtx ;
InternalFileHierarchyStorage *mFileHierarchy ;
time_t mLastSavedTime ;
bool mChanged ;
std::string mFileName;
};
class RemoteDirectoryStorage: public DirectoryStorage
@ -192,31 +204,20 @@ public:
*/
bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ;
/*!
* \brief checkSave
* Checks the time of last saving, last modification time, and saves if needed.
*/
void checkSave() ;
/*!
* \brief lastSweepTime
* returns the last time a sweep has been done over the directory in order to check update TS.
* \return
*/
time_t& lastSweepTime() { return mLastSweepTime ; }
const std::string& filename() const { return mFileName ; }
private:
time_t mLastSavedTime ;
time_t mLastSweepTime ;
bool mChanged ;
std::string mFileName;
};
class LocalDirectoryStorage: public DirectoryStorage
{
public:
LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {}
LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id,fname) {}
virtual ~LocalDirectoryStorage() {}
/*!
@ -289,19 +290,18 @@ public:
bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ;
private:
static RsFileHash makeEncryptedHash(const RsFileHash& hash);
bool locked_findRealHash(const RsFileHash& hash, RsFileHash& real_hash) const;
std::string locked_getVirtualPath(EntryIndex indx) const ;
std::string locked_getVirtualDirName(EntryIndex indx) const ;
static RsFileHash makeEncryptedHash(const RsFileHash& hash);
bool locked_findRealHash(const RsFileHash& hash, RsFileHash& real_hash) const;
std::string locked_getVirtualPath(EntryIndex indx) const ;
std::string locked_getVirtualDirName(EntryIndex indx) const ;
bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId>& parent_groups);
std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const;
bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId>& parent_groups);
std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const;
std::map<std::string,SharedDirInfo> mLocalDirs ; // map is better for search. it->first=it->second.filename
std::map<RsFileHash,RsFileHash> mEncryptedHashes; // map such that hash(it->second) = it->first
std::string mFileName;
std::map<std::string,SharedDirInfo> mLocalDirs ; // map is better for search. it->first=it->second.filename
std::map<RsFileHash,RsFileHash> mEncryptedHashes; // map such that hash(it->second) = it->first
bool mTSChanged ;
bool mTSChanged ;
};

5
libretroshare/src/file_sharing/file_sharing_defaults.h
View File

@ -39,8 +39,9 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
static const std::string LOCAL_SHARED_DIRS_FILE_NAME = "local_dir_hierarchy.bin" ; // hard-coded directory name to store encrypted local dir hierarchy.
static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs.
static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this

8
libretroshare/src/file_sharing/p3filelists.cc
View File

@ -66,7 +66,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
mRemoteDirectories.clear() ; // we should load them!
mOwnId = mpeers->getOwnId() ;
mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId);
mLocalSharedDirs = new LocalDirectoryStorage(mFileSharingDir + "/" + LOCAL_SHARED_DIRS_FILE_NAME,mOwnId);
mHashCache = new HashStorage(mFileSharingDir + "/" + HASH_CACHE_FILE_NAME) ;
mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ;
@ -219,6 +219,7 @@ int p3FileDatabase::tick()
}
mLastRemoteDirSweepTS = now;
mLocalSharedDirs->checkSave() ;
// This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up,
// so the first notify is ignored, and no other notify will happen next.
@ -1447,7 +1448,10 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem)
RS_STACK_MUTEX(mFLSMtx) ;
if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data))
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed
{
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed
mRemoteDirectories[fi]->lastSweepTime() = time(NULL) - DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP + 10 ; // force re-sweep in 10 secs, so as to fasten updated
}
else
P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl;

36
retroshare-gui/src/gui/RemoteDirModel.cpp
View File

@ -622,9 +622,29 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const
if (!index.isValid())
return QVariant();
// First we take care of the cases that do not require requestDirDetails()
int coln = index.column();
if (role == Qt::SizeHintRole)
return QVariant();
if (role == Qt::TextAlignmentRole)
{
if(coln == 1)
return int( Qt::AlignRight | Qt::AlignVCenter);
else
return QVariant();
}
// This makes sorting a bit arbitrary, but prevents calling requestDirDetails(). The number of calls to sortRole is
// indeed sometimes very high and somewhat kills the GUI responsivness.
//
//if (role == SortRole)
// return QVariant(index.row()) ;
/* get the data from the index */
void *ref = index.internalPointer();
int coln = index.column();
DirDetails details ;
@ -674,20 +694,6 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const
Qt::SizeHintRole
****************/
if (role == Qt::SizeHintRole)
{
return QVariant(); // Use standard
} /* end of SizeHintRole */
if (role == Qt::TextAlignmentRole)
{
if(coln == 1)
{
return int( Qt::AlignRight | Qt::AlignVCenter);
}
return QVariant();
} /* end of TextAlignmentRole */
if (role == Qt::DisplayRole)
return displayRole(details,coln) ;