From 36755c40927186827af92b80640ae45e19217e59 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 17 Nov 2016 19:03:53 +0100 Subject: [PATCH 1/4] added load/save of own file hierarchy --- .../src/file_sharing/directory_storage.cc | 49 ++++++++++--------- .../src/file_sharing/directory_storage.h | 46 ++++++++--------- .../src/file_sharing/file_sharing_defaults.h | 5 +- libretroshare/src/file_sharing/p3filelists.cc | 3 +- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index be439f464..e7b6e0fd9 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -82,11 +82,14 @@ std::string DirectoryStorage::DirIterator::name() const { const InternalFil /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirectoryStorage(const RsPeerId &pid) - : mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString()) +DirectoryStorage::DirectoryStorage(const RsPeerId &pid,const std::string& fname) + : mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString()),mLastSavedTime(0),mChanged(false),mFileName(fname) { - RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy = new InternalFileHierarchyStorage(); + { + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy = new InternalFileHierarchyStorage(); + } + load(fname) ; } DirectoryStorage::EntryIndex DirectoryStorage::root() const @@ -131,14 +134,14 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std:: { RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs,hash_salt) ; - locked_check() ; + mChanged = true ; return res ; } bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; - locked_check() ; + mChanged = true ; return res ; } bool DirectoryStorage::removeDirectory(const EntryIndex& indx) @@ -146,7 +149,6 @@ bool DirectoryStorage::removeDirectory(const EntryIndex& indx) RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->removeDirectory(indx); - locked_check(); return res ; } @@ -160,11 +162,13 @@ void DirectoryStorage::locked_check() bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) { RS_STACK_MUTEX(mDirStorageMtx) ; + mChanged = true ; return mFileHierarchy->updateFile(index,hash,fname,size,modf_time); } bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash) { RS_STACK_MUTEX(mDirStorageMtx) ; + mChanged = true ; return mFileHierarchy->updateHash(index,hash); } @@ -177,6 +181,7 @@ void DirectoryStorage::getStatistics(SharedDirStats& stats) bool DirectoryStorage::load(const std::string& local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; + mChanged = false ; return mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) @@ -289,6 +294,19 @@ bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& in return mFileHierarchy->getIndexFromDirHash(hash,index) ; } +void DirectoryStorage::checkSave() +{ + time_t now = time(NULL); + + if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now) + { + locked_check(); + + save(mFileName); + mLastSavedTime = now ; + mChanged = false ; + } +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -753,15 +771,14 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary return true ; } + /******************************************************************************************************************/ /* Remote Directory Storage */ /******************************************************************************************************************/ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) - : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname) + : DirectoryStorage(pid,fname) { - load(fname) ; - mLastSweepTime = time(NULL) - (RSRandom::random_u32() % DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP) ; std::cerr << "Loaded remote directory for peer " << pid << ", inited last sweep time to " << time(NULL) - mLastSweepTime << " secs ago." << std::endl; @@ -770,18 +787,6 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st #endif } -void RemoteDirectoryStorage::checkSave() -{ - time_t now = time(NULL); - - if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now) - { - save(mFileName); - mLastSavedTime = now ; - mChanged = false ; - } -} - bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) { const unsigned char *section_data = (unsigned char*)bindata.bin_data ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index b7b64ed96..5f46867f6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -41,7 +41,7 @@ class RsTlvBinaryData ; class DirectoryStorage { public: - DirectoryStorage(const RsPeerId& pid) ; + DirectoryStorage(const RsPeerId& pid, const std::string& fname) ; virtual ~DirectoryStorage() {} typedef uint32_t EntryIndex ; @@ -156,6 +156,14 @@ class DirectoryStorage void print(); void cleanup(); + /*! + * \brief checkSave + * Checks the time of last saving, last modification time, and saves if needed. + */ + void checkSave() ; + + const std::string& filename() const { return mFileName ; } + protected: bool load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; @@ -173,6 +181,10 @@ class DirectoryStorage mutable RsMutex mDirStorageMtx ; InternalFileHierarchyStorage *mFileHierarchy ; + + time_t mLastSavedTime ; + bool mChanged ; + std::string mFileName; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -192,31 +204,20 @@ public: */ bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; - /*! - * \brief checkSave - * Checks the time of last saving, last modification time, and saves if needed. - */ - void checkSave() ; - /*! * \brief lastSweepTime * returns the last time a sweep has been done over the directory in order to check update TS. * \return */ time_t& lastSweepTime() { return mLastSweepTime ; } - - const std::string& filename() const { return mFileName ; } private: - time_t mLastSavedTime ; time_t mLastSweepTime ; - bool mChanged ; - std::string mFileName; }; class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {} + LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id,fname) {} virtual ~LocalDirectoryStorage() {} /*! @@ -289,19 +290,18 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: - static RsFileHash makeEncryptedHash(const RsFileHash& hash); - bool locked_findRealHash(const RsFileHash& hash, RsFileHash& real_hash) const; - std::string locked_getVirtualPath(EntryIndex indx) const ; - std::string locked_getVirtualDirName(EntryIndex indx) const ; + static RsFileHash makeEncryptedHash(const RsFileHash& hash); + bool locked_findRealHash(const RsFileHash& hash, RsFileHash& real_hash) const; + std::string locked_getVirtualPath(EntryIndex indx) const ; + std::string locked_getVirtualDirName(EntryIndex indx) const ; - bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); - std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; - std::map mLocalDirs ; // map is better for search. it->first=it->second.filename - std::map mEncryptedHashes; // map such that hash(it->second) = it->first - std::string mFileName; + std::map mLocalDirs ; // map is better for search. it->first=it->second.filename + std::map mEncryptedHashes; // map such that hash(it->second) = it->first - bool mTSChanged ; + bool mTSChanged ; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 63ad5e9f9..90060dd41 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -39,8 +39,9 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names -static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. -static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. +static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. +static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. +static const std::string LOCAL_SHARED_DIRS_FILE_NAME = "local_dir_hierarchy.bin" ; // hard-coded directory name to store encrypted local dir hierarchy. static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 91a7546df..e53058bc9 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -66,7 +66,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mRemoteDirectories.clear() ; // we should load them! mOwnId = mpeers->getOwnId() ; - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); + mLocalSharedDirs = new LocalDirectoryStorage(mFileSharingDir + "/" + LOCAL_SHARED_DIRS_FILE_NAME,mOwnId); mHashCache = new HashStorage(mFileSharingDir + "/" + HASH_CACHE_FILE_NAME) ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; @@ -219,6 +219,7 @@ int p3FileDatabase::tick() } mLastRemoteDirSweepTS = now; + mLocalSharedDirs->checkSave() ; // This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up, // so the first notify is ignored, and no other notify will happen next. From 5339e991276d19000ade0d01fc707d483809c0db Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 17 Nov 2016 21:00:34 +0100 Subject: [PATCH 2/4] force re-sweep of remote directory that has been updated with new dir content to 10 sec --- libretroshare/src/file_sharing/p3filelists.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index e53058bc9..36bad6cf0 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1448,7 +1448,10 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) RS_STACK_MUTEX(mFLSMtx) ; if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed + { + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed + mRemoteDirectories[fi]->lastSweepTime() = time(NULL) - DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP + 10 ; // force re-sweep in 10 secs, so as to fasten updated + } else P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; From deadb1fcf1fd6ec0234211bcb2fdccaaed1ae555 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 17 Nov 2016 21:01:01 +0100 Subject: [PATCH 3/4] small optimization of RemoteDirModel::data() --- retroshare-gui/src/gui/RemoteDirModel.cpp | 36 +++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 9c010e904..6334c187b 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -622,9 +622,29 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if (!index.isValid()) return QVariant(); + // First we take care of the cases that do not require requestDirDetails() + + int coln = index.column(); + + if (role == Qt::SizeHintRole) + return QVariant(); + + if (role == Qt::TextAlignmentRole) + { + if(coln == 1) + return int( Qt::AlignRight | Qt::AlignVCenter); + else + return QVariant(); + } + + // This makes sorting a bit arbitrary, but prevents calling requestDirDetails(). The number of calls to sortRole is + // indeed sometimes very high and somewhat kills the GUI responsivness. + // + //if (role == SortRole) + // return QVariant(index.row()) ; + /* get the data from the index */ void *ref = index.internalPointer(); - int coln = index.column(); DirDetails details ; @@ -674,20 +694,6 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const Qt::SizeHintRole ****************/ - if (role == Qt::SizeHintRole) - { - return QVariant(); // Use standard - } /* end of SizeHintRole */ - - if (role == Qt::TextAlignmentRole) - { - if(coln == 1) - { - return int( Qt::AlignRight | Qt::AlignVCenter); - } - return QVariant(); - } /* end of TextAlignmentRole */ - if (role == Qt::DisplayRole) return displayRole(details,coln) ; From bdc8086c2e82907f011803bd835af972955f763d Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 17 Nov 2016 21:27:37 +0100 Subject: [PATCH 4/4] added missing mChanged=true when removign a directory --- libretroshare/src/file_sharing/directory_storage.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e7b6e0fd9..ca8e3dc48 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -148,6 +148,7 @@ bool DirectoryStorage::removeDirectory(const EntryIndex& indx) { RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->removeDirectory(indx); + mChanged = true ; return res ; }