From 884b3a62200604dfb308390be3e6d85325e1dc34 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Sep 2017 19:58:57 +0200 Subject: [PATCH 1/4] added infrastructure to add a ignore list in shared files --- .../src/file_sharing/directory_updater.cc | 44 ++++++++++--- .../src/file_sharing/directory_updater.h | 9 +++ .../src/file_sharing/file_sharing_defaults.h | 3 + libretroshare/src/file_sharing/p3filelists.cc | 62 +++++++++++++++++++ libretroshare/src/file_sharing/p3filelists.h | 3 + libretroshare/src/ft/ftserver.cc | 10 +++ libretroshare/src/ft/ftserver.h | 3 + libretroshare/src/retroshare/rsfiles.h | 10 ++- .../src/gui/settings/TransferPage.cpp | 50 +++++++++++++++ .../src/gui/settings/TransferPage.h | 1 + .../src/gui/settings/TransferPage.ui | 32 +++++++++- 11 files changed, 216 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 21703cfa2..d603e5900 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -232,16 +232,17 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // now go through list of subfiles and request the hash to hashcache for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) - { - // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. + if(filterFile(dit.name())) + { + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - RsFileHash hash ; + RsFileHash hash ; - // mSharedDirectories des two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed. + // mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed. - if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit)) - mSharedDirectories->updateHash(*dit,hash,hash != dit.hash()); - } + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit)) + mSharedDirectories->updateHash(*dit,hash,hash != dit.hash()); + } } #ifdef DEBUG_LOCAL_DIR_UPDATER else @@ -259,6 +260,21 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p } } +bool LocalDirectoryUpdater::filterFile(const std::string& fname) const +{ + if(mIgnoreFlags & RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES) + for(auto it(mIgnoredSuffixes.begin());it!=mIgnoredSuffixes.end();++it) + if(fname.size() >= (*it).size() && fname.substr( fname.size() - (*it).size()) == *it) + return false ; + + if(mIgnoreFlags & RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES) + for(auto it(mIgnoredPrefixes.begin());it!=mIgnoredPrefixes.end();++it) + if(fname.size() >= (*it).size() && fname.substr( 0,(*it).size()) == *it) + return false ; + + return true ; +} + bool LocalDirectoryUpdater::inDirectoryCheck() const { return mHashCache->isRunning(); @@ -301,5 +317,19 @@ bool LocalDirectoryUpdater::followSymLinks() const return mFollowSymLinks ; } +void LocalDirectoryUpdater::setIgnoreLists(const std::list& ignored_prefixes,const std::list& ignored_suffixes,uint32_t ignore_flags) +{ + mIgnoredPrefixes = ignored_prefixes ; + mIgnoredSuffixes = ignored_suffixes ; + mIgnoreFlags = ignore_flags; +} +bool LocalDirectoryUpdater::getIgnoreLists(std::list& ignored_prefixes,std::list& ignored_suffixes,uint32_t& ignore_flags) const +{ + ignored_prefixes = mIgnoredPrefixes; + ignored_suffixes = mIgnoredSuffixes; + ignore_flags = mIgnoreFlags ; + + return true; +} diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index b55d38d9e..be7c9c610 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -53,6 +53,9 @@ public: void setEnabled(bool b) ; bool isEnabled() const ; + void setIgnoreLists(const std::list& ignored_prefixes,const std::list& ignored_suffixes,uint32_t ignore_flags) ; + bool getIgnoreLists(std::list& ignored_prefixes,std::list& ignored_suffixes,uint32_t& ignore_flags) const ; + protected: virtual void data_tick() ; @@ -63,6 +66,8 @@ protected: bool sweepSharedDirectories(); private: + bool filterFile(const std::string& fname) const ; // reponds true if the file passes the ignore lists test. + HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; @@ -77,5 +82,9 @@ private: bool mNeedsFullRecheck ; bool mIsChecking ; bool mForceUpdate ; + + uint32_t mIgnoreFlags ; + std::list mIgnoredPrefixes ; + std::list mIgnoredSuffixes ; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 386dfbae0..44b7cfca9 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -39,6 +39,9 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them. static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names +static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes +static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes +static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 1f138454a..8b420fc2c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -81,6 +81,17 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) addSerialType(new RsFileListsSerialiser()) ; } +void p3FileDatabase::setIgnoreLists(const std::list& ignored_prefixes,const std::list& ignored_suffixes, uint32_t ignore_flags) +{ + RS_STACK_MUTEX(mFLSMtx) ; + mLocalDirWatcher->setIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; +} +bool p3FileDatabase::getIgnoreLists(std::list& ignored_prefixes,std::list& ignored_suffixes, uint32_t& ignore_flags) +{ + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->getIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; +} + RsSerialiser *p3FileDatabase::setupSerialiser() { // This one is for saveList/loadList @@ -345,6 +356,28 @@ cleanup = true; rskv->tlvkvs.pairs.push_back(kv); } + { + std::list prefix_list,suffix_list; + uint32_t flags ; + + mLocalDirWatcher->getIgnoreLists(prefix_list,suffix_list,flags) ; + + std::string prefix_string, suffix_string ; + + for(auto it(prefix_list.begin());it!=prefix_list.end();++it) prefix_string += *it + ";" ; + for(auto it(suffix_list.begin());it!=suffix_list.end();++it) suffix_string += *it + ";" ; + + RsTlvKeyValue kv; + + kv.key = IGNORED_PREFIXES_SS; kv.value = prefix_string; rskv->tlvkvs.pairs.push_back(kv); + kv.key = IGNORED_SUFFIXES_SS; kv.value = suffix_string; rskv->tlvkvs.pairs.push_back(kv); + + std::string s ; + rs_sprintf(s, "%lu", flags) ; + + kv.key = IGNORE_LIST_FLAGS_SS; kv.value = s; rskv->tlvkvs.pairs.push_back(kv); + } + /* Add KeyValue to saveList */ sList.push_back(rskv); @@ -363,6 +396,8 @@ bool p3FileDatabase::loadList(std::list& load) #endif std::list dirList; + std::list ignored_prefixes,ignored_suffixes ; + uint32_t ignore_flags ; for(std::list::iterator it = load.begin(); it != load.end(); ++it) { @@ -400,6 +435,31 @@ bool p3FileDatabase::loadList(std::list& load) std::cerr << "Initing directory watcher with saved secret salt..." << std::endl; mLocalDirWatcher->setHashSalt(RsFileHash(kit->value)) ; } + else if(kit->key == IGNORED_PREFIXES_SS) + { + std::string b ; + for(uint32_t i=0;ivalue.size();++i) + if(kit->value[i] == ';') + ignored_prefixes.push_back(b) ; + else + b.push_back(kit->value[i]) ; + } + else if(kit->key == IGNORED_SUFFIXES_SS) + { + std::string b ; + for(uint32_t i=0;ivalue.size();++i) + if(kit->value[i] == ';') + ignored_suffixes.push_back(b) ; + else + b.push_back(kit->value[i]) ; + } + else if(kit->key == IGNORE_LIST_FLAGS_SS) + { + int t=0 ; + if(sscanf(kit->value.c_str(),"%d",&t) == 1) + ignore_flags = (uint32_t)t ; + } + delete *it ; continue ; } @@ -427,6 +487,8 @@ bool p3FileDatabase::loadList(std::list& load) /* set directories */ mLocalSharedDirs->setSharedDirectoryList(dirList); + mLocalDirWatcher->setIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; + load.clear() ; return true; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 81630bb74..d8bdf423f 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -127,6 +127,9 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path,std::string& fullpath); + void setIgnoreLists(const std::list& ignored_prefixes,const std::list& ignored_suffixes, uint32_t ignore_flags) ; + bool getIgnoreLists(std::list& ignored_prefixes,std::list& ignored_suffixes, uint32_t& ignore_flags) ; + // computes/gathers statistics about shared directories int getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& stats); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 3951f9de1..568bec61a 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -826,6 +826,16 @@ bool ftServer::removeSharedDirectory(std::string dir) return true; } + +bool ftServer::getIgnoreLists(std::list& ignored_prefixes, std::list& ignored_suffixes,uint32_t& ignore_flags) +{ + return mFileDatabase->getIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; +} +void ftServer::setIgnoreLists(const std::list& ignored_prefixes, const std::list& ignored_suffixes, uint32_t ignore_flags) +{ + mFileDatabase->setIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; +} + bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; } int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; } bool ftServer::followSymLinks() const { return mFileDatabase->followSymLinks() ; } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index a8780ee9a..e56102a3b 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -213,6 +213,9 @@ public: virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist ! virtual bool removeSharedDirectory(std::string dir); + virtual bool getIgnoreLists(std::list& ignored_prefixes, std::list& ignored_suffixes, uint32_t& ignore_flags) ; + virtual void setIgnoreLists(const std::list& ignored_prefixes, const std::list& ignored_suffixes,uint32_t ignore_flags) ; + virtual bool getShareDownloadDirectory(); virtual bool shareDownloadDirectory(bool share); diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 4604cb700..34e82deed 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -41,7 +41,7 @@ namespace RsRegularExpression { class Expression; } /* These are used mainly by ftController at the moment */ const uint32_t RS_FILE_CTRL_PAUSE = 0x00000100; const uint32_t RS_FILE_CTRL_START = 0x00000200; -const uint32_t RS_FILE_CTRL_FORCE_CHECK = 0x00000400; +const uint32_t RS_FILE_CTRL_FORCE_CHECK = 0x00000400; const uint32_t RS_FILE_CTRL_ENCRYPTION_POLICY_STRICT = 0x00000001 ; const uint32_t RS_FILE_CTRL_ENCRYPTION_POLICY_PERMISSIVE = 0x00000002 ; @@ -60,6 +60,9 @@ const uint32_t RS_FILE_RATE_STREAM_VIDEO = 0x00000006; const uint32_t RS_FILE_PEER_ONLINE = 0x00001000; const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000; +const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES = 0x0001 ; +const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES = 0x0002 ; + /************************************ * Used To indicate where to search. * @@ -251,9 +254,12 @@ class RsFiles virtual bool updateShareFlags(const SharedDirInfo& dir) = 0; // updates the flags. The directory should already exist ! virtual bool removeSharedDirectory(std::string dir) = 0; + virtual bool getIgnoreLists(std::list& ignored_prefixes, std::list& ignored_suffixes,uint32_t& flags) =0; + virtual void setIgnoreLists(const std::list& ignored_prefixes, const std::list& ignored_suffixes,uint32_t flags) =0; + virtual void setWatchPeriod(int minutes) =0; virtual void setWatchEnabled(bool b) =0; - virtual int watchPeriod() const =0; + virtual int watchPeriod() const =0; virtual bool watchEnabled() =0; virtual bool followSymLinks() const=0; virtual void setFollowSymLinks(bool b)=0 ; diff --git a/retroshare-gui/src/gui/settings/TransferPage.cpp b/retroshare-gui/src/gui/settings/TransferPage.cpp index f3c5630bf..851bb1d0d 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.cpp +++ b/retroshare-gui/src/gui/settings/TransferPage.cpp @@ -54,6 +54,30 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) QObject::connect(ui.autoCheckDirectoriesDelay_SB,SIGNAL(valueChanged(int)),this,SLOT(updateAutoScanDirectoriesPeriod())) ; QObject::connect(ui.shareDownloadDirectoryCB, SIGNAL(toggled(bool)), this,SLOT(updateShareDownloadDirectory())) ; QObject::connect(ui.followSymLinks_CB, SIGNAL(toggled(bool)), this,SLOT(updateFollowSymLinks())) ; + + QObject::connect(ui.prefixesIgnoreList_LE, SIGNAL(textChanged(QString)), this,SLOT(updateIgnoreLists())) ; + QObject::connect(ui.prefixesIgnoreList_CB, SIGNAL(toggled(bool)), this,SLOT(updateIgnoreLists())) ; + QObject::connect(ui.suffixesIgnoreList_LE, SIGNAL(textChanged(QString)), this,SLOT(updateIgnoreLists())) ; + QObject::connect(ui.suffixesIgnoreList_CB, SIGNAL(toggled(bool)), this,SLOT(updateIgnoreLists())) ; +} + +void TransferPage::updateIgnoreLists() +{ + uint32_t flags = 0 ; + if(ui.prefixesIgnoreList_CB->isChecked()) flags |= RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES ; + if(ui.suffixesIgnoreList_CB->isChecked()) flags |= RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES ; + + std::list lp,ls ; + { QStringList L = ui.prefixesIgnoreList_LE->text().split(';') ; for(QStringList::const_iterator it(L.begin());it!=L.end();++it) lp.push_back((*it).toStdString()) ; } + { QStringList L = ui.suffixesIgnoreList_LE->text().split(';') ; for(QStringList::const_iterator it(L.begin());it!=L.end();++it) ls.push_back((*it).toStdString()) ; } + + rsFiles->setIgnoreLists(lp,ls,flags) ; + + std::cerr << "Setting ignore lists: " << std::endl; + + std::cerr << " flags: " << flags << std::endl; + std::cerr << " prefixes: " ; for(auto it(lp.begin());it!=lp.end();++it) std::cerr << "\"" << *it << "\" " ; std::cerr << std::endl; + std::cerr << " suffixes: " ; for(auto it(ls.begin());it!=ls.end();++it) std::cerr << "\"" << *it << "\" " ; std::cerr << std::endl; } void TransferPage::updateMaxTRUpRate(int b) @@ -125,6 +149,32 @@ void TransferPage::load() case RS_FILE_PERM_DIRECT_DL_NO: whileBlocking(ui._filePermDirectDL_CB)->setCurrentIndex(1) ; break ; default: whileBlocking(ui._filePermDirectDL_CB)->setCurrentIndex(2) ; break ; } + + std::list suffixes, prefixes; + uint32_t ignore_flags ; + + rsFiles->getIgnoreLists(prefixes,suffixes,ignore_flags) ; + + QString ignore_prefixes_string,ignore_suffixes_string ; + + for(auto it(prefixes.begin());it!=prefixes.end();) + { + ignore_prefixes_string += QString::fromStdString(*it) ; + + if(++it != prefixes.end()) + ignore_prefixes_string += ";" ; + } + for(auto it(suffixes.begin());it!=suffixes.end();) + { + ignore_suffixes_string += QString::fromStdString(*it) ; + + if(++it != suffixes.end()) + ignore_suffixes_string += ";" ; + } + whileBlocking(ui.prefixesIgnoreList_CB)->setChecked( ignore_flags & RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES ) ; + whileBlocking(ui.suffixesIgnoreList_CB)->setChecked( ignore_flags & RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES ) ; + whileBlocking(ui.prefixesIgnoreList_LE)->setText( ignore_prefixes_string ); + whileBlocking(ui.suffixesIgnoreList_LE)->setText( ignore_suffixes_string ); } void TransferPage::updateDefaultStrategy(int i) diff --git a/retroshare-gui/src/gui/settings/TransferPage.h b/retroshare-gui/src/gui/settings/TransferPage.h index ed6cbb8e2..fc39160c3 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.h +++ b/retroshare-gui/src/gui/settings/TransferPage.h @@ -50,6 +50,7 @@ class TransferPage: public ConfigPage void updateEncryptionPolicy(int); void updateMaxUploadSlots(int); void updateFilePermDirectDL(int); + void updateIgnoreLists(); void editDirectories() ; void setIncomingDirectory(); diff --git a/retroshare-gui/src/gui/settings/TransferPage.ui b/retroshare-gui/src/gui/settings/TransferPage.ui index 3f349411b..eb684a078 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.ui +++ b/retroshare-gui/src/gui/settings/TransferPage.ui @@ -7,7 +7,7 @@ 0 0 1126 - 1099 + 1103 @@ -16,7 +16,7 @@ Shared Directories - + @@ -88,6 +88,34 @@ + + + + + + ignore files with these suffixes: + + + + + + + + + + + + + + ignore files with these prefixes: + + + + + + + + From ea25d4b5a43ce4b017a92e0dce45a45a955b0283 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Sep 2017 20:22:17 +0200 Subject: [PATCH 2/4] improved update of ignore list --- libretroshare/src/file_sharing/p3filelists.cc | 2 ++ retroshare-gui/src/gui/settings/TransferPage.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 8b420fc2c..ef572c440 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -85,6 +85,8 @@ void p3FileDatabase::setIgnoreLists(const std::list& ignored_prefix { RS_STACK_MUTEX(mFLSMtx) ; mLocalDirWatcher->setIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ; + + IndicateConfigChanged(); } bool p3FileDatabase::getIgnoreLists(std::list& ignored_prefixes,std::list& ignored_suffixes, uint32_t& ignore_flags) { diff --git a/retroshare-gui/src/gui/settings/TransferPage.cpp b/retroshare-gui/src/gui/settings/TransferPage.cpp index 851bb1d0d..33ba576f8 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.cpp +++ b/retroshare-gui/src/gui/settings/TransferPage.cpp @@ -55,9 +55,9 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) QObject::connect(ui.shareDownloadDirectoryCB, SIGNAL(toggled(bool)), this,SLOT(updateShareDownloadDirectory())) ; QObject::connect(ui.followSymLinks_CB, SIGNAL(toggled(bool)), this,SLOT(updateFollowSymLinks())) ; - QObject::connect(ui.prefixesIgnoreList_LE, SIGNAL(textChanged(QString)), this,SLOT(updateIgnoreLists())) ; + QObject::connect(ui.prefixesIgnoreList_LE, SIGNAL(editingFinished()), this,SLOT(updateIgnoreLists())) ; QObject::connect(ui.prefixesIgnoreList_CB, SIGNAL(toggled(bool)), this,SLOT(updateIgnoreLists())) ; - QObject::connect(ui.suffixesIgnoreList_LE, SIGNAL(textChanged(QString)), this,SLOT(updateIgnoreLists())) ; + QObject::connect(ui.suffixesIgnoreList_LE, SIGNAL(editingFinished()), this,SLOT(updateIgnoreLists())) ; QObject::connect(ui.suffixesIgnoreList_CB, SIGNAL(toggled(bool)), this,SLOT(updateIgnoreLists())) ; } From eff5c5d6eec545bddc58e512430d497670618eba Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Sep 2017 22:07:36 +0200 Subject: [PATCH 3/4] added pause to file hashing --- .../src/file_sharing/directory_updater.cc | 9 ++++++ .../src/file_sharing/directory_updater.h | 2 ++ libretroshare/src/file_sharing/hash_cache.cc | 28 +++++++++++++++++++ libretroshare/src/file_sharing/hash_cache.h | 3 ++ libretroshare/src/file_sharing/p3filelists.cc | 16 +++++++++++ libretroshare/src/file_sharing/p3filelists.h | 2 ++ libretroshare/src/ft/ftserver.cc | 3 ++ libretroshare/src/ft/ftserver.h | 2 ++ libretroshare/src/retroshare/rsfiles.h | 2 ++ .../src/gui/statusbar/hashingstatus.cpp | 28 +++++++++++++++++-- .../src/gui/statusbar/hashingstatus.h | 2 ++ 11 files changed, 95 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index d603e5900..1c7d45d5f 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -275,6 +275,15 @@ bool LocalDirectoryUpdater::filterFile(const std::string& fname) const return true ; } +void LocalDirectoryUpdater::togglePauseHashingProcess() +{ + mHashCache->togglePauseHashingProcess() ; +} +bool LocalDirectoryUpdater::hashingProcessPaused() +{ + return mHashCache->hashingProcessPaused(); +} + bool LocalDirectoryUpdater::inDirectoryCheck() const { return mHashCache->isRunning(); diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index be7c9c610..5930985e3 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -40,6 +40,8 @@ public: void forceUpdate(); bool inDirectoryCheck() const ; + void togglePauseHashingProcess(); + bool hashingProcessPaused(); void setHashSalt(const RsFileHash& hash) { mHashSalt = hash; } const RsFileHash& hashSalt() const { return mHashSalt; } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 52cd05bda..b1fc2576e 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -44,6 +44,7 @@ HashStorage::HashStorage(const std::string& save_file_name) mTotalSizeToHash = 0; mTotalFilesToHash = 0; mMaxStorageDurationDays = DEFAULT_HASH_STORAGE_DURATION_DAYS ; + mHashingProcessPaused = false; { RS_STACK_MUTEX(mHashMtx) ; @@ -52,6 +53,18 @@ HashStorage::HashStorage(const std::string& save_file_name) try_load_import_old_hash_cache(); } } + +void HashStorage::togglePauseHashingProcess() +{ + RS_STACK_MUTEX(mHashMtx) ; + mHashingProcessPaused = !mHashingProcessPaused ; +} +bool HashStorage::hashingProcessPaused() +{ + RS_STACK_MUTEX(mHashMtx) ; + return mHashingProcessPaused; +} + static std::string friendlyUnit(uint64_t val) { const std::string units[5] = {"B","KB","MB","GB","TB"}; @@ -78,12 +91,14 @@ void HashStorage::data_tick() RsFileHash hash; uint64_t size = 0; + { bool empty ; uint32_t st ; { RS_STACK_MUTEX(mHashMtx) ; + if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) { locked_save(); @@ -136,6 +151,19 @@ void HashStorage::data_tick() } mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; + bool paused = false ; + { + RS_STACK_MUTEX(mHashMtx) ; + paused = mHashingProcessPaused ; + } + + if(paused) + { + usleep(MAX_INACTIVITY_SLEEP_TIME) ; + std::cerr << "Hashing process currently paused." << std::endl; + return; + } + else { RS_STACK_MUTEX(mHashMtx) ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 512056596..7ca12c10b 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -84,6 +84,8 @@ public: uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } void clear() { mFiles.clear(); mChanged=true; } // drop all known hashes. Not something to do, except if you want to rehash the entire database bool empty() const { return mFiles.empty() ; } + void togglePauseHashingProcess() ; + bool hashingProcessPaused(); // Functions called by the thread @@ -112,6 +114,7 @@ private: std::map mFiles ; // stored as (full_path, hash_info) std::string mFilePath ; // file where the hash database is stored bool mChanged ; + bool mHashingProcessPaused ; struct FileHashJob { diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ef572c440..34a16b4ba 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -442,7 +442,10 @@ bool p3FileDatabase::loadList(std::list& load) std::string b ; for(uint32_t i=0;ivalue.size();++i) if(kit->value[i] == ';') + { ignored_prefixes.push_back(b) ; + b.clear(); + } else b.push_back(kit->value[i]) ; } @@ -451,7 +454,10 @@ bool p3FileDatabase::loadList(std::list& load) std::string b ; for(uint32_t i=0;ivalue.size();++i) if(kit->value[i] == ';') + { ignored_suffixes.push_back(b) ; + b.clear(); + } else b.push_back(kit->value[i]) ; } @@ -972,6 +978,16 @@ void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the di { mLocalDirWatcher->forceUpdate(); } +void p3FileDatabase::togglePauseHashingProcess() +{ + RS_STACK_MUTEX(mFLSMtx) ; + mLocalDirWatcher->togglePauseHashingProcess(); +} +bool p3FileDatabase::hashingProcessPaused() +{ + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->hashingProcessPaused(); +} bool p3FileDatabase::inDirectoryCheck() { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index d8bdf423f..13ec0a086 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -148,6 +148,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed bool inDirectoryCheck(); + void togglePauseHashingProcess(); + bool hashingProcessPaused(); protected: diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 568bec61a..46d3380a9 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -844,6 +844,9 @@ void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnab void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; } void ftServer::setFollowSymLinks(bool b) { mFileDatabase->setFollowSymLinks(b) ; } +void ftServer::togglePauseHashingProcess() { mFileDatabase->togglePauseHashingProcess() ; } +bool ftServer::hashingProcessPaused() { return mFileDatabase->hashingProcessPaused() ; } + bool ftServer::getShareDownloadDirectory() { std::list dirList; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index e56102a3b..fa64a3fb9 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -225,6 +225,8 @@ public: virtual bool watchEnabled() ; virtual bool followSymLinks() const; virtual void setFollowSymLinks(bool b); + virtual void togglePauseHashingProcess(); + virtual bool hashingProcessPaused(); /***************************************************************/ /*************** Data Transfer Interface ***********************/ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 34e82deed..a65706f94 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -263,6 +263,8 @@ class RsFiles virtual bool watchEnabled() =0; virtual bool followSymLinks() const=0; virtual void setFollowSymLinks(bool b)=0 ; + virtual void togglePauseHashingProcess() =0; // pauses/resumes the hashing process. + virtual bool hashingProcessPaused() =0; virtual bool getShareDownloadDirectory() = 0; virtual bool shareDownloadDirectory(bool share) = 0; diff --git a/retroshare-gui/src/gui/statusbar/hashingstatus.cpp b/retroshare-gui/src/gui/statusbar/hashingstatus.cpp index 3cf36409f..f0d5b3f7b 100644 --- a/retroshare-gui/src/gui/statusbar/hashingstatus.cpp +++ b/retroshare-gui/src/gui/statusbar/hashingstatus.cpp @@ -22,7 +22,9 @@ #include #include #include +#include +#include "retroshare/rsfiles.h" #include "hashingstatus.h" #include "gui/common/ElidedLabel.h" #include "gui/notifyqt.h" @@ -61,13 +63,14 @@ HashingStatus::~HashingStatus() void HashingStatus::updateHashingInfo(const QString& s) { - if (s.isEmpty()) { + if (s.isEmpty()) + { statusHashing->hide() ; hashloader->hide() ; movie->stop() ; } else { - hashloader->setToolTip(s) ; + setToolTip(s + "\n"+QObject::tr("Click to pause the hashing process")); if (_compactMode) { statusHashing->hide() ; @@ -80,3 +83,24 @@ void HashingStatus::updateHashingInfo(const QString& s) movie->start() ; } } + +void HashingStatus::mousePressEvent(QMouseEvent *) +{ + rsFiles->togglePauseHashingProcess() ; + + if(rsFiles->hashingProcessPaused()) + { + movie->stop() ; + hashloader->setPixmap(QPixmap(":/images/resume.png")) ; + + mLastText = statusHashing->text(); + statusHashing->setText(QObject::tr("[Hashing is paused]")); + setToolTip(QObject::tr("Click to resume the hashing process")); + } + else + { + hashloader->setMovie(movie) ; + statusHashing->setText(mLastText); + movie->start() ; + } +} diff --git a/retroshare-gui/src/gui/statusbar/hashingstatus.h b/retroshare-gui/src/gui/statusbar/hashingstatus.h index c26cbfc9a..752e5cf32 100644 --- a/retroshare-gui/src/gui/statusbar/hashingstatus.h +++ b/retroshare-gui/src/gui/statusbar/hashingstatus.h @@ -35,6 +35,7 @@ public: ~HashingStatus(); void setCompactMode(bool compact) {_compactMode = compact; } + void mousePressEvent(QMouseEvent *); public slots: void updateHashingInfo(const QString&) ; @@ -42,6 +43,7 @@ public slots: private: ElidedLabel *statusHashing; QLabel *hashloader; + QString mLastText ; QMovie *movie; bool _compactMode; }; From 0ca0b72a5a84868d60e5eb7349c95733c902b545 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 11 Sep 2017 19:02:07 +0200 Subject: [PATCH 4/4] moved file ignore test to a more appropriate place --- .../src/file_sharing/directory_updater.cc | 50 +++++++++++-------- libretroshare/src/file_sharing/hash_cache.cc | 2 +- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 1c7d45d5f..609371e39 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -199,25 +199,26 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::set subdirs ; for(;dirIt.isValid();dirIt.next()) - { - switch(dirIt.file_type()) - { - case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); + if(filterFile(dirIt.file_name())) + { + switch(dirIt.file_type()) + { + case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; + subfiles[dirIt.file_name()].size = dirIt.file_size(); #ifdef DEBUG_LOCAL_DIR_UPDATER - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; #endif - break; + break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()); + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()); #ifdef DEBUG_LOCAL_DIR_UPDATER - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; #endif - break; - default: - std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; - } - } + break; + default: + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; + } + } // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files. mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ; @@ -232,17 +233,16 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // now go through list of subfiles and request the hash to hashcache for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) - if(filterFile(dit.name())) - { - // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. + { + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - RsFileHash hash ; + RsFileHash hash ; - // mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed. + // mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed. - if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit)) - mSharedDirectories->updateHash(*dit,hash,hash != dit.hash()); - } + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit)) + mSharedDirectories->updateHash(*dit,hash,hash != dit.hash()); + } } #ifdef DEBUG_LOCAL_DIR_UPDATER else @@ -265,12 +265,18 @@ bool LocalDirectoryUpdater::filterFile(const std::string& fname) const if(mIgnoreFlags & RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES) for(auto it(mIgnoredSuffixes.begin());it!=mIgnoredSuffixes.end();++it) if(fname.size() >= (*it).size() && fname.substr( fname.size() - (*it).size()) == *it) + { + std::cerr << "(II) ignoring file " << fname << ", because it matches suffix \"" << *it << "\"" << std::endl; return false ; + } if(mIgnoreFlags & RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES) for(auto it(mIgnoredPrefixes.begin());it!=mIgnoredPrefixes.end();++it) if(fname.size() >= (*it).size() && fname.substr( 0,(*it).size()) == *it) + { + std::cerr << "(II) ignoring file " << fname << ", because it matches prefix \"" << *it << "\"" << std::endl; return false ; + } return true ; } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index b1fc2576e..2c504bbc5 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -157,7 +157,7 @@ void HashStorage::data_tick() paused = mHashingProcessPaused ; } - if(paused) + if(paused) // we need to wait off mutex!! { usleep(MAX_INACTIVITY_SLEEP_TIME) ; std::cerr << "Hashing process currently paused." << std::endl;