From 3f88e3e9018edcf264081a27a4e0f7c15bf03101 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 16 May 2018 13:24:52 +0200 Subject: [PATCH] added check to avoid hashing files that are currently being modified --- .../src/file_sharing/directory_storage.cc | 2 +- .../src/file_sharing/directory_updater.cc | 49 ++++++++++++++----- .../src/file_sharing/directory_updater.h | 4 +- .../src/file_sharing/file_sharing_defaults.h | 21 ++++---- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index eb3c58626..d39401e06 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -623,7 +623,7 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i if(it == mLocalDirs.end()) { - std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; + std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; return false ; } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 6d95961ab..dffef92c7 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -73,10 +73,23 @@ void LocalDirectoryUpdater::data_tick() { if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime) { - if(sweepSharedDirectories()) + bool some_files_not_ready = false ; + + if(sweepSharedDirectories(some_files_not_ready)) { - mNeedsFullRecheck = false; - mLastSweepTime = now ; + if(some_files_not_ready) + { + mNeedsFullRecheck = true ; + mLastSweepTime = now - mDelayBetweenDirectoryUpdates + 60 ; // retry 20 secs from now + + std::cerr << "(II) some files being modified. Will re-scan in 60 secs." << std::endl; + } + else + { + mNeedsFullRecheck = false ; + mLastSweepTime = now ; + } + mSharedDirectories->notifyTSChanged(); mForceUpdate = false ; } @@ -111,7 +124,7 @@ void LocalDirectoryUpdater::forceUpdate() mHashCache->togglePauseHashingProcess(); } -bool LocalDirectoryUpdater::sweepSharedDirectories() +bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready) { if(mHashSalt.isNull()) { @@ -158,8 +171,8 @@ bool LocalDirectoryUpdater::sweepSharedDirectories() #endif existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name())); - recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1) ; // here we need to use the list that was stored, instead of the shared dir list, because the two - // are not necessarily in the same order. + recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ; // here we need to use the list that was stored, instead of the shared dir list, because the two + // are not necessarily in the same order. } RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); @@ -168,7 +181,7 @@ bool LocalDirectoryUpdater::sweepSharedDirectories() return true ; } -void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set& existing_directories,uint32_t current_depth) +void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set& existing_directories,uint32_t current_depth,bool& some_files_not_ready) { #ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl; @@ -187,6 +200,8 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p return; } + time_t now = time(NULL) ; + if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match. // we only want to detect when the directory has changed on the disk { @@ -200,11 +215,23 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); + case librs::util::FolderIterator::TYPE_FILE: + + if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now) + { + subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; + subfiles[dirIt.file_name()].size = dirIt.file_size(); #ifdef DEBUG_LOCAL_DIR_UPDATER - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; #endif + } + else + { + some_files_not_ready = true ; + + std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being modified. Keeping it for later." << std::endl; + } + break; case librs::util::FolderIterator::TYPE_DIR: @@ -276,7 +303,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p #ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " recursing into " << stored_dir_it.name() << std::endl; #endif - recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1) ; + recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ; } } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index e4adb28b4..4c6e05649 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -70,8 +70,8 @@ protected: virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); virtual bool hash_confirm(uint32_t client_param) ; - void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set& existing_directories, uint32_t current_depth); - bool sweepSharedDirectories(); + void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set& existing_directories, uint32_t current_depth,bool& files_not_ready); + bool sweepSharedDirectories(bool &some_files_not_ready); private: bool filterFile(const std::string& fname) const ; // reponds true if the file passes the ignore lists test. diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 9e2dd6241..8de4b31c8 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -34,16 +34,16 @@ static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP = 60 ; // 60 se static const uint32_t DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR = 60*24*86400 ; // delete non empty remoe directories after 60 days of inactivity static const uint32_t DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR = 5*24*86400 ; // delete empty remote directories after 5 days of inactivity -static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time -static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files -static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch -static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them. -static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links -static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names -static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes -static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes -static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags -static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories +static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time +static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch +static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them. +static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links +static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names +static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes +static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes +static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags +static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. @@ -51,6 +51,7 @@ static const std::string LOCAL_SHARED_DIRS_FILE_NAME = "local_dir_hierarchy.bin" static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this +static const uint32_t MIN_TIME_AFTER_LAST_MODIFICATION = 20 ; // never hash a file that is just being modified, otherwise we end up with a corrupted hash static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days