Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-28 08:59:37 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

added check to avoid hashing files that are currently being modified

Browse Source
This commit is contained in:
csoler 2018-05-16 13:24:52 +02:00
parent 8451550561
commit 3f88e3e901
No known key found for this signature in database
GPG Key ID: 7BCA522266C0804C
4 changed files with 52 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -623,7 +623,7 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i
if(it == mLocalDirs.end()) if(it == mLocalDirs.end())
{ {
std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
return false ; return false ;
} }

49
libretroshare/src/file_sharing/directory_updater.cc
View File

@ -73,10 +73,23 @@ void LocalDirectoryUpdater::data_tick()
{ {
if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime) if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime)
{ {
if(sweepSharedDirectories()) bool some_files_not_ready = false ;
if(sweepSharedDirectories(some_files_not_ready))
{ {
mNeedsFullRecheck = false; if(some_files_not_ready)
mLastSweepTime = now ; {
mNeedsFullRecheck = true ;
mLastSweepTime = now - mDelayBetweenDirectoryUpdates + 60 ; // retry 20 secs from now
std::cerr << "(II) some files being modified. Will re-scan in 60 secs." << std::endl;
}
else
{
mNeedsFullRecheck = false ;
mLastSweepTime = now ;
}
mSharedDirectories->notifyTSChanged(); mSharedDirectories->notifyTSChanged();
mForceUpdate = false ; mForceUpdate = false ;
} }
@ -111,7 +124,7 @@ void LocalDirectoryUpdater::forceUpdate()
mHashCache->togglePauseHashingProcess(); mHashCache->togglePauseHashingProcess();
} }
bool LocalDirectoryUpdater::sweepSharedDirectories() bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
{ {
if(mHashSalt.isNull()) if(mHashSalt.isNull())
{ {
@ -158,8 +171,8 @@ bool LocalDirectoryUpdater::sweepSharedDirectories()
#endif #endif
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name())); existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1) ; // here we need to use the list that was stored, instead of the shared dir list, because the two recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order. // are not necessarily in the same order.
} }
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
@ -168,7 +181,7 @@ bool LocalDirectoryUpdater::sweepSharedDirectories()
return true ; return true ;
} }
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth) void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
{ {
#ifdef DEBUG_LOCAL_DIR_UPDATER #ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl; std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
@ -187,6 +200,8 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
return; return;
} }
time_t now = time(NULL) ;
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match. if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match.
// we only want to detect when the directory has changed on the disk // we only want to detect when the directory has changed on the disk
{ {
@ -200,11 +215,23 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
{ {
switch(dirIt.file_type()) switch(dirIt.file_type())
{ {
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; case librs::util::FolderIterator::TYPE_FILE:
subfiles[dirIt.file_name()].size = dirIt.file_size();
if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER #ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif #endif
}
else
{
some_files_not_ready = true ;
std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being modified. Keeping it for later." << std::endl;
}
break; break;
case librs::util::FolderIterator::TYPE_DIR: case librs::util::FolderIterator::TYPE_DIR:
@ -276,7 +303,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
#ifdef DEBUG_LOCAL_DIR_UPDATER #ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl; std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif #endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1) ; recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
} }
} }

4
libretroshare/src/file_sharing/directory_updater.h
View File

@ -70,8 +70,8 @@ protected:
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size);
virtual bool hash_confirm(uint32_t client_param) ; virtual bool hash_confirm(uint32_t client_param) ;
void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set<std::string>& existing_directories, uint32_t current_depth); void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set<std::string>& existing_directories, uint32_t current_depth,bool& files_not_ready);
bool sweepSharedDirectories(); bool sweepSharedDirectories(bool &some_files_not_ready);
private: private:
bool filterFile(const std::string& fname) const ; // reponds true if the file passes the ignore lists test. bool filterFile(const std::string& fname) const ; // reponds true if the file passes the ignore lists test.

21
libretroshare/src/file_sharing/file_sharing_defaults.h
View File

@ -34,16 +34,16 @@ static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP = 60 ; // 60 se
static const uint32_t DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR = 60*24*86400 ; // delete non empty remoe directories after 60 days of inactivity static const uint32_t DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR = 60*24*86400 ; // delete non empty remoe directories after 60 days of inactivity
static const uint32_t DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR = 5*24*86400 ; // delete empty remote directories after 5 days of inactivity static const uint32_t DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR = 5*24*86400 ; // delete empty remote directories after 5 days of inactivity
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them. static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them.
static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes
static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes
static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags
static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
@ -51,6 +51,7 @@ static const std::string LOCAL_SHARED_DIRS_FILE_NAME = "local_dir_hierarchy.bin"
static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs.
static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this
static const uint32_t MIN_TIME_AFTER_LAST_MODIFICATION = 20 ; // never hash a file that is just being modified, otherwise we end up with a corrupted hash
static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission
static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days