Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-07 16:45:11 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1050 from csoler/v0.6-FT

Browse source 
V0.6 ft
This commit is contained in:
csoler 2017-09-25 20:59:32 +02:00 • committed by GitHub
commit b969f313b0
12 changed files with 284 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

78
libretroshare/src/file_sharing/directory_updater.cc
View file

@ -40,9 +40,10 @@ LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStora
, mLastSweepTime(0), mLastTSUpdateTime(0)
, mDelayBetweenDirectoryUpdates(DELAY_BETWEEN_DIRECTORY_UPDATES)
, mIsEnabled(false), mFollowSymLinks(FOLLOW_SYMLINKS_DEFAULT)
, mIgnoreDuplicates(true)
/* Can be left to false, but setting it to true will force to re-hash any file that has been left unhashed in the last session.*/
, mNeedsFullRecheck(true)
, mIsChecking(false), mForceUpdate(false), mIgnoreFlags (0)
, mIsChecking(false), mForceUpdate(false), mIgnoreFlags (0), mMaxShareDepth(0)
{
}
@ -146,8 +147,9 @@ bool LocalDirectoryUpdater::sweepSharedDirectories()
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
}
@ -157,23 +159,12 @@ bool LocalDirectoryUpdater::sweepSharedDirectories()
return true ;
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories)
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
if(mFollowSymLinks)
{
std::string real_path = RsDirUtil::removeSymLinks(cumulated_path) ;
if(existing_directories.end() != existing_directories.find(real_path))
{
std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
return ;
}
existing_directories.insert(real_path) ;
}
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
@ -207,10 +198,33 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
#endif
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name());
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true ;
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64)) // 64 is here as a safe limit, to make loops impossible.
dir_is_accepted = false ;
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
if(existing_directories.end() != existing_directories.find(real_path))
{
std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
dir_is_accepted = false ;
}
else
existing_directories.insert(real_path) ;
}
if(dir_is_accepted)
subdirs.insert(dirIt.file_name());
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
@ -248,13 +262,13 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
// go through the list of sub-dirs and recursively update
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
{
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories) ;
}
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1) ;
}
}
bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@ -344,4 +358,28 @@ bool LocalDirectoryUpdater::getIgnoreLists(std::list<std::string>& ignored_prefi
return true;
}
int LocalDirectoryUpdater::maxShareDepth() const
{
return mMaxShareDepth ;
}
void LocalDirectoryUpdater::setMaxShareDepth(uint32_t d)
{
if(d != mMaxShareDepth)
mNeedsFullRecheck = true ;
mMaxShareDepth = d ;
}
bool LocalDirectoryUpdater::ignoreDuplicates() const
{
return mIgnoreDuplicates;
}
void LocalDirectoryUpdater::setIgnoreDuplicates(bool b)
{
if(b != mIgnoreDuplicates)
mNeedsFullRecheck = true ;
mIgnoreDuplicates = b ;
}

11
libretroshare/src/file_sharing/directory_updater.h
View file

@ -58,13 +58,19 @@ public:
void setIgnoreLists(const std::list<std::string>& ignored_prefixes,const std::list<std::string>& ignored_suffixes,uint32_t ignore_flags) ;
bool getIgnoreLists(std::list<std::string>& ignored_prefixes,std::list<std::string>& ignored_suffixes,uint32_t& ignore_flags) const ;
void setMaxShareDepth(uint32_t i) ;
int maxShareDepth() const;
void setIgnoreDuplicates(bool b) ;
bool ignoreDuplicates() const;
protected:
virtual void data_tick() ;
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size);
virtual bool hash_confirm(uint32_t client_param) ;
void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set<std::string>& existing_directories);
void recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx, std::set<std::string>& existing_directories, uint32_t current_depth);
bool sweepSharedDirectories();
private:
@ -81,11 +87,14 @@ private:
uint32_t mDelayBetweenDirectoryUpdates;
bool mIsEnabled ;
bool mFollowSymLinks;
bool mIgnoreDuplicates;
bool mNeedsFullRecheck ;
bool mIsChecking ;
bool mForceUpdate ;
uint32_t mIgnoreFlags ;
uint32_t mMaxShareDepth ;
std::list<std::string> mIgnoredPrefixes ;
std::list<std::string> mIgnoredSuffixes ;
};

2
libretroshare/src/file_sharing/file_sharing_defaults.h
View file

@ -38,10 +38,12 @@ static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // ke
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them.
static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes
static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes
static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags
static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.

11
libretroshare/src/file_sharing/hash_cache.cc
View file

@ -223,8 +223,10 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t
#endif
RS_STACK_MUTEX(mHashMtx) ;
std::string real_path = RsDirUtil::removeSymLinks(full_path) ;
time_t now = time(NULL) ;
std::map<std::string,HashStorageInfo>::iterator it = mFiles.find(full_path) ;
std::map<std::string,HashStorageInfo>::iterator it = mFiles.find(real_path) ;
// On windows we compare the time up to +/- 3600 seconds. This avoids re-hashing files in case of daylight saving change.
//
@ -261,7 +263,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t
// we need to schedule a re-hashing
if(mFilesToHash.find(full_path) != mFilesToHash.end())
if(mFilesToHash.find(real_path) != mFilesToHash.end())
return false ;
FileHashJob job ;
@ -272,7 +274,10 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t
job.full_path = full_path ;
job.ts = mod_time ;
mFilesToHash[full_path] = job;
// We store the files indexed by their real path, so that we allow to not re-hash files that are pointed multiple times through the directory links
// The client will be notified with the full path instead of the real path.
mFilesToHash[real_path] = job;
mTotalSizeToHash += size ;
++mTotalFilesToHash;

55
libretroshare/src/file_sharing/p3filelists.cc
View file

@ -332,6 +332,17 @@ cleanup = true;
kv.key = WATCH_FILE_DURATION_SS;
kv.value = s ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
std::string s ;
rs_sprintf(s, "%d", maxShareDepth()) ;
RsTlvKeyValue kv;
kv.key = MAX_SHARE_DEPTH;
kv.value = s ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
@ -342,6 +353,14 @@ cleanup = true;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
kv.key = IGNORE_DUPLICATES;
kv.value = ignoreDuplicates()?"YES":"NO" ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
@ -400,6 +419,7 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
std::list<SharedDirInfo> dirList;
std::list<std::string> ignored_prefixes,ignored_suffixes ;
uint32_t ignore_flags = RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES | RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES ;
uint32_t max_share_depth = 0;
// OS-dependent default ignore lists
#ifdef WINDOWS_SYS
@ -433,10 +453,14 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
if(sscanf(kit->value.c_str(),"%d",&t) == 1)
setWatchPeriod(t);
}
else if(kit->key == FOLLOW_SYMLINKS_SS)
else if(kit->key == FOLLOW_SYMLINKS_SS)
{
setFollowSymLinks(kit->value == "YES") ;
}
else if(kit->key == IGNORE_DUPLICATES)
{
setIgnoreDuplicates(kit->value == "YES") ;
}
else if(kit->key == WATCH_FILE_ENABLED_SS)
{
setWatchEnabled(kit->value == "YES") ;
@ -480,6 +504,12 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
if(sscanf(kit->value.c_str(),"%d",&t) == 1)
ignore_flags = (uint32_t)t ;
}
else if(kit->key == MAX_SHARE_DEPTH)
{
int t=0 ;
if(sscanf(kit->value.c_str(),"%d",&t) == 1)
max_share_depth = (uint32_t)t ;
}
delete *it ;
continue ;
@ -509,6 +539,7 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
/* set directories */
mLocalSharedDirs->setSharedDirectoryList(dirList);
mLocalDirWatcher->setIgnoreLists(ignored_prefixes,ignored_suffixes,ignore_flags) ;
mLocalDirWatcher->setMaxShareDepth(max_share_depth);
load.clear() ;
@ -1070,6 +1101,28 @@ bool p3FileDatabase::followSymLinks() const
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->followSymLinks() ;
}
void p3FileDatabase::setIgnoreDuplicates(bool i)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalDirWatcher->setIgnoreDuplicates(i) ;
IndicateConfigChanged();
}
bool p3FileDatabase::ignoreDuplicates() const
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->ignoreDuplicates() ;
}
void p3FileDatabase::setMaxShareDepth(int i)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalDirWatcher->setMaxShareDepth(i) ;
IndicateConfigChanged();
}
int p3FileDatabase::maxShareDepth() const
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->maxShareDepth() ;
}
void p3FileDatabase::setWatchEnabled(bool b)
{
RS_STACK_MUTEX(mFLSMtx) ;

6
libretroshare/src/file_sharing/p3filelists.h
View file

@ -130,6 +130,12 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
void setIgnoreLists(const std::list<std::string>& ignored_prefixes,const std::list<std::string>& ignored_suffixes, uint32_t ignore_flags) ;
bool getIgnoreLists(std::list<std::string>& ignored_prefixes,std::list<std::string>& ignored_suffixes, uint32_t& ignore_flags) ;
void setIgnoreDuplicates(bool i) ;
bool ignoreDuplicates() const ;
void setMaxShareDepth(int i) ;
int maxShareDepth() const ;
// computes/gathers statistics about shared directories
int getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& stats);

4
libretroshare/src/ft/ftserver.cc
View file

@ -839,10 +839,14 @@ void ftServer::setIgnoreLists(const std::list<std::string>& ignored_prefixes, co
bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; }
int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; }
bool ftServer::followSymLinks() const { return mFileDatabase->followSymLinks() ; }
bool ftServer::ignoreDuplicates() { return mFileDatabase->ignoreDuplicates() ; }
int ftServer::maxShareDepth() const { return mFileDatabase->maxShareDepth() ; }
void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnabled(b) ; }
void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; }
void ftServer::setFollowSymLinks(bool b) { mFileDatabase->setFollowSymLinks(b) ; }
void ftServer::setIgnoreDuplicates(bool ignore) { mFileDatabase->setIgnoreDuplicates(ignore); }
void ftServer::setMaxShareDepth(int depth) { mFileDatabase->setMaxShareDepth(depth) ; }
void ftServer::togglePauseHashingProcess() { mFileDatabase->togglePauseHashingProcess() ; }
bool ftServer::hashingProcessPaused() { return mFileDatabase->hashingProcessPaused() ; }

6
libretroshare/src/ft/ftserver.h
View file

@ -228,6 +228,12 @@ public:
virtual void togglePauseHashingProcess();
virtual bool hashingProcessPaused();
virtual void setMaxShareDepth(int depth) ;
virtual int maxShareDepth() const;
virtual bool ignoreDuplicates() ;
virtual void setIgnoreDuplicates(bool ignore) ;
/***************************************************************/
/*************** Data Transfer Interface ***********************/
/***************************************************************/

13
libretroshare/src/retroshare/rsfiles.h
View file

@ -60,8 +60,11 @@ const uint32_t RS_FILE_RATE_STREAM_VIDEO = 0x00000006;
const uint32_t RS_FILE_PEER_ONLINE = 0x00001000;
const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000;
const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES = 0x0001 ;
const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES = 0x0002 ;
const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_PREFIXES = 0x0001 ;
const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_SUFFIXES = 0x0002 ;
const uint32_t RS_FILE_SHARE_FLAGS_IGNORE_DUPLICATES = 0x0004 ;
const uint32_t RS_FILE_SHARE_PARAMETER_DEFAULT_MAXIMUM_DEPTH = 8;
/************************************
* Used To indicate where to search.
@ -269,6 +272,12 @@ class RsFiles
virtual bool getShareDownloadDirectory() = 0;
virtual bool shareDownloadDirectory(bool share) = 0;
virtual void setMaxShareDepth(int depth) =0;
virtual int maxShareDepth() const=0;
virtual bool ignoreDuplicates() = 0;
virtual void setIgnoreDuplicates(bool ignore) = 0;
};