diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index c4f60ee81..56b1e7bc2 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -789,7 +789,7 @@ void FileIndexMonitor::updateCycle() to_hash.back().realpath = realpath ; to_hash.back().dirpath = dirpath ; - while(isRunning() && dirIt.readdir()) + for(;dirIt.isValid() && isRunning();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index acfdcbf25..0b7e100e4 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -5,6 +5,9 @@ class InternalFileHierarchyStorage class FileStorageNode { public: + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; + virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode @@ -19,12 +22,12 @@ class InternalFileHierarchyStorage public: virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + std::set subdirs ; + std::set subfiles ; }; // file/dir access and modification - bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e uint32_t root ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0399d9d9e..0fb94037a 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -39,27 +39,44 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; + DirIterator(const EntryIndex& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the directory that is pointed by the iterator + + const std::string& name() const ; + }; class FileIterator { public: FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(EntryIndex& e); // crawls all files in specified directory - uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the file that is pointed by the iterator + + std::string name() const ; + std::string fullpath() const ; + uint64_t size() const ; + RsFileHash hash() const ; + time_t modtime() const ; + }; virtual DirIterator root() ; // returns the index of the root directory entry. - private: + void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void removeDirectory(const EntryIndex& indx) ; + + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c20c3703b..c88820fb0 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,19 +1,24 @@ +#include "util/folderiterator.h" + #include "directory_storage.h" #include "directory_updater.h" +#define DEBUG_LOCAL_DIR_UPDATER 1 + void RemoteDirectoryUpdater::tick() { // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater() +LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc) + : mHashCache(hc) { - // tell the hash cache that we're the client - mHashCache->setClient(this) ; } void LocalDirectoryUpdater::tick() { + std::cerr << "LocalDirectoryUpdater::tick()" << std::endl; + // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. @@ -21,28 +26,29 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; + std::list sub_dir_list ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) - sub_dir_list.push_back( (*it).filename ) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) + sub_dir_list.push_back( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ; // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; } -LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) { + std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl; + // make sure list of subdirs is the same - // make sure list of subfiles is the same - // request all hashes to the hashcache librs::util::FolderIterator dirIt(cumulated_path); @@ -60,22 +66,18 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c while(dirIt.readdir()) { - std::string name ; - uint64_t size ; - uint8_t type ; - time_t mod_time ; - - dirIt.readEntryInformation(type,name,size,mod_time) ; - - switch(type) + switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; - break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + break; + + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; default: - std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } } // update file and dir lists for current directory. @@ -87,18 +89,16 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c for(DirectoryStorage::FileIterator dit(indx);dit;++dit) { - // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - if() - if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) - mSharedDirectories->updateFileHash(*dit,hash) ; + mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ; } // go through the list of sub-dirs and recursively update DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 54b70f30c..35f069b3c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,7 +3,8 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // -class HashCache ; +#include "file_sharing/hash_cache.h" + class LocalDirectoryStorage ; class DirectoryUpdater @@ -19,12 +20,16 @@ class DirectoryUpdater // }; -class LocalDirectoryUpdater: public DirectoryUpdater +class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; virtual void tick() ; +protected: + virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ; + void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); + private: HashCache *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6734310fe..f867f6109 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -2,14 +2,28 @@ #include #include "util/rsthreads.h" +#include "retroshare/rsfiles.h" + +class HashCacheClient +{ +public: + virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ; +}; + +class FileHashingThread: public RsTickingThread +{ +public: + FileHashingThread() {} + + virtual void data_tick() ; +}; class HashCache { public: HashCache(const std::string& save_file_name) ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; + bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ; struct HashCacheInfo { @@ -21,10 +35,10 @@ public: } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; } + uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; } + void clear() { mFiles.clear(); } + bool empty() const { return mFiles.empty() ; } private: void clean() ; @@ -33,7 +47,7 @@ private: void load() ; // threaded stuff - RsTickingThread mHashingThread ; + FileHashingThread mHashingThread ; RsMutex mHashMtx ; // Local configuration and storage @@ -45,6 +59,6 @@ private: // current work - std::map mFilesToHash ; + std::map mFilesToHash ; }; diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 7d101fdf3..31ee2a39f 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -146,7 +146,7 @@ void RsPluginManager::loadPlugins(const std::vector& plugin_directo continue ; } - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { std::string fname; dirIt.d_name(fname); diff --git a/libretroshare/src/rsserver/rsaccounts.cc b/libretroshare/src/rsserver/rsaccounts.cc index ad9e1306e..9bb8b49ed 100644 --- a/libretroshare/src/rsserver/rsaccounts.cc +++ b/libretroshare/src/rsserver/rsaccounts.cc @@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map & struct stat64 buf; - while (dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 01ff94461..747ec93d9 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -1,11 +1,17 @@ +#include +#include +#include +#include +#include + #include "folderiterator.h" #include "rsstring.h" - namespace librs { namespace util { FolderIterator::FolderIterator(const std::string& folderName) + : mFolderName(folderName) { #ifdef WINDOWS_SYS std::wstring utf16Name; @@ -18,10 +24,10 @@ FolderIterator::FolderIterator(const std::string& folderName) handle = FindFirstFileW(utf16Name.c_str(), &fileInfo); validity = handle != INVALID_HANDLE_VALUE; - isFirstCall = true; #else handle = opendir(folderName.c_str()); validity = handle != NULL; + next(); #endif } @@ -30,19 +36,71 @@ FolderIterator::~FolderIterator() closedir(); } -bool FolderIterator::readdir() { +void FolderIterator::next() +{ + do { + if(!readdir()) + { + validity = false ; + break ; + } + + d_name(mFileName); + } while(mFileName == "." || mFileName == "..") ; + + mFullPath = mFolderName + "/" + mFileName ; + + struct stat64 buf ; + +#ifdef WINDOWS_SYS + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) +#else + if ( 0 == stat64(mFullPath.c_str(), &buf)) +#endif + { + mFileModTime = buf.st_mtime ; + mStatInfoOk = true; + + if (S_ISDIR(buf.st_mode)) + { + mType = TYPE_DIR ; + mFileSize = 0 ; + mFileModTime = buf.st_mtime; + } + else if (S_ISREG(buf.st_mode)) + { + mType = TYPE_FILE ; + mFileSize = buf.st_size; + mFileModTime = buf.st_mtime; + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + } + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + validity = false ; + } +} + +bool FolderIterator::readdir() +{ if(!validity) return false; #ifdef WINDOWS_SYS - if(isFirstCall) { - isFirstCall = false; - return true; - } return FindNextFileW(handle, &fileInfo) != 0; #else ent = ::readdir(handle); - return ent != 0; + return ent != NULL; #endif } @@ -65,6 +123,11 @@ bool FolderIterator::d_name(std::string& dest) return true; } +const std::string& FolderIterator::file_fullpath() { return mFullPath ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } + bool FolderIterator::closedir() { if(!validity) diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 8007db1a1..823aa9a9a 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -2,6 +2,7 @@ #define FOLDERITERATOR_H +#include #include #include @@ -24,14 +25,24 @@ public: FolderIterator(const std::string& folderName); ~FolderIterator(); - bool isValid() const { return validity; } + enum { TYPE_UNKNOWN = 0x00, + TYPE_FILE = 0x01, + TYPE_DIR = 0x02 + }; + bool isValid() const { return validity; } bool readdir(); + void next(); bool d_name(std::string& dest); - bool closedir(); + const std::string& file_name() ; + const std::string& file_fullpath() ; + uint64_t file_size() ; + uint64_t file_type() ; + time_t file_modtime() ; + private: bool validity; @@ -43,7 +54,15 @@ private: DIR* handle; struct dirent* ent; #endif + void updateStatsInfo() ; + bool mStatInfoOk ; + time_t mFileModTime ; + uint64_t mFileSize ; + uint8_t mType ; + std::string mFileName ; + std::string mFullPath ; + std::string mFolderName ; }; diff --git a/retroshare-nogui/src/introserver.cc b/retroshare-nogui/src/introserver.cc index 5445dc814..3734154b2 100644 --- a/retroshare-nogui/src/introserver.cc +++ b/retroshare-nogui/src/introserver.cc @@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts() mCertCheckTime = now; struct stat64 buf; - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname;