From 463f21c85c0ea1a1fc9f2e72074913573f9decc8 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 00:48:28 +0200 Subject: [PATCH] fixed hashing thread --- .../src/file_sharing/directory_storage.cc | 61 +++++++++++++++++-- .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 14 +++-- .../src/file_sharing/directory_updater.h | 2 +- libretroshare/src/file_sharing/hash_cache.cc | 52 ++++++++-------- libretroshare/src/file_sharing/hash_cache.h | 1 + 6 files changed, 92 insertions(+), 40 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 813b0c38d..ee89d572c 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -46,7 +46,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(DirectoryStorage::NO_INDEX) {} + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -226,7 +226,7 @@ class InternalFileHierarchyStorage if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) return DirectoryStorage::NO_INDEX; - return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } bool check() // checks consistency of storage. @@ -234,10 +234,57 @@ class InternalFileHierarchyStorage return true; } - void print() + void print() const { + int nfiles = 0 ; + int ndirs = 0 ; + int nempty = 0 ; + int nunknown = 0; + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++ndirs; + } + else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++nfiles; + } + else + { + ++nunknown; + std::cerr << "(EE) Error: unknown type node found!" << std::endl; + } + + std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; + + recursPrint(0,DirectoryStorage::EntryIndex(0)); } private: + void recursPrint(int depth,DirectoryStorage::EntryIndex node) const + { + std::string indent(2*depth,' '); + DirEntry& d(*static_cast(mNodes[node])); + + std::cerr << indent << d.dir_name << std::endl; + + for(int i=0;i(mNodes[i])); + std::cerr << indent << " " << f.file_hash << " " << f.file_modtime << " " << f.file_name << std::endl; + } + } + static bool nodeAccessError(const std::string& s) { std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; @@ -300,7 +347,6 @@ DirectoryStorage::DirIterator ::operator bool() const { return **this != Directo RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); } uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; } std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } -std::string DirectoryStorage::FileIterator::fullpath() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } /******************************************************************************************************************/ @@ -368,7 +414,12 @@ void DirectoryStorage::save(const std::string& local_file_name) { // first write the header, than all fields. } - +void DirectoryStorage::print() +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + std::cerr << "LocalDirectoryStorage:" << std::endl; + mFileHierarchy->print(); +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fe7307d6d..9f8293963 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -73,7 +73,6 @@ class DirectoryStorage // info about the file that is pointed by the iterator std::string name() const ; - std::string fullpath() const ; uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; @@ -93,6 +92,7 @@ class DirectoryStorage bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + void print(); void cleanup(); private: diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 3d6c5bb74..9367e826e 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,10 +10,9 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc) - : mHashCache(hc) +LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) + : mHashCache(hc),mSharedDirectories(lds) { - mSharedDirectories = new LocalDirectoryStorage("local_storage.txt"); } void LocalDirectoryUpdater::tick() @@ -27,7 +26,7 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - + mSharedDirectories->getSharedDirectoryList(shared_directory_list); std::set sub_dir_list ; @@ -43,7 +42,10 @@ void LocalDirectoryUpdater::tick() DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + { + std::cerr << " recursing into " << real_dir_it->filename << std::endl; recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; + } } void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) @@ -98,7 +100,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p RsFileHash hash ; - if(mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) mSharedDirectories->updateHash(*dit,hash); } @@ -107,7 +109,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + recursUpdateSharedDir(cumulated_path + "/" + *real_dir_it, *stored_dir_it) ; } void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index b70164bdd..f0317b36d 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -23,7 +23,7 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient { public: - LocalDirectoryUpdater(HashStorage *hash_cache) ; + LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} virtual void tick() ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 25af17984..0acd323c2 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -6,13 +6,14 @@ HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { + mRunning = false ; } void HashStorage::data_tick() { - std::cerr << "Ticking hash thread." << std::endl; - FileHashJob job; + RsFileHash hash; + uint64_t size ; { RS_STACK_MUTEX(mHashMtx) ; @@ -21,33 +22,29 @@ void HashStorage::data_tick() return ; job = mFilesToHash.begin()->second ; + + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + else + std::cerr << "done."<< std::endl; + + mFilesToHash.erase(mFilesToHash.begin()) ; + + if(mFilesToHash.empty()) + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + } } - - std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - - RsFileHash hash; - uint64_t size ; - - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - { - std::cerr << "ERROR" << std::endl; - return; - } - // update the hash storage - - // call the client - job.client->hash_callback(job.client_param, job.full_path, hash, size); - - std::cerr << "done."<< std::endl; - - if(mFilesToHash.empty()) - { - std::cerr << "Starting hashing thread." << std::endl; - fullstop(); - std::cerr << "done." << std::endl; - } + if(!hash.isNull()) + job.client->hash_callback(job.client_param, job.full_path, hash, size); } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -81,8 +78,9 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mFilesToHash[full_path] = job; - if(!isRunning()) + if(!mRunning) { + mRunning = true ; std::cerr << "Starting hashing thread." << std::endl; start() ; } diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index fa73797c2..88b4206f9 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -77,5 +77,6 @@ private: // thread/mutex stuff RsMutex mHashMtx ; + bool mRunning; };