From bd9a464d11796316146cdd694771e3952d4bea85 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 12 Oct 2016 23:20:38 +0200 Subject: [PATCH] added deterministic but unredictable hash generation for dir hashes, which should be preserved accross reboots. Should fix the msg from friends sending requests for the "wrong" dir hashes. --- .../src/file_sharing/dir_hierarchy.cc | 20 ++++++++++++---- .../src/file_sharing/dir_hierarchy.h | 4 ++-- .../src/file_sharing/directory_storage.cc | 4 ++-- .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 22 ++++++++++------- .../src/file_sharing/directory_updater.h | 5 ++++ .../src/file_sharing/file_sharing_defaults.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 24 +++++++++++++++++-- 8 files changed, 63 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 3e112ef95..26de88688 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -149,7 +149,7 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) return e < mNodes.size() && mNodes[e] != NULL ; } -bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) +bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs,const RsFileHash& random_hash_seed) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; @@ -190,7 +190,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->parent_index = indx; de->dir_modtime = 0;// forces parsing.it->second; de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; - de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ; + de->dir_hash = createDirHash(de->dir_name,d.dir_hash,random_hash_seed) ; mDirHashes[de->dir_hash] = mNodes.size() ; @@ -201,7 +201,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage return true; } -RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_name*/, const std::string &/*dir_parent_path*/) +RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name, const RsFileHash& dir_parent_hash, const RsFileHash& random_hash_salt) { // What we need here: a unique identifier // - that cannot be bruteforced to find the real directory name and path @@ -215,7 +215,19 @@ RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_ // Option 3: just compute something random, but then we need to store it so as to not // confuse friends when restarting. - return RsFileHash::random(); + RsTemporaryMemory mem(dir_name.size() + 2*RsFileHash::SIZE_IN_BYTES) ; + + memcpy( mem, random_hash_salt.toByteArray(),RsFileHash::SIZE_IN_BYTES) ; + memcpy(&mem[ RsFileHash::SIZE_IN_BYTES], dir_parent_hash.toByteArray(),RsFileHash::SIZE_IN_BYTES) ; + memcpy(&mem[2*RsFileHash::SIZE_IN_BYTES],dir_name.c_str(), dir_name.size()) ; + + RsFileHash res = RsDirUtil::sha1sum( mem,mem.size() ) ; + +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "Creating new dir hash for dir " << dir_name << ", parent dir hash=" << dir_parent_hash << " seed=[hidden]" << " result is " << res << std::endl; +#endif + + return res ; } bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index fe937fac9..364b8f6d8 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -95,7 +95,7 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::map& subdirs, const RsFileHash &random_hash_seed); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); @@ -153,7 +153,7 @@ public: private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; static bool nodeAccessError(const std::string& s); - static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; + static RsFileHash createDirHash(const std::string& dir_name, const RsFileHash &dir_parent_hash, const RsFileHash &random_hash_salt) ; // Allocates a new entry in mNodes, possible re-using an empty slot and returns its index. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7780717ca..2c8f48743 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -127,10 +127,10 @@ bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } -bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs,const RsFileHash& hash_salt) { RS_STACK_MUTEX(mDirStorageMtx) ; - bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs,hash_salt) ; locked_check() ; return res ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 2d8119445..c920a2df4 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -133,7 +133,7 @@ class DirectoryStorage // Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with // just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache. // - bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs, const RsFileHash &random_hash_salt) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 41c4a941e..de92c6ee1 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -55,7 +55,7 @@ void LocalDirectoryUpdater::setEnabled(bool b) return ; if(b) - start("fs dir updater") ; + start("fs dir updater") ; else shutdown(); @@ -88,17 +88,23 @@ void LocalDirectoryUpdater::forceUpdate() void LocalDirectoryUpdater::sweepSharedDirectories() { + if(mHashSalt.isNull()) + { + std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl; + return ; + } + RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); #ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl; #endif - // recursive update algorithm works that way: - // - the external loop starts on the shared directory list and goes through sub-directories + // recursive update algorithm works that way: + // - the external loop starts on the shared directory list and goes through sub-directories // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. - // - the information that is costly to compute (the hash) is store externally into a separate structure. - // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. - // + // - the information that is costly to compute (the hash) is store externally into a separate structure. + // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. + // std::list shared_directory_list ; mSharedDirectories->getSharedDirectoryList(shared_directory_list); @@ -109,7 +115,7 @@ void LocalDirectoryUpdater::sweepSharedDirectories() // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ; // now for each of them, go recursively and match both files and dirs @@ -177,7 +183,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // update file and dir lists for current directory. - mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ; std::map new_files ; mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index d0f920cb0..85df00d6c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -41,6 +41,9 @@ public: void forceUpdate(); bool inDirectoryCheck() const ; + void setHashSalt(const RsFileHash& hash) { mHashSalt = hash; } + const RsFileHash& hashSalt() const { return mHashSalt; } + void setFileWatchPeriod(int seconds) ; uint32_t fileWatchPeriod() const ; @@ -60,6 +63,8 @@ private: HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; + RsFileHash mHashSalt ; + time_t mLastSweepTime; time_t mLastTSUpdateTime; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 94e2981f7..520fc61ea 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -33,6 +33,7 @@ static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 se static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch +static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index f8f87c701..a35de0e79 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -326,7 +326,14 @@ cleanup = true; rskv->tlvkvs.pairs.push_back(kv); } + { + RsTlvKeyValue kv; + kv.key = WATCH_HASH_SALT_SS; + kv.value = mLocalDirWatcher->hashSalt().toStdString(); + + rskv->tlvkvs.pairs.push_back(kv); + } /* Add KeyValue to saveList */ sList.push_back(rskv); @@ -373,6 +380,11 @@ bool p3FileDatabase::loadList(std::list& load) { setWatchEnabled(kit->value == "YES") ; } + else if(kit->key == WATCH_HASH_SALT_SS) + { + std::cerr << "Initing directory watcher with saved secret salt..." << std::endl; + mLocalDirWatcher->setHashSalt(RsFileHash(kit->value)) ; + } delete *it ; continue ; } @@ -398,6 +410,14 @@ bool p3FileDatabase::loadList(std::list& load) delete *it ; } + if(mLocalDirWatcher->hashSalt().isNull()) + { + std::cerr << "(WW) Initialising directory watcher salt to some random value " << std::endl; + mLocalDirWatcher->setHashSalt(RsFileHash::random()) ; + + IndicateConfigChanged(); + } + /* set directories */ mLocalSharedDirs->setSharedDirectoryList(dirList); @@ -642,8 +662,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } - else - return false; + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ;