From 3c976bb7ee181d4183d43355c7f530f26cc7318d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 21 Jul 2016 00:16:12 -0400 Subject: [PATCH] added squeleton code for own directory update --- libretroshare/src/file_sharing/README.txt | 16 +-- .../src/file_sharing/directory_storage.cc | 60 +++++++++++ .../src/file_sharing/directory_storage.h | 48 +++++++-- .../src/file_sharing/directory_updater.cc | 101 +++++++++++++++--- .../src/file_sharing/directory_updater.h | 12 ++- libretroshare/src/file_sharing/hash_cache.h | 65 +++++++---- 6 files changed, 248 insertions(+), 54 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 9cd98b84c..1b6cccb2b 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -160,14 +160,14 @@ Generating sync events Roadmap ------- -- complete this file until a proper description of the whole thing is achieved. -- create a new directory and implement the .h for the basic functionality -- look into existing code in ftServer for the integration, but don't change anything yet -- setup class hierarchy -- merge hash cache into file lists. - -- optionally - - change the saving system of FileIndex to make it locally encrypted and compact +[X] complete this file until a proper description of the whole thing is achieved. +[X] create a new directory and implement the .h for the basic functionality +[ ] look into existing code in ftServer for the integration, but don't change anything yet +[X] setup class hierarchy +[ ] merge hash cache into file lists. +[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible +[ ] create basic directory functionality with own files: re-hash, and store +[ ] display own files in GUI, with proper update and working sort TODO ==== diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e69de29bb..acfdcbf25 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -0,0 +1,60 @@ +#include "directory_storage.h" + +class InternalFileHierarchyStorage +{ + class FileStorageNode + { + public: + virtual uint32_t type() const =0; + }; + class FileEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + + }; + + class DirEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } + + std::set subdirs ; + std::set subfiles ; + }; + + // file/dir access and modification + bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + + uint32_t root ; + + std::vector mNodes;// uses pointers to keep information about valid/invalid objects. + + void compress() ; // use empty space in the vector, mostly due to deleted entries. + + friend class DirectoryStorage ; // only class that can use this. +}; + +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; + +void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) +{ + entry_tag = data[offset++] ; +} +void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size) +{ +} + +void DirectoryStorage::load(const std::string& local_file_name) +{ + // first load the header, than all fields. +} +void DirectoryStorage::save(const std::string& local_file_name) +{ + // first write the header, than all fields. +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 6106e4bb6..0399d9d9e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -14,6 +14,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; +class InternalFileHierarchyStorage ; + class DirectoryStorage { public: @@ -36,30 +38,37 @@ class DirectoryStorage class DirIterator { public: - DirIterator(const DirectoryStorage& d) ; + DirIterator(const DirIterator& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; class FileIterator { public: - FileIterator(const DirectoryStorage& d) ; + FileIterator(DirIterator& d); // crawls all files in specified directory + uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; + virtual DirIterator root() ; // returns the index of the root directory entry. + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; - void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ; - void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; + void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. + + InternalFileHierarchyStorage *mFileHierarchy ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -74,5 +83,32 @@ class LocalDirectoryStorage: public DirectoryStorage public: LocalDirectoryStorage() ; virtual ~LocalDirectoryStorage() {} + + void setSharedDirectoryList(const std::list& lst) ; + void getSharedDirectoryList(std::list& lst) ; + + /*! + * \brief addFile + * \param dir + * \param hash + * \param modf_time + */ + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + +private: + std::list mLocalDirs ; + + std::map mHashes ; // used for fast search access }; + + + + + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 72917248b..c20c3703b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,30 +6,99 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } +LocalDirectoryUpdater::LocalDirectoryUpdater() +{ + // tell the hash cache that we're the client + mHashCache->setClient(this) ; +} + void LocalDirectoryUpdater::tick() { -#ifdef TODO // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories - // - at the same time, it updates the list of shared directories + // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. // - the information that is costly to compute (the hash) is store externally into a separate structure. // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // - for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it) - { - librs::util::FolderIterator dirIt(realpath); - if(!dirIt.isValid()) - mLocalSharedDirs.removeDirectory(it) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + std::list shared_directory_list ; - while(dirIt.readdir()) - { + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) + sub_dir_list.push_back( (*it).filename ) ; - } + // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash)) - { - ; - } - } -#endif + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + + // now for each of them, go recursively and match both files and dirs + + DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } + +LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +{ + // make sure list of subdirs is the same + + // make sure list of subfiles is the same + + // request all hashes to the hashcache + + librs::util::FolderIterator dirIt(cumulated_path); + + if(!dirIt.isValid()) + { + mSharedDirectories->removeDirectory(indx) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + return ; + } + + // collect subdirs and subfiles + + std::list subfiles ; + std::list subdirs ; + + while(dirIt.readdir()) + { + std::string name ; + uint64_t size ; + uint8_t type ; + time_t mod_time ; + + dirIt.readEntryInformation(type,name,size,mod_time) ; + + switch(type) + { + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; + break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + break; + + default: + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + } + } + // update file and dir lists for current directory. + + mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + // now go through list of subfiles and request the hash to hashcache + + for(DirectoryStorage::FileIterator dit(indx);dit;++dit) + { + // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + + if() + if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) + mSharedDirectories->updateFileHash(*dit,hash) ; + } + + // go through the list of sub-dirs and recursively update + + DirectoryStorage::DirIterator stored_dir_it(indx) ; + + for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; +} + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 3571247ab..54b70f30c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,6 +3,9 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // +class HashCache ; +class LocalDirectoryStorage ; + class DirectoryUpdater { public: @@ -18,8 +21,13 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater { - public: - virtual void tick() ; +public: + LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual void tick() ; + +private: + HashCache *mHashCache ; + LocalDirectoryStorage *mSharedDirectories ; }; class RemoteDirectoryUpdater: public DirectoryUpdater diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 144e6d8ee..6734310fe 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -1,29 +1,50 @@ +#pragma once + +#include +#include "util/rsthreads.h" + class HashCache { - public: - HashCache(const std::string& save_file_name) ; +public: + HashCache(const std::string& save_file_name) ; - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; + void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; + bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; + struct HashCacheInfo + { + std::string filename ; + uint64_t size ; + uint32_t time_stamp ; + uint32_t modf_stamp ; + RsFileHash hash ; + } ; - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; + // interaction with GUI, called from p3FileLists + void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } + uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } + void clear() { _files.clear(); } + bool empty() const { return _files.empty() ; } + +private: + void clean() ; + + void save() ; + void load() ; + + // threaded stuff + RsTickingThread mHashingThread ; + RsMutex mHashMtx ; + + // Local configuration and storage + + uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries + std::map mFiles ; + std::string mFilePath ; + bool mChanged ; + + // current work + + std::map mFilesToHash ; };