From a6d467d13851aaad75bd1773309b9be80ea47402 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 18 Jul 2016 21:33:54 -0400 Subject: [PATCH] added interface for p3FileLists --- libretroshare/src/file_sharing/README.txt | 89 ++++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 61 +++++++++++++- 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 19a24db43..9cd98b84c 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -40,9 +40,35 @@ Big picture - the same file should be able to be held by two different directories. Hash search will return a single hit. - the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad! +Directory storage file format +----------------------------- + * should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway) + => works with binary fields + => + + [= version =] + [= peer id =] + [= num entries =] + [= some information =] + + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + ... + 2 1-5 v 2 1-5 v + * entry content + Tag | Content | Size + ----------------+--------------------------------------+------ + 01 | sha1 hash | 20 + 01 | sha1^2 hash | 20 + 02 | file name | < 512 + 03 | file size | 8 + 04 | dir name | < 512 + 05 | last modif time local | 4 + 06 | last modif time including sub-dirs | 4 + Classes ------- - Rs p3ShareManager - tick() @@ -75,6 +101,29 @@ Classes - parent groups - group flags +Best data structure for file index +---------------------------------- + + | Hash map map list + ----------------+-----------------+------------+-------------- + Adding | Constant | log(n) | O(n) + Hash search | Constant | log(n) | O(n) + Name/exp search | O(n) | O(n) | O(n) + Recursive browse| Constant | log(n) | O(n) + + Should we use the same struct for files and directories? + + Sol 1: + DirClass + PersonClass + FileEntry class + - each has pointers to elements list of the same type + - lists are handled for Files (all file entries), + + Directories are represented by the hash of the full path + + Sol 2: + Same class for all elements, in a single hash map. Each element is + defined by its type (Dir, Person, File) which all have a hash. + Syncing between peers --------------------- @@ -83,10 +132,18 @@ Generating sync events - for each directory, in breadth first order - if directory has changed, or last update is old => push a sync request + - store the peer's last up time. Compare with peer uptimes recursively. * Server side - after a change, broadcast a "directory changed" packet to all connected friends - + + * directoy updater + - crawl through directories + - compare TS of files, missing files, new files + - feed a queue of files to hash + - directory whatcher gets notified when files are hashed + + - a separate component hashes files (FileHashingProcess) DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy) | | @@ -100,7 +157,6 @@ Generating sync events +----------- own file list -------+---------- Encrypted/compressed save to disk | | | +----------- friend file lists ---+ - Roadmap ------- @@ -115,12 +171,29 @@ Roadmap TODO ==== - [ ] implement directory updater - [ ] local directory updater - [ ] remote directory updater - [ ] implement directory handler - [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own + [ ] directory handler + [ ] abstract functions to keep a directory and get updates to it. + [ ] hierarchical storage representation. + [ ] allow add/delete entries + [ ] auto-cleanup + + [ ] directory updater + [ ] abstract layer + [ ] crawls the directory and ask updates + + [ ] derive local directory updater + [ ] crawl local files, and asks updates to storage class + [ ] derive remote directory updater + [ ] crawl stored files, and request updates to storage class + + [ ] load/save of directory content. Should be extensible + [ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists + [ ] service (items) for p3FileLists + [ ] connect RemoteDirModel to new system + [ ] test GUI functions + [ ] test update between peers + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 7bddccc53..bca69ca6f 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -19,11 +19,68 @@ // The file lists are not directry updated. A FileListWatcher class is responsible for this // in every case. // -class p3FileLists: public p3Service +#include +#include + +class p3FileLists: public p3Service, public p3Config, public RsSharedFileService { public: - p3FileLists() ; + typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + p3FileLists(mPeerMgr *mpeers) ; + /* + */ + + // derived from p3Service + // + virtual int tick() ; + + // access to own/remote shared files + // + virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; + + virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; + virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + + // Interface for browsing dir hierarchy + // + int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ; + uint32_t getType(const EntryIndex&) const ; + int RequestDirDetails(const std::string& path, DirDetails &details) const ; + + // set/update shared directories + void setSharedDirectories(const std::list& dirs); + void getSharedDirectories(std::list& dirs); + void updateShareFlags(const SharedDirInfo& info) ; + + void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed + bool inDirectoryCheck(); + + // Derived from p3Config + // + private: + p3PeerMgr *mPeers ; + + // File sync request queues. The fast one is used for online browsing when friends are connected. + // The slow one is used for background update of file lists. + // + std::list mFastRequestQueue ; + std::list mSlowRequestQueue ; + + // Directory storage hierarchies + // + std::map mRemoteDirectories ; + + LocalSharedDirectoryMap *mLocalSharedDirs ; + + RemoteSharedDirectoryWatcher *mRemoteDirWatcher ; + LocalSharedDirectoryWatcher *mLocalDirWatcher ; + + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. + // + HashCache *mHashCache ; + + RsMutex mFLSMtx ; };