From 13a41309a7b716d2b4e5d5dc16bda1be527d1472 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Aug 2014 14:22:30 +0000 Subject: [PATCH 01/92] branched for new file list sharing, and getting rid of cache system git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.6-FileLists@7493 b45a01b8-16f6-495d-af2f-9b41ad6348cc From 89af650f7447b2038dd8902b0cfe45c0d6742c55 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 14 Apr 2016 18:25:12 -0400 Subject: [PATCH 02/92] basic bits of new file list sharing system --- libretroshare/src/file_sharing/README.txt | 41 +++++++++++++++++++ .../src/file_sharing/directory_list.h | 13 ++++++ .../src/file_sharing/directory_updater.h | 25 +++++++++++ libretroshare/src/file_sharing/p3filelists.h | 29 +++++++++++++ libretroshare/src/libretroshare.pro | 4 ++ 5 files changed, 112 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_list.h create mode 100644 libretroshare/src/file_sharing/directory_updater.h create mode 100644 libretroshare/src/file_sharing/p3filelists.h diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index f5e298afc..19a24db43 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -87,6 +87,20 @@ Generating sync events * Server side - after a change, broadcast a "directory changed" packet to all connected friends + + DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy) + | | + | | + | | + +-----------------------+------------------+ + | | + Shared File Service | + | | + | | + +----------- own file list -------+---------- Encrypted/compressed save to disk + | | | + +----------- friend file lists ---+ + Roadmap ------- @@ -99,3 +113,30 @@ Roadmap - optionally - change the saving system of FileIndex to make it locally encrypted and compact +TODO +==== + [ ] implement directory updater + [ ] local directory updater + [ ] remote directory updater + + [ ] implement directory handler + [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own + + + + + + + + + + + + + + + + + + + diff --git a/libretroshare/src/file_sharing/directory_list.h b/libretroshare/src/file_sharing/directory_list.h new file mode 100644 index 000000000..c643b6153 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_list.h @@ -0,0 +1,13 @@ +// This class keeps a shared directory. It's quite the equivalent of the old "FileIndex" class +// The main difference is that it is +// - extensible +// - fast to search (at least for hashes). Should provide possibly multiple search handles for +// the same file, e.g. if connexion is encrypted. +// - abstracts the browsing in a same manner. +// +class SharedDirectoryList +{ + public: + + DirEntry mRoot ; +}; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h new file mode 100644 index 000000000..be59eb987 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -0,0 +1,25 @@ +// This class crawls the given directry hierarchy and updates it. It does so by calling the +// shared file list source. This source may be of two types: +// - local: directories are crawled n disk and files are hashed / requested from a cache +// - remote: directories are requested remotely to a providing client +// +class DirectoryUpdater +{ + public: + DirectoryUpdater() ; + + // Does some updating job. Crawls the existing directories and checks wether it has been updated + // recently enough. If not, calls the directry source. + // + void tick() ; + + // +}; + +class LocalDirectoryUpdater: public DirectoryUpdater +{ +}; + +class RemoteDirectoryUpdater: public DirectoryUpdater +{ +}; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h new file mode 100644 index 000000000..7bddccc53 --- /dev/null +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -0,0 +1,29 @@ +// This class is responsible for +// - maintaining a list of shared file hierarchies for each known friends +// - talking to the GUI +// - providing handles for the directory tree listing GUI +// - providing search handles for FT +// - keeping these lists up to date +// - sending our own file list to friends depending on the defined access rights +// - serving file search requests from other services such as file transfer +// +// p3FileList does the following micro-tasks: +// - tick the watchers +// - get incoming info from the service layer, which can be: +// - directory content request => the directory content is shared to the friend +// - directory content => the directory watcher is notified +// - keep two queues of update requests: +// - fast queue that is handled in highest priority. This one is used for e.g. updating while browsing. +// - slow queue that is handled slowly. Used in background update of shared directories. +// +// The file lists are not directry updated. A FileListWatcher class is responsible for this +// in every case. +// +class p3FileLists: public p3Service +{ + public: + p3FileLists() ; + + +}; + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index d01d4c9dd..13b315713 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -354,6 +354,10 @@ HEADERS += ft/ftchunkmap.h \ ft/fttransfermodule.h \ ft/ftturtlefiletransferitem.h +HEADERS += directory_updater.h \ + directory_list.h \ + p3filelists.h + HEADERS += chat/distantchat.h \ chat/p3chatservice.h \ chat/distributedchat.h \ From a6d467d13851aaad75bd1773309b9be80ea47402 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 18 Jul 2016 21:33:54 -0400 Subject: [PATCH 03/92] added interface for p3FileLists --- libretroshare/src/file_sharing/README.txt | 89 ++++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 61 +++++++++++++- 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 19a24db43..9cd98b84c 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -40,9 +40,35 @@ Big picture - the same file should be able to be held by two different directories. Hash search will return a single hit. - the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad! +Directory storage file format +----------------------------- + * should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway) + => works with binary fields + => + + [= version =] + [= peer id =] + [= num entries =] + [= some information =] + + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + ... + 2 1-5 v 2 1-5 v + * entry content + Tag | Content | Size + ----------------+--------------------------------------+------ + 01 | sha1 hash | 20 + 01 | sha1^2 hash | 20 + 02 | file name | < 512 + 03 | file size | 8 + 04 | dir name | < 512 + 05 | last modif time local | 4 + 06 | last modif time including sub-dirs | 4 + Classes ------- - Rs p3ShareManager - tick() @@ -75,6 +101,29 @@ Classes - parent groups - group flags +Best data structure for file index +---------------------------------- + + | Hash map map list + ----------------+-----------------+------------+-------------- + Adding | Constant | log(n) | O(n) + Hash search | Constant | log(n) | O(n) + Name/exp search | O(n) | O(n) | O(n) + Recursive browse| Constant | log(n) | O(n) + + Should we use the same struct for files and directories? + + Sol 1: + DirClass + PersonClass + FileEntry class + - each has pointers to elements list of the same type + - lists are handled for Files (all file entries), + + Directories are represented by the hash of the full path + + Sol 2: + Same class for all elements, in a single hash map. Each element is + defined by its type (Dir, Person, File) which all have a hash. + Syncing between peers --------------------- @@ -83,10 +132,18 @@ Generating sync events - for each directory, in breadth first order - if directory has changed, or last update is old => push a sync request + - store the peer's last up time. Compare with peer uptimes recursively. * Server side - after a change, broadcast a "directory changed" packet to all connected friends - + + * directoy updater + - crawl through directories + - compare TS of files, missing files, new files + - feed a queue of files to hash + - directory whatcher gets notified when files are hashed + + - a separate component hashes files (FileHashingProcess) DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy) | | @@ -100,7 +157,6 @@ Generating sync events +----------- own file list -------+---------- Encrypted/compressed save to disk | | | +----------- friend file lists ---+ - Roadmap ------- @@ -115,12 +171,29 @@ Roadmap TODO ==== - [ ] implement directory updater - [ ] local directory updater - [ ] remote directory updater - [ ] implement directory handler - [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own + [ ] directory handler + [ ] abstract functions to keep a directory and get updates to it. + [ ] hierarchical storage representation. + [ ] allow add/delete entries + [ ] auto-cleanup + + [ ] directory updater + [ ] abstract layer + [ ] crawls the directory and ask updates + + [ ] derive local directory updater + [ ] crawl local files, and asks updates to storage class + [ ] derive remote directory updater + [ ] crawl stored files, and request updates to storage class + + [ ] load/save of directory content. Should be extensible + [ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists + [ ] service (items) for p3FileLists + [ ] connect RemoteDirModel to new system + [ ] test GUI functions + [ ] test update between peers + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 7bddccc53..bca69ca6f 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -19,11 +19,68 @@ // The file lists are not directry updated. A FileListWatcher class is responsible for this // in every case. // -class p3FileLists: public p3Service +#include +#include + +class p3FileLists: public p3Service, public p3Config, public RsSharedFileService { public: - p3FileLists() ; + typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + p3FileLists(mPeerMgr *mpeers) ; + /* + */ + + // derived from p3Service + // + virtual int tick() ; + + // access to own/remote shared files + // + virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; + + virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; + virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + + // Interface for browsing dir hierarchy + // + int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ; + uint32_t getType(const EntryIndex&) const ; + int RequestDirDetails(const std::string& path, DirDetails &details) const ; + + // set/update shared directories + void setSharedDirectories(const std::list& dirs); + void getSharedDirectories(std::list& dirs); + void updateShareFlags(const SharedDirInfo& info) ; + + void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed + bool inDirectoryCheck(); + + // Derived from p3Config + // + private: + p3PeerMgr *mPeers ; + + // File sync request queues. The fast one is used for online browsing when friends are connected. + // The slow one is used for background update of file lists. + // + std::list mFastRequestQueue ; + std::list mSlowRequestQueue ; + + // Directory storage hierarchies + // + std::map mRemoteDirectories ; + + LocalSharedDirectoryMap *mLocalSharedDirs ; + + RemoteSharedDirectoryWatcher *mRemoteDirWatcher ; + LocalSharedDirectoryWatcher *mLocalDirWatcher ; + + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. + // + HashCache *mHashCache ; + + RsMutex mFLSMtx ; }; From b31e396c2bb577811d66a26f1d496845997c52c6 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 18 Jul 2016 21:52:44 -0400 Subject: [PATCH 04/92] added skeleton code for p3filelists, and interface files for directory storage and hash cache --- .../src/file_sharing/directory_storage.h | 65 ++++++++++ libretroshare/src/file_sharing/hash_cache.h | 29 +++++ libretroshare/src/file_sharing/p3filelists.cc | 113 ++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_storage.h create mode 100644 libretroshare/src/file_sharing/hash_cache.h create mode 100644 libretroshare/src/file_sharing/p3filelists.cc diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h new file mode 100644 index 000000000..1ba46a062 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -0,0 +1,65 @@ +#include + +static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; + +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; +static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; +static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; +static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; + +class DirectoryStorage +{ + public: + DirectoryStorage(const std::string& local_file_name) ; + + void save() const ; + + virtual int searchTerms(const std::list& terms, std::list &results) const; + virtual int searchHash(const RsFileHash& hash, std::list &results) const; + virtual int searchBoolExp(Expression * exp, std::list &results) const; + + void getFileDetails(EntryIndex i) ; + + // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. + // It gives access to sub-files and sub-directories below. + // + class DirIterator + { + public: + DirIterator(const DirectoryStorage& d) ; + + DirIterator& operator++() ; + EntryIndex operator*() const ; // current directory entry + + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + }; + class FileIterator + { + public: + FileIterator(const DirectoryStorage& d) ; + + FileIterator& operator++() ; + EntryIndex operator*() const ; // current file entry + + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + }; + + private: + void load(const std::string& local_file_name) ; + void save(const std::string& local_file_name) ; + + void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ; + void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; +}; + +class RemoteDirectoryStorage: public DirectoryStorage +{ +}; + +class LocalDirectoryStorage: public DirectoryStorage +{ +}; + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h new file mode 100644 index 000000000..144e6d8ee --- /dev/null +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -0,0 +1,29 @@ +class HashCache +{ + public: + HashCache(const std::string& save_file_name) ; + + void save() ; + void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; + bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; + void clean() ; + + typedef struct + { + uint64_t size ; + uint64_t time_stamp ; + uint64_t modf_stamp ; + RsFileHash hash ; + } HashCacheInfo ; + + void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } + uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } + void clear() { _files.clear(); } + bool empty() const { return _files.empty() ; } + private: + uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries + std::map _files ; + std::string _path ; + bool _changed ; +}; + diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc new file mode 100644 index 000000000..baf87dbf3 --- /dev/null +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -0,0 +1,113 @@ +#include +#include + +#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; + +static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; + +p3FileLists::p3FileLists(mPeerPgr *mpeers) + : mPeers(mpeers) +{ + // loads existing indexes for friends. Some might be already present here. + // + + // init the directory watchers + + // local variables/structures + // + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; +} +void p3FileLists::~p3FileLists() +{ + // delete all pointers +} + +void p3FileLists::tick() +{ + // tick the watchers, possibly create new ones if additional friends do connect. + // + tickWatchers(); + + // tick the input/output list of update items and process them + // + tickRecv() ; + tickSend() ; + + // cleanup + // - remove/delete shared file lists for people who are not friend anymore + // - + + cleanup(); +} + +void p3FileLists::tickRecv() +{ +} +void p3FileLists::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. +} + +void p3FileLists::loadList(std::list& items) +{ + // This loads + // + // - list of locally shared directories, and the permissions that go with them +} + +void p3FileLists::saveList(const std::list& items) +{ +} + +void p3FileLists::cleanup() +{ + // look through the list of friend directories. Remove those who are not our friends anymore. + // + P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; + + std::set friend_set ; + { +#warning we should use a std::set in the first place. Modify rsPeers? + std::list friend_list ; + mPeers->getFriendList(friend_list) ; + + for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) + friend_set.insert(*it) ; + } + + for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) + if(friend_set.find(it->first) == friend_set.end()) + { + P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; + + delete it->second ; + std::map::iterator tmp(it) ; + ++tmp ; + mRemoteDirectories.erase(it) ; + it=tmp ; + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } + else + ++it ; + + // look through the list of friends, and add a directory storage when it's missing + // + for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) + if(mRemoteDirectories.find(*it) == mRemoteDirectories.end()) + { + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + + mRemoteDirectories[*it] = new RemoteDirectoryStorage(*it); + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } + + if(mUpdateFlags) + IndicateConfigChanged(); +} + + From 3e20b9ddf74ad3f292c30b941c12b9ed0592fe9e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 15:28:55 -0400 Subject: [PATCH 05/92] added p3FileLists service --- libretroshare/src/file_sharing/p3filelists.cc | 4 ++-- libretroshare/src/file_sharing/p3filelists.h | 4 +++- libretroshare/src/libretroshare.pro | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index baf87dbf3..20fd14db6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,5 +1,5 @@ -#include -#include +#include "file_sharing/p3filelists.h" +#include "retroshare/rsids.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index bca69ca6f..f0a6e2a1d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -19,10 +19,12 @@ // The file lists are not directry updated. A FileListWatcher class is responsible for this // in every case. // +#pragma once + #include #include -class p3FileLists: public p3Service, public p3Config, public RsSharedFileService +class p3FileLists: public p3Service, public p3Config, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 94c527b73..b7a113fd7 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -33,6 +33,20 @@ debug { QMAKE_CXXFLAGS *= -g -fno-omit-frame-pointer } +CONFIG += file_lists + +file_lists { + HEADERS *= file_sharing/p3filelists.h \ + file_sharing/hash_cache.h \ + file_sharing/directory_storage.h \ + file_sharing/directory_updater.h + + SOURCES *= file_sharing/p3filelists.cc \ + file_sharing/hash_cache.cc \ + file_sharing/directory_storage.cc \ + file_sharing/directory_updater.cc +} + dsdv { DEFINES *= SERVICES_DSDV HEADERS += services/p3dsdv.h \ From 63063102bb5e6076e14d2a3554087dc355f54c1d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 16:10:51 -0400 Subject: [PATCH 06/92] fixed compilation of file lists code base --- .gitignore | 3 +- .../src/file_sharing/directory_storage.cc | 0 .../src/file_sharing/directory_storage.h | 15 ++++++- .../src/file_sharing/directory_updater.h | 6 ++- libretroshare/src/file_sharing/hash_cache.cc | 0 libretroshare/src/file_sharing/p3filelists.cc | 35 +++++++++++----- libretroshare/src/file_sharing/p3filelists.h | 40 ++++++++++++++++--- 7 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 libretroshare/src/file_sharing/directory_storage.cc create mode 100644 libretroshare/src/file_sharing/hash_cache.cc diff --git a/.gitignore b/.gitignore index eb9b055b3..b799c1eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ moc_*.cpp qrc_*.cpp ui_*.h -Makefile.* +Makefile* +*.sw? *~ Thumbs.db diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc new file mode 100644 index 000000000..e69de29bb diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 1ba46a062..6106e4bb6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,8 +1,12 @@ #include +#include +#include + +#include "retroshare/rsids.h" +#include "retroshare/rsfiles.h" static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; @@ -14,6 +18,9 @@ class DirectoryStorage { public: DirectoryStorage(const std::string& local_file_name) ; + virtual ~DirectoryStorage() {} + + typedef uint32_t EntryIndex ; void save() const ; @@ -57,9 +64,15 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { +public: + RemoteDirectoryStorage(const RsPeerId& pid) ; + virtual ~RemoteDirectoryStorage() {} }; class LocalDirectoryStorage: public DirectoryStorage { +public: + LocalDirectoryStorage() ; + virtual ~LocalDirectoryStorage() {} }; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index be59eb987..3571247ab 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -11,15 +11,19 @@ class DirectoryUpdater // Does some updating job. Crawls the existing directories and checks wether it has been updated // recently enough. If not, calls the directry source. // - void tick() ; + virtual void tick() =0; // }; class LocalDirectoryUpdater: public DirectoryUpdater { + public: + virtual void tick() ; }; class RemoteDirectoryUpdater: public DirectoryUpdater { + public: + virtual void tick() ; }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc new file mode 100644 index 000000000..e69de29bb diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 20fd14db6..7c35fe332 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,15 +1,16 @@ #include "file_sharing/p3filelists.h" +#include "file_sharing/directory_storage.h" #include "retroshare/rsids.h" -#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; +#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -p3FileLists::p3FileLists(mPeerPgr *mpeers) - : mPeers(mpeers) +p3FileLists::p3FileLists(p3LinkMgr *mpeers) + : mLinkMgr(mpeers), mFLSMtx("p3FileLists") { // loads existing indexes for friends. Some might be already present here. // @@ -20,12 +21,19 @@ p3FileLists::p3FileLists(mPeerPgr *mpeers) // mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -void p3FileLists::~p3FileLists() +p3FileLists::~p3FileLists() { + RS_STACK_MUTEX(mFLSMtx) ; + + for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();++it) + delete it->second ; + + mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around. + // delete all pointers } -void p3FileLists::tick() +int p3FileLists::tick() { // tick the watchers, possibly create new ones if additional friends do connect. // @@ -41,6 +49,8 @@ void p3FileLists::tick() // - cleanup(); + + return 0; } void p3FileLists::tickRecv() @@ -51,20 +61,25 @@ void p3FileLists::tickSend() // go through the list of out requests and send them to the corresponding friends, if they are online. } -void p3FileLists::loadList(std::list& items) +bool p3FileLists::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + + return true ; } -void p3FileLists::saveList(const std::list& items) +bool p3FileLists::saveList(const std::list& items) { + return true ; } void p3FileLists::cleanup() { - // look through the list of friend directories. Remove those who are not our friends anymore. + RS_STACK_MUTEX(mFLSMtx) ; + + // look through the list of friend directories. Remove those who are not our friends anymore. // P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; @@ -72,13 +87,13 @@ void p3FileLists::cleanup() { #warning we should use a std::set in the first place. Modify rsPeers? std::list friend_list ; - mPeers->getFriendList(friend_list) ; + mLinkMgr->getFriendList(friend_list) ; for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) friend_set.insert(*it) ; } - for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) + for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) if(friend_set.find(it->first) == friend_set.end()) { P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f0a6e2a1d..e2c9d2016 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -21,15 +21,32 @@ // #pragma once -#include -#include +#include "retroshare/rsfiles.h" +#include "services/p3service.h" -class p3FileLists: public p3Service, public p3Config, public RsSharedFileService +#include "pqi/p3cfgmgr.h" +#include "pqi/p3linkmgr.h" + +class RemoteDirectoryStorage ; +class RemoteSharedDirectoryWatcher ; +class LocalSharedDirectoryWatcher ; +class LocalSharedDirectoryMap ; +class HashCache ; + +class p3FileLists: public p3Service, public p3Config //, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere - p3FileLists(mPeerMgr *mpeers) ; + struct RsFileListSyncRequest + { + RsPeerId peerId ; + EntryIndex index ; + // [...] more to add here + }; + + p3FileLists(p3LinkMgr *mpeers) ; + ~p3FileLists(); /* */ @@ -61,8 +78,16 @@ class p3FileLists: public p3Service, public p3Config, public RsSharedFileService // Derived from p3Config // - private: - p3PeerMgr *mPeers ; + protected: + virtual bool loadList(std::list& items); + virtual bool saveList(const std::list& items); + void cleanup(); + void tickRecv(); + void tickSend(); + void tickWatchers(); + + private: + p3LinkMgr *mLinkMgr ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -83,6 +108,9 @@ class p3FileLists: public p3Service, public p3Config, public RsSharedFileService // HashCache *mHashCache ; + // Local flags and mutexes + RsMutex mFLSMtx ; + uint32_t mUpdateFlags ; }; From 3bf3d0c360848e78b48fd64a6ddd970ce909ac48 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 16:11:26 -0400 Subject: [PATCH 07/92] added blank cc file --- .../src/file_sharing/directory_updater.cc | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_updater.cc diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc new file mode 100644 index 000000000..72917248b --- /dev/null +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -0,0 +1,35 @@ +#include "directory_storage.h" +#include "directory_updater.h" + +void RemoteDirectoryUpdater::tick() +{ + // use the stored iterator +} + +void LocalDirectoryUpdater::tick() +{ +#ifdef TODO + // recursive update algorithm works that way: + // - the external loop starts on the shared directory list and goes through sub-directories + // - at the same time, it updates the list of shared directories + // - the information that is costly to compute (the hash) is store externally into a separate structure. + // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. + // + for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it) + { + librs::util::FolderIterator dirIt(realpath); + if(!dirIt.isValid()) + mLocalSharedDirs.removeDirectory(it) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + + while(dirIt.readdir()) + { + + } + + if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash)) + { + ; + } + } +#endif +} From 3c976bb7ee181d4183d43355c7f530f26cc7318d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 21 Jul 2016 00:16:12 -0400 Subject: [PATCH 08/92] added squeleton code for own directory update --- libretroshare/src/file_sharing/README.txt | 16 +-- .../src/file_sharing/directory_storage.cc | 60 +++++++++++ .../src/file_sharing/directory_storage.h | 48 +++++++-- .../src/file_sharing/directory_updater.cc | 101 +++++++++++++++--- .../src/file_sharing/directory_updater.h | 12 ++- libretroshare/src/file_sharing/hash_cache.h | 65 +++++++---- 6 files changed, 248 insertions(+), 54 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 9cd98b84c..1b6cccb2b 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -160,14 +160,14 @@ Generating sync events Roadmap ------- -- complete this file until a proper description of the whole thing is achieved. -- create a new directory and implement the .h for the basic functionality -- look into existing code in ftServer for the integration, but don't change anything yet -- setup class hierarchy -- merge hash cache into file lists. - -- optionally - - change the saving system of FileIndex to make it locally encrypted and compact +[X] complete this file until a proper description of the whole thing is achieved. +[X] create a new directory and implement the .h for the basic functionality +[ ] look into existing code in ftServer for the integration, but don't change anything yet +[X] setup class hierarchy +[ ] merge hash cache into file lists. +[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible +[ ] create basic directory functionality with own files: re-hash, and store +[ ] display own files in GUI, with proper update and working sort TODO ==== diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e69de29bb..acfdcbf25 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -0,0 +1,60 @@ +#include "directory_storage.h" + +class InternalFileHierarchyStorage +{ + class FileStorageNode + { + public: + virtual uint32_t type() const =0; + }; + class FileEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + + }; + + class DirEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } + + std::set subdirs ; + std::set subfiles ; + }; + + // file/dir access and modification + bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + + uint32_t root ; + + std::vector mNodes;// uses pointers to keep information about valid/invalid objects. + + void compress() ; // use empty space in the vector, mostly due to deleted entries. + + friend class DirectoryStorage ; // only class that can use this. +}; + +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; + +void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) +{ + entry_tag = data[offset++] ; +} +void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size) +{ +} + +void DirectoryStorage::load(const std::string& local_file_name) +{ + // first load the header, than all fields. +} +void DirectoryStorage::save(const std::string& local_file_name) +{ + // first write the header, than all fields. +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 6106e4bb6..0399d9d9e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -14,6 +14,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; +class InternalFileHierarchyStorage ; + class DirectoryStorage { public: @@ -36,30 +38,37 @@ class DirectoryStorage class DirIterator { public: - DirIterator(const DirectoryStorage& d) ; + DirIterator(const DirIterator& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; class FileIterator { public: - FileIterator(const DirectoryStorage& d) ; + FileIterator(DirIterator& d); // crawls all files in specified directory + uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; + virtual DirIterator root() ; // returns the index of the root directory entry. + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; - void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ; - void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; + void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. + + InternalFileHierarchyStorage *mFileHierarchy ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -74,5 +83,32 @@ class LocalDirectoryStorage: public DirectoryStorage public: LocalDirectoryStorage() ; virtual ~LocalDirectoryStorage() {} + + void setSharedDirectoryList(const std::list& lst) ; + void getSharedDirectoryList(std::list& lst) ; + + /*! + * \brief addFile + * \param dir + * \param hash + * \param modf_time + */ + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + +private: + std::list mLocalDirs ; + + std::map mHashes ; // used for fast search access }; + + + + + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 72917248b..c20c3703b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,30 +6,99 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } +LocalDirectoryUpdater::LocalDirectoryUpdater() +{ + // tell the hash cache that we're the client + mHashCache->setClient(this) ; +} + void LocalDirectoryUpdater::tick() { -#ifdef TODO // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories - // - at the same time, it updates the list of shared directories + // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. // - the information that is costly to compute (the hash) is store externally into a separate structure. // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // - for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it) - { - librs::util::FolderIterator dirIt(realpath); - if(!dirIt.isValid()) - mLocalSharedDirs.removeDirectory(it) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + std::list shared_directory_list ; - while(dirIt.readdir()) - { + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) + sub_dir_list.push_back( (*it).filename ) ; - } + // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash)) - { - ; - } - } -#endif + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + + // now for each of them, go recursively and match both files and dirs + + DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } + +LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +{ + // make sure list of subdirs is the same + + // make sure list of subfiles is the same + + // request all hashes to the hashcache + + librs::util::FolderIterator dirIt(cumulated_path); + + if(!dirIt.isValid()) + { + mSharedDirectories->removeDirectory(indx) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + return ; + } + + // collect subdirs and subfiles + + std::list subfiles ; + std::list subdirs ; + + while(dirIt.readdir()) + { + std::string name ; + uint64_t size ; + uint8_t type ; + time_t mod_time ; + + dirIt.readEntryInformation(type,name,size,mod_time) ; + + switch(type) + { + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; + break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + break; + + default: + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + } + } + // update file and dir lists for current directory. + + mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + // now go through list of subfiles and request the hash to hashcache + + for(DirectoryStorage::FileIterator dit(indx);dit;++dit) + { + // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + + if() + if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) + mSharedDirectories->updateFileHash(*dit,hash) ; + } + + // go through the list of sub-dirs and recursively update + + DirectoryStorage::DirIterator stored_dir_it(indx) ; + + for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; +} + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 3571247ab..54b70f30c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,6 +3,9 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // +class HashCache ; +class LocalDirectoryStorage ; + class DirectoryUpdater { public: @@ -18,8 +21,13 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater { - public: - virtual void tick() ; +public: + LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual void tick() ; + +private: + HashCache *mHashCache ; + LocalDirectoryStorage *mSharedDirectories ; }; class RemoteDirectoryUpdater: public DirectoryUpdater diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 144e6d8ee..6734310fe 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -1,29 +1,50 @@ +#pragma once + +#include +#include "util/rsthreads.h" + class HashCache { - public: - HashCache(const std::string& save_file_name) ; +public: + HashCache(const std::string& save_file_name) ; - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; + void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; + bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; + struct HashCacheInfo + { + std::string filename ; + uint64_t size ; + uint32_t time_stamp ; + uint32_t modf_stamp ; + RsFileHash hash ; + } ; - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; + // interaction with GUI, called from p3FileLists + void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } + uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } + void clear() { _files.clear(); } + bool empty() const { return _files.empty() ; } + +private: + void clean() ; + + void save() ; + void load() ; + + // threaded stuff + RsTickingThread mHashingThread ; + RsMutex mHashMtx ; + + // Local configuration and storage + + uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries + std::map mFiles ; + std::string mFilePath ; + bool mChanged ; + + // current work + + std::map mFilesToHash ; }; From 3e48b0fd9515f80fddafa23ddc9f664ae53a0133 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 23 Jul 2016 22:14:43 -0400 Subject: [PATCH 09/92] improved FolderIterator class --- libretroshare/src/dbase/fimonitor.cc | 2 +- .../src/file_sharing/directory_storage.cc | 9 ++- .../src/file_sharing/directory_storage.h | 29 +++++-- .../src/file_sharing/directory_updater.cc | 58 +++++++------- .../src/file_sharing/directory_updater.h | 9 ++- libretroshare/src/file_sharing/hash_cache.h | 30 +++++-- libretroshare/src/plugins/pluginmanager.cc | 2 +- libretroshare/src/rsserver/rsaccounts.cc | 2 +- libretroshare/src/util/folderiterator.cc | 79 +++++++++++++++++-- libretroshare/src/util/folderiterator.h | 23 +++++- retroshare-nogui/src/introserver.cc | 2 +- 11 files changed, 183 insertions(+), 62 deletions(-) diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index c4f60ee81..56b1e7bc2 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -789,7 +789,7 @@ void FileIndexMonitor::updateCycle() to_hash.back().realpath = realpath ; to_hash.back().dirpath = dirpath ; - while(isRunning() && dirIt.readdir()) + for(;dirIt.isValid() && isRunning();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index acfdcbf25..0b7e100e4 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -5,6 +5,9 @@ class InternalFileHierarchyStorage class FileStorageNode { public: + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; + virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode @@ -19,12 +22,12 @@ class InternalFileHierarchyStorage public: virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + std::set subdirs ; + std::set subfiles ; }; // file/dir access and modification - bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e uint32_t root ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0399d9d9e..0fb94037a 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -39,27 +39,44 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; + DirIterator(const EntryIndex& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the directory that is pointed by the iterator + + const std::string& name() const ; + }; class FileIterator { public: FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(EntryIndex& e); // crawls all files in specified directory - uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the file that is pointed by the iterator + + std::string name() const ; + std::string fullpath() const ; + uint64_t size() const ; + RsFileHash hash() const ; + time_t modtime() const ; + }; virtual DirIterator root() ; // returns the index of the root directory entry. - private: + void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void removeDirectory(const EntryIndex& indx) ; + + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c20c3703b..c88820fb0 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,19 +1,24 @@ +#include "util/folderiterator.h" + #include "directory_storage.h" #include "directory_updater.h" +#define DEBUG_LOCAL_DIR_UPDATER 1 + void RemoteDirectoryUpdater::tick() { // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater() +LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc) + : mHashCache(hc) { - // tell the hash cache that we're the client - mHashCache->setClient(this) ; } void LocalDirectoryUpdater::tick() { + std::cerr << "LocalDirectoryUpdater::tick()" << std::endl; + // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. @@ -21,28 +26,29 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; + std::list sub_dir_list ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) - sub_dir_list.push_back( (*it).filename ) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) + sub_dir_list.push_back( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ; // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; } -LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) { + std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl; + // make sure list of subdirs is the same - // make sure list of subfiles is the same - // request all hashes to the hashcache librs::util::FolderIterator dirIt(cumulated_path); @@ -60,22 +66,18 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c while(dirIt.readdir()) { - std::string name ; - uint64_t size ; - uint8_t type ; - time_t mod_time ; - - dirIt.readEntryInformation(type,name,size,mod_time) ; - - switch(type) + switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; - break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + break; + + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; default: - std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } } // update file and dir lists for current directory. @@ -87,18 +89,16 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c for(DirectoryStorage::FileIterator dit(indx);dit;++dit) { - // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - if() - if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) - mSharedDirectories->updateFileHash(*dit,hash) ; + mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ; } // go through the list of sub-dirs and recursively update DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 54b70f30c..35f069b3c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,7 +3,8 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // -class HashCache ; +#include "file_sharing/hash_cache.h" + class LocalDirectoryStorage ; class DirectoryUpdater @@ -19,12 +20,16 @@ class DirectoryUpdater // }; -class LocalDirectoryUpdater: public DirectoryUpdater +class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; virtual void tick() ; +protected: + virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ; + void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); + private: HashCache *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6734310fe..f867f6109 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -2,14 +2,28 @@ #include #include "util/rsthreads.h" +#include "retroshare/rsfiles.h" + +class HashCacheClient +{ +public: + virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ; +}; + +class FileHashingThread: public RsTickingThread +{ +public: + FileHashingThread() {} + + virtual void data_tick() ; +}; class HashCache { public: HashCache(const std::string& save_file_name) ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; + bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ; struct HashCacheInfo { @@ -21,10 +35,10 @@ public: } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; } + uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; } + void clear() { mFiles.clear(); } + bool empty() const { return mFiles.empty() ; } private: void clean() ; @@ -33,7 +47,7 @@ private: void load() ; // threaded stuff - RsTickingThread mHashingThread ; + FileHashingThread mHashingThread ; RsMutex mHashMtx ; // Local configuration and storage @@ -45,6 +59,6 @@ private: // current work - std::map mFilesToHash ; + std::map mFilesToHash ; }; diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 7d101fdf3..31ee2a39f 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -146,7 +146,7 @@ void RsPluginManager::loadPlugins(const std::vector& plugin_directo continue ; } - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { std::string fname; dirIt.d_name(fname); diff --git a/libretroshare/src/rsserver/rsaccounts.cc b/libretroshare/src/rsserver/rsaccounts.cc index ad9e1306e..9bb8b49ed 100644 --- a/libretroshare/src/rsserver/rsaccounts.cc +++ b/libretroshare/src/rsserver/rsaccounts.cc @@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map & struct stat64 buf; - while (dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 01ff94461..747ec93d9 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -1,11 +1,17 @@ +#include +#include +#include +#include +#include + #include "folderiterator.h" #include "rsstring.h" - namespace librs { namespace util { FolderIterator::FolderIterator(const std::string& folderName) + : mFolderName(folderName) { #ifdef WINDOWS_SYS std::wstring utf16Name; @@ -18,10 +24,10 @@ FolderIterator::FolderIterator(const std::string& folderName) handle = FindFirstFileW(utf16Name.c_str(), &fileInfo); validity = handle != INVALID_HANDLE_VALUE; - isFirstCall = true; #else handle = opendir(folderName.c_str()); validity = handle != NULL; + next(); #endif } @@ -30,19 +36,71 @@ FolderIterator::~FolderIterator() closedir(); } -bool FolderIterator::readdir() { +void FolderIterator::next() +{ + do { + if(!readdir()) + { + validity = false ; + break ; + } + + d_name(mFileName); + } while(mFileName == "." || mFileName == "..") ; + + mFullPath = mFolderName + "/" + mFileName ; + + struct stat64 buf ; + +#ifdef WINDOWS_SYS + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) +#else + if ( 0 == stat64(mFullPath.c_str(), &buf)) +#endif + { + mFileModTime = buf.st_mtime ; + mStatInfoOk = true; + + if (S_ISDIR(buf.st_mode)) + { + mType = TYPE_DIR ; + mFileSize = 0 ; + mFileModTime = buf.st_mtime; + } + else if (S_ISREG(buf.st_mode)) + { + mType = TYPE_FILE ; + mFileSize = buf.st_size; + mFileModTime = buf.st_mtime; + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + } + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + validity = false ; + } +} + +bool FolderIterator::readdir() +{ if(!validity) return false; #ifdef WINDOWS_SYS - if(isFirstCall) { - isFirstCall = false; - return true; - } return FindNextFileW(handle, &fileInfo) != 0; #else ent = ::readdir(handle); - return ent != 0; + return ent != NULL; #endif } @@ -65,6 +123,11 @@ bool FolderIterator::d_name(std::string& dest) return true; } +const std::string& FolderIterator::file_fullpath() { return mFullPath ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } + bool FolderIterator::closedir() { if(!validity) diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 8007db1a1..823aa9a9a 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -2,6 +2,7 @@ #define FOLDERITERATOR_H +#include #include #include @@ -24,14 +25,24 @@ public: FolderIterator(const std::string& folderName); ~FolderIterator(); - bool isValid() const { return validity; } + enum { TYPE_UNKNOWN = 0x00, + TYPE_FILE = 0x01, + TYPE_DIR = 0x02 + }; + bool isValid() const { return validity; } bool readdir(); + void next(); bool d_name(std::string& dest); - bool closedir(); + const std::string& file_name() ; + const std::string& file_fullpath() ; + uint64_t file_size() ; + uint64_t file_type() ; + time_t file_modtime() ; + private: bool validity; @@ -43,7 +54,15 @@ private: DIR* handle; struct dirent* ent; #endif + void updateStatsInfo() ; + bool mStatInfoOk ; + time_t mFileModTime ; + uint64_t mFileSize ; + uint8_t mType ; + std::string mFileName ; + std::string mFullPath ; + std::string mFolderName ; }; diff --git a/retroshare-nogui/src/introserver.cc b/retroshare-nogui/src/introserver.cc index 5445dc814..3734154b2 100644 --- a/retroshare-nogui/src/introserver.cc +++ b/retroshare-nogui/src/introserver.cc @@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts() mCertCheckTime = now; struct stat64 buf; - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname; From 25c824fd5f05c9a1b9d9b66b8945fc2a766419c6 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 24 Jul 2016 23:48:22 -0400 Subject: [PATCH 10/92] added iterators for directory storage class --- .../src/file_sharing/directory_storage.cc | 268 +++++++++++++++++- .../src/file_sharing/directory_storage.h | 50 ++-- .../src/file_sharing/directory_updater.cc | 20 +- .../src/file_sharing/directory_updater.h | 11 +- libretroshare/src/util/folderiterator.cc | 7 +- libretroshare/src/util/folderiterator.h | 2 +- 6 files changed, 320 insertions(+), 38 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 0b7e100e4..a955df832 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,22 @@ +#include #include "directory_storage.h" +/******************************************************************************************************************/ +/* Internal File Hierarchy Storage */ +/******************************************************************************************************************/ + +template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) +{ + typename std::set::iterator tmp(it); + ++tmp; + s.erase(it) ; + return tmp; +} + +// This class handles the file hierarchy +// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, +// so that the hierarchy stays completely coherent between calls. + class InternalFileHierarchyStorage { class FileStorageNode @@ -8,36 +25,279 @@ class InternalFileHierarchyStorage static const uint32_t TYPE_FILE = 0x0001 ; static const uint32_t TYPE_DIR = 0x0002 ; + virtual ~FileStorageNode() {} virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode { public: + FileEntry(const std::string& name) : file_name(name) {} virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + virtual ~FileEntry() {} + // local stuff + std::string file_name ; + uint64_t file_size ; + RsFileHash file_hash ; + time_t file_modtime ; }; class DirEntry: public FileStorageNode { public: + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + virtual ~DirEntry() {} + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + // local stuff + std::string dir_name ; + DirectoryStorage::EntryIndex parent_index; + + std::vector subdirs ; + std::vector subfiles ; }; + // class stuff + InternalFileHierarchyStorage() : mRoot(0) + { + mNodes.push_back(new DirEntry("",0)) ; + } + + // high level modification routines + + bool isIndexValid(DirectoryStorage::EntryIndex e) + { + return e < mNodes.size() && mNodes[e] != NULL ; + } + + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubDirectoryList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + + std::set should_create(subdirs); + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) + removeDirectory(d.subdirs[i]) ; + else + { + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; + ++i; + } + + for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) + { + d.subdirs.push_back(mNodes.size()) ; + mNodes.push_back(new DirEntry(*it,indx)); + } + + return true; + } + bool removeDirectory(const DirectoryStorage::EntryIndex& indx) + { + // check that it's a directory + + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("removeDirectory(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("removeDirectory(): Node is not a directory") ; + + if(indx == 0) + return nodeAccessError("removeDirectory(): Cannot remove top level directory") ; + + // remove from parent + + DirEntry& d(*static_cast(mNodes[indx])) ; + DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); + + for(uint32_t i=0;i& subfiles,std::set& new_files) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubFilesList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubFilesList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + new_files = subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name) == subfiles.end()) + { + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; + d.subfiles.pop_back(); + } + else + { + new_files.erase(static_cast(mNodes[d.subfiles[i]])->file_name) ; + ++i; + } + + for(std::set::const_iterator it(new_files.begin());it!=new_files.end();++it) + { + d.subfiles.push_back(mNodes.size()) ; + mNodes.push_back(new FileEntry(*it)); + } + return true; + } + void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ; + // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e - uint32_t root ; + uint32_t mRoot ; std::vector mNodes;// uses pointers to keep information about valid/invalid objects. - void compress() ; // use empty space in the vector, mostly due to deleted entries. + void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to + // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that + // are not referenced. friend class DirectoryStorage ; // only class that can use this. + + // low level stuff. Should normally not be used externally. + + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; + } + DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + } + + bool check() // checks consistency of storage. + { + return true; + } + + void print() + { + } +private: + bool nodeAccessError(const std::string& s) + { + std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; + return false ; + } + + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning + // phase. That allows to keep file information when moving them around. + + bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir) + { + DirEntry& d(*static_cast(mNodes[dir])) ; + + for(uint32_t i=0;imFileHierarchy ; + mParentIndex = i; + mDirTabIndex = 0; +} + +DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i) +{ + mStorage = s->mFileHierarchy ; + mParentIndex = i; + mFileTabIndex = 0; +} + +DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++() +{ + ++mDirTabIndex ; + + return *this; +} +DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++() +{ + ++mFileTabIndex ; + + return *this; +} +DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const +{ + return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; +} +DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const +{ + return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; +} +DirectoryStorage::DirIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} +DirectoryStorage::FileIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} + +/******************************************************************************************************************/ +/* Directory Storage */ +/******************************************************************************************************************/ + +DirectoryStorage::DirIterator DirectoryStorage::root() +{ + return DirIterator(this,EntryIndex(0)) ; +} + +void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; +} +void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; +} +void DirectoryStorage::removeDirectory(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->removeDirectory(indx); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0fb94037a..90a0a889e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -23,12 +25,13 @@ class DirectoryStorage virtual ~DirectoryStorage() {} typedef uint32_t EntryIndex ; + static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const; - virtual int searchHash(const RsFileHash& hash, std::list &results) const; - virtual int searchBoolExp(Expression * exp, std::list &results) const; + virtual int searchTerms(const std::list& terms, std::list &results) const { return 0;} + virtual int searchHash(const RsFileHash& hash, std::list &results) const { return 0;} + virtual int searchBoolExp(Expression * exp, std::list &results) const { return 0; } void getFileDetails(EntryIndex i) ; @@ -39,25 +42,31 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; - DirIterator(const EntryIndex& d) ; + DirIterator(DirectoryStorage *d,EntryIndex i) ; DirIterator& operator++() ; - EntryIndex operator*() const ; // current directory entry + EntryIndex operator*() const ; operator bool() const ; // used in for loops. Returns true when the iterator is valid. // info about the directory that is pointed by the iterator const std::string& name() const ; + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mDirTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; + + friend class DirectoryStorage ; }; class FileIterator { public: - FileIterator(DirIterator& d); // crawls all files in specified directory - FileIterator(EntryIndex& e); // crawls all files in specified directory + FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory FileIterator& operator++() ; - EntryIndex operator*() const ; // current file entry + EntryIndex operator*() const ; // current file entry operator bool() const ; // used in for loops. Returns true when the iterator is valid. @@ -68,14 +77,24 @@ class DirectoryStorage uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; + + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mFileTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; }; virtual DirIterator root() ; // returns the index of the root directory entry. - void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; - void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; void removeDirectory(const EntryIndex& indx) ; + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + + void cleanup(); + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; @@ -86,6 +105,8 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. InternalFileHierarchyStorage *mFileHierarchy ; + + RsMutex mDirStorageMtx ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -104,15 +125,6 @@ public: void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; - /*! - * \brief addFile - * \param dir - * \param hash - * \param modf_time - */ - void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; - private: std::list mLocalDirs ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c88820fb0..c80968d8b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -26,10 +26,12 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - std::list sub_dir_list ; + + + std::set sub_dir_list ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) - sub_dir_list.push_back( (*real_dir_it).filename ) ; + sub_dir_list.insert( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. @@ -61,18 +63,18 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles - std::list subfiles ; - std::list subdirs ; + std::set subfiles ; + std::set subdirs ; while(dirIt.readdir()) { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ; std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; @@ -83,7 +85,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // update file and dir lists for current directory. mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + std::set new_files ; + mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; // now go through list of subfiles and request the hash to hashcache @@ -98,7 +102,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 35f069b3c..74795ae14 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -4,13 +4,13 @@ // - remote: directories are requested remotely to a providing client // #include "file_sharing/hash_cache.h" - -class LocalDirectoryStorage ; +#include "file_sharing/directory_storage.h" class DirectoryUpdater { public: - DirectoryUpdater() ; + DirectoryUpdater() {} + virtual ~DirectoryUpdater(){} // Does some updating job. Crawls the existing directories and checks wether it has been updated // recently enough. If not, calls the directry source. @@ -24,6 +24,8 @@ class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual ~LocalDirectoryUpdater() {} + virtual void tick() ; protected: @@ -38,5 +40,8 @@ private: class RemoteDirectoryUpdater: public DirectoryUpdater { public: + RemoteDirectoryUpdater() {} + virtual ~RemoteDirectoryUpdater() {} + virtual void tick() ; }; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 747ec93d9..105c7776c 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -124,9 +124,10 @@ bool FolderIterator::d_name(std::string& dest) } const std::string& FolderIterator::file_fullpath() { return mFullPath ; } -const std::string& FolderIterator::file_name() { return mFileName ; } -uint64_t FolderIterator::file_size() { return mFileSize ; } -time_t FolderIterator::file_modtime() { return mFileModTime ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } +uint8_t FolderIterator::file_type() { return mType ; } bool FolderIterator::closedir() { diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 823aa9a9a..dcf8ed3f3 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -40,7 +40,7 @@ public: const std::string& file_name() ; const std::string& file_fullpath() ; uint64_t file_size() ; - uint64_t file_type() ; + uint8_t file_type() ; time_t file_modtime() ; private: From d86b89b15acd2428603ad259e7b25ff6ea722360 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 27 Jul 2016 21:22:59 +0200 Subject: [PATCH 11/92] fixed compilation/running of directory updater --- .../src/file_sharing/directory_storage.cc | 166 ++++++++++++------ .../src/file_sharing/directory_storage.h | 16 +- .../src/file_sharing/directory_updater.cc | 24 ++- .../src/file_sharing/directory_updater.h | 8 +- libretroshare/src/file_sharing/hash_cache.cc | 124 +++++++++++++ libretroshare/src/file_sharing/hash_cache.h | 65 ++++--- 6 files changed, 309 insertions(+), 94 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index a955df832..813b0c38d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -22,8 +22,9 @@ class InternalFileHierarchyStorage class FileStorageNode { public: - static const uint32_t TYPE_FILE = 0x0001 ; - static const uint32_t TYPE_DIR = 0x0002 ; + static const uint32_t TYPE_UNKNOWN = 0x0000 ; + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; virtual ~FileStorageNode() {} virtual uint32_t type() const =0; @@ -45,7 +46,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(DirectoryStorage::NO_INDEX) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -66,18 +67,15 @@ class InternalFileHierarchyStorage // high level modification routines - bool isIndexValid(DirectoryStorage::EntryIndex e) + bool isIndexValid(DirectoryStorage::EntryIndex e) const { return e < mNodes.size() && mNodes[e] != NULL ; } bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) { - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("updateSubDirectoryList(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; DirEntry& d(*static_cast(mNodes[indx])) ; @@ -104,14 +102,11 @@ class InternalFileHierarchyStorage { // check that it's a directory - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("removeDirectory(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("removeDirectory(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; if(indx == 0) - return nodeAccessError("removeDirectory(): Cannot remove top level directory") ; + return nodeAccessError("checkIndex(): Cannot remove top level directory") ; // remove from parent @@ -125,13 +120,21 @@ class InternalFileHierarchyStorage return nodeAccessError("removeDirectory(): inconsistency!!") ; } + bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const + { + if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("checkIndex(): Node does not exist") ; + + if(mNodes[indx]->type() != type) + return nodeAccessError("checkIndex(): Node is of wrong type") ; + + return true; + } + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set& subfiles,std::set& new_files) { - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("updateSubFilesList(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("updateSubFilesList(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; DirEntry& d(*static_cast(mNodes[indx])) ; new_files = subfiles ; @@ -155,14 +158,32 @@ class InternalFileHierarchyStorage } return true; } - void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ; + bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) + { + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + return false; + + static_cast(mNodes[file_index])->file_hash = hash ; + return true; + } + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const uint32_t modf_time) + { + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + return false; + + FileEntry& fe(*static_cast(mNodes[file_index])) ; + + fe.file_hash = hash; + fe.file_size = size; + fe.file_modtime = modf_time; + + return true; + } // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e uint32_t mRoot ; - std::vector mNodes;// uses pointers to keep information about valid/invalid objects. void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to @@ -171,11 +192,25 @@ class InternalFileHierarchyStorage friend class DirectoryStorage ; // only class that can use this. - // low level stuff. Should normally not be used externally. + // Low level stuff. Should normally not be used externally. + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return NULL ; + + return static_cast(mNodes[indx]) ; + } + const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const + { + if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) + return NULL ; + + return static_cast(mNodes[indx]) ; + } DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) { - if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) return DirectoryStorage::NO_INDEX; if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) @@ -185,7 +220,7 @@ class InternalFileHierarchyStorage } DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) { - if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) return DirectoryStorage::NO_INDEX; if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) @@ -203,7 +238,7 @@ class InternalFileHierarchyStorage { } private: - bool nodeAccessError(const std::string& s) + static bool nodeAccessError(const std::string& s) { std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; return false ; @@ -256,46 +291,58 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++() return *this; } -DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const -{ - return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; -} -DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const -{ - return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; -} -DirectoryStorage::DirIterator::operator bool() const -{ - return **this != DirectoryStorage::NO_INDEX; -} -DirectoryStorage::FileIterator::operator bool() const -{ - return **this != DirectoryStorage::NO_INDEX; -} +DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; } +DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; } + +DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; } +DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; } + +RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); } +uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; } +std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } +std::string DirectoryStorage::FileIterator::fullpath() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } +time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } /******************************************************************************************************************/ /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirIterator DirectoryStorage::root() +DirectoryStorage::DirectoryStorage(const std::string& local_file_name) + : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) { - return DirIterator(this,EntryIndex(0)) ; + mFileHierarchy = new InternalFileHierarchyStorage(); } -void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +DirectoryStorage::EntryIndex DirectoryStorage::root() const { - RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + return EntryIndex(0) ; } -void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) + +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; } -void DirectoryStorage::removeDirectory(const EntryIndex& indx) +bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->removeDirectory(indx); + return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; +} +bool DirectoryStorage::removeDirectory(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->removeDirectory(indx); +} + +bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->updateFile(index,hash,fname,size,modf_time); +} +bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->updateHash(index,hash); } // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; @@ -321,3 +368,18 @@ void DirectoryStorage::save(const std::string& local_file_name) { // first write the header, than all fields. } + +/******************************************************************************************************************/ +/* Local Directory Storage */ +/******************************************************************************************************************/ + +void LocalDirectoryStorage::setSharedDirectoryList(const std::list& lst) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mLocalDirs = lst ; +} +void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + lst = mLocalDirs ; +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 90a0a889e..fe7307d6d 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -84,14 +84,14 @@ class DirectoryStorage InternalFileHierarchyStorage *mStorage ; }; - virtual DirIterator root() ; // returns the index of the root directory entry. + EntryIndex root() const ; // returns the index of the root directory entry. - void updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; - void updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; - void removeDirectory(const EntryIndex& indx) ; + bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + bool updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; + bool removeDirectory(const EntryIndex& indx) ; - void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; + bool updateHash(const EntryIndex& index,const RsFileHash& hash); void cleanup(); @@ -105,7 +105,9 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. InternalFileHierarchyStorage *mFileHierarchy ; + std::string mFileName; + protected: RsMutex mDirStorageMtx ; }; @@ -119,7 +121,7 @@ public: class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage() ; + LocalDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c80968d8b..3d6c5bb74 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,9 +10,10 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc) +LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc) : mHashCache(hc) { + mSharedDirectories = new LocalDirectoryStorage("local_storage.txt"); } void LocalDirectoryUpdater::tick() @@ -35,11 +36,11 @@ void LocalDirectoryUpdater::tick() // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; @@ -66,7 +67,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::set subfiles ; std::set subdirs ; - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { switch(dirIt.file_type()) { @@ -91,18 +92,27 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // now go through list of subfiles and request the hash to hashcache - for(DirectoryStorage::FileIterator dit(indx);dit;++dit) + for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) { // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ; + RsFileHash hash ; + + if(mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + mSharedDirectories->updateHash(*dit,hash); } // go through the list of sub-dirs and recursively update - DirectoryStorage::DirIterator stored_dir_it(indx) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } +void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) +{ + if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) + std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; +} + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 74795ae14..b70164bdd 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -20,20 +20,20 @@ class DirectoryUpdater // }; -class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient +class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient { public: - LocalDirectoryUpdater(HashCache *hash_cache) ; + LocalDirectoryUpdater(HashStorage *hash_cache) ; virtual ~LocalDirectoryUpdater() {} virtual void tick() ; protected: - virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ; + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); private: - HashCache *mHashCache ; + HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index e69de29bb..25af17984 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -0,0 +1,124 @@ +#include "util/rsdir.h" +#include "hash_cache.h" + +#define HASHSTORAGE_DEBUG 1 + +HashStorage::HashStorage(const std::string& save_file_name) + : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") +{ +} + +void HashStorage::data_tick() +{ + std::cerr << "Ticking hash thread." << std::endl; + + FileHashJob job; + + { + RS_STACK_MUTEX(mHashMtx) ; + + if(mFilesToHash.empty()) + return ; + + job = mFilesToHash.begin()->second ; + } + + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + + RsFileHash hash; + uint64_t size ; + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + { + std::cerr << "ERROR" << std::endl; + return; + } + // update the hash storage + + + // call the client + + job.client->hash_callback(job.client_param, job.full_path, hash, size); + + std::cerr << "done."<< std::endl; + + if(mFilesToHash.empty()) + { + std::cerr << "Starting hashing thread." << std::endl; + fullstop(); + std::cerr << "done." << std::endl; + } +} + +bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) +{ + // check if the hash is up to date w.r.t. cache. + + RS_STACK_MUTEX(mHashMtx) ; + + time_t now = time(NULL) ; + std::map::iterator it = mFiles.find(full_path) ; + + if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) + { + it->second.time_stamp = now ; +#ifdef HASHCACHE_DEBUG + std::cerr << "Found in cache." << std::endl ; +#endif + return true ; + } + + // we need to schedule a re-hashing + + if(mFilesToHash.find(full_path) != mFilesToHash.end()) + return false ; + + FileHashJob job ; + + job.client = c ; + job.client_param = client_param ; + job.full_path = full_path ; + + mFilesToHash[full_path] = job; + + if(!isRunning()) + { + std::cerr << "Starting hashing thread." << std::endl; + start() ; + } + + return false; +} + +void HashStorage::clean() +{ +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Cleaning HashStorage..." << std::endl ; +#endif + time_t now = time(NULL) ; + time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "cleaning hash cache." << std::endl ; +#endif + + for(std::map::iterator it(mFiles.begin());it!=mFiles.end();) + if(it->second.time_stamp + duration < (uint64_t)now) + { +#ifdef HASHSTORAGE_DEBUG + std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; +#endif + std::map::iterator tmp(it) ; + ++tmp ; + mFiles.erase(it) ; + it=tmp ; + mChanged = true ; + } + else + ++it ; + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Done." << std::endl; +#endif +} + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index f867f6109..fa73797c2 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -4,61 +4,78 @@ #include "util/rsthreads.h" #include "retroshare/rsfiles.h" -class HashCacheClient +class HashStorageClient { public: - virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ; + HashStorageClient() {} + virtual ~HashStorageClient() {} + + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0; }; -class FileHashingThread: public RsTickingThread +class HashStorage: public RsTickingThread { public: - FileHashingThread() {} + HashStorage(const std::string& save_file_name) ; - virtual void data_tick() ; -}; + /*! + * \brief requestHash Requests the hash for the given file, assuming size and mod_time are the same. + * + * \param full_path Full path to reach the file + * \param size Actual file size + * \param mod_time Actual file modification time + * \param known_hash Returned hash for the file. + * \param c Hash cache client to which the hash should be sent once calculated + * \param client_param Param to be passed to the client callback + * + * \return true if the supplied hash info is up to date. + */ + bool requestHash(const std::string& full_path, uint64_t size, time_t mod_time, RsFileHash& known_hash, HashStorageClient *c, uint32_t client_param) ; -class HashCache -{ -public: - HashCache(const std::string& save_file_name) ; - - bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ; - - struct HashCacheInfo + struct HashStorageInfo { std::string filename ; uint64_t size ; - uint32_t time_stamp ; + uint32_t time_stamp ; // last time the hash was tested/requested uint32_t modf_stamp ; RsFileHash hash ; } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; } - uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } + uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } void clear() { mFiles.clear(); } bool empty() const { return mFiles.empty() ; } + // Functions called by the thread + + virtual void data_tick() ; private: void clean() ; void save() ; void load() ; - // threaded stuff - FileHashingThread mHashingThread ; - RsMutex mHashMtx ; - // Local configuration and storage - uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries - std::map mFiles ; + uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries + std::map mFiles ; // stored as (full_path, hash_info) std::string mFilePath ; bool mChanged ; + struct FileHashJob + { + std::string full_path; + HashStorageClient *client; + uint32_t client_param ; + }; + // current work - std::map mFilesToHash ; + std::map mFilesToHash ; + + // thread/mutex stuff + + RsMutex mHashMtx ; }; From 463f21c85c0ea1a1fc9f2e72074913573f9decc8 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 00:48:28 +0200 Subject: [PATCH 12/92] fixed hashing thread --- .../src/file_sharing/directory_storage.cc | 61 +++++++++++++++++-- .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 14 +++-- .../src/file_sharing/directory_updater.h | 2 +- libretroshare/src/file_sharing/hash_cache.cc | 52 ++++++++-------- libretroshare/src/file_sharing/hash_cache.h | 1 + 6 files changed, 92 insertions(+), 40 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 813b0c38d..ee89d572c 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -46,7 +46,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(DirectoryStorage::NO_INDEX) {} + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -226,7 +226,7 @@ class InternalFileHierarchyStorage if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) return DirectoryStorage::NO_INDEX; - return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } bool check() // checks consistency of storage. @@ -234,10 +234,57 @@ class InternalFileHierarchyStorage return true; } - void print() + void print() const { + int nfiles = 0 ; + int ndirs = 0 ; + int nempty = 0 ; + int nunknown = 0; + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++ndirs; + } + else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++nfiles; + } + else + { + ++nunknown; + std::cerr << "(EE) Error: unknown type node found!" << std::endl; + } + + std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; + + recursPrint(0,DirectoryStorage::EntryIndex(0)); } private: + void recursPrint(int depth,DirectoryStorage::EntryIndex node) const + { + std::string indent(2*depth,' '); + DirEntry& d(*static_cast(mNodes[node])); + + std::cerr << indent << d.dir_name << std::endl; + + for(int i=0;i(mNodes[i])); + std::cerr << indent << " " << f.file_hash << " " << f.file_modtime << " " << f.file_name << std::endl; + } + } + static bool nodeAccessError(const std::string& s) { std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; @@ -300,7 +347,6 @@ DirectoryStorage::DirIterator ::operator bool() const { return **this != Directo RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); } uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; } std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } -std::string DirectoryStorage::FileIterator::fullpath() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } /******************************************************************************************************************/ @@ -368,7 +414,12 @@ void DirectoryStorage::save(const std::string& local_file_name) { // first write the header, than all fields. } - +void DirectoryStorage::print() +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + std::cerr << "LocalDirectoryStorage:" << std::endl; + mFileHierarchy->print(); +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fe7307d6d..9f8293963 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -73,7 +73,6 @@ class DirectoryStorage // info about the file that is pointed by the iterator std::string name() const ; - std::string fullpath() const ; uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; @@ -93,6 +92,7 @@ class DirectoryStorage bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + void print(); void cleanup(); private: diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 3d6c5bb74..9367e826e 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,10 +10,9 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc) - : mHashCache(hc) +LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) + : mHashCache(hc),mSharedDirectories(lds) { - mSharedDirectories = new LocalDirectoryStorage("local_storage.txt"); } void LocalDirectoryUpdater::tick() @@ -27,7 +26,7 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - + mSharedDirectories->getSharedDirectoryList(shared_directory_list); std::set sub_dir_list ; @@ -43,7 +42,10 @@ void LocalDirectoryUpdater::tick() DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + { + std::cerr << " recursing into " << real_dir_it->filename << std::endl; recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; + } } void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) @@ -98,7 +100,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p RsFileHash hash ; - if(mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) mSharedDirectories->updateHash(*dit,hash); } @@ -107,7 +109,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + recursUpdateSharedDir(cumulated_path + "/" + *real_dir_it, *stored_dir_it) ; } void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index b70164bdd..f0317b36d 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -23,7 +23,7 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient { public: - LocalDirectoryUpdater(HashStorage *hash_cache) ; + LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} virtual void tick() ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 25af17984..0acd323c2 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -6,13 +6,14 @@ HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { + mRunning = false ; } void HashStorage::data_tick() { - std::cerr << "Ticking hash thread." << std::endl; - FileHashJob job; + RsFileHash hash; + uint64_t size ; { RS_STACK_MUTEX(mHashMtx) ; @@ -21,33 +22,29 @@ void HashStorage::data_tick() return ; job = mFilesToHash.begin()->second ; + + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + else + std::cerr << "done."<< std::endl; + + mFilesToHash.erase(mFilesToHash.begin()) ; + + if(mFilesToHash.empty()) + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + } } - - std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - - RsFileHash hash; - uint64_t size ; - - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - { - std::cerr << "ERROR" << std::endl; - return; - } - // update the hash storage - - // call the client - job.client->hash_callback(job.client_param, job.full_path, hash, size); - - std::cerr << "done."<< std::endl; - - if(mFilesToHash.empty()) - { - std::cerr << "Starting hashing thread." << std::endl; - fullstop(); - std::cerr << "done." << std::endl; - } + if(!hash.isNull()) + job.client->hash_callback(job.client_param, job.full_path, hash, size); } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -81,8 +78,9 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mFilesToHash[full_path] = job; - if(!isRunning()) + if(!mRunning) { + mRunning = true ; std::cerr << "Starting hashing thread." << std::endl; start() ; } diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index fa73797c2..88b4206f9 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -77,5 +77,6 @@ private: // thread/mutex stuff RsMutex mHashMtx ; + bool mRunning; }; From 2aac88464df96de1a73bf4f9079be2ad8b09ea7e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 09:29:15 +0200 Subject: [PATCH 13/92] fixed a few bugs in directory parsing --- .../src/file_sharing/directory_storage.cc | 41 ++++++++++++------- .../src/file_sharing/directory_storage.h | 8 +++- .../src/file_sharing/directory_updater.cc | 16 ++++---- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index ee89d572c..b178daf85 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -32,7 +32,7 @@ class InternalFileHierarchyStorage class FileEntry: public FileStorageNode { public: - FileEntry(const std::string& name) : file_name(name) {} + FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } virtual ~FileEntry() {} @@ -131,7 +131,7 @@ class InternalFileHierarchyStorage return true; } - bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set& subfiles,std::set& new_files) + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; @@ -139,22 +139,33 @@ class InternalFileHierarchyStorage DirEntry& d(*static_cast(mNodes[indx])) ; new_files = subfiles ; + // remove from new_files the ones that already exist and have a modf time that is not older. + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name) == subfiles.end()) + { + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])) ; + std::map::const_iterator it = subfiles.find(f.file_name) ; + + if(it == subfiles.end()) // file does not exist anymore => delete { d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; d.subfiles.pop_back(); - } - else - { - new_files.erase(static_cast(mNodes[d.subfiles[i]])->file_name) ; - ++i; + + continue; } - for(std::set::const_iterator it(new_files.begin());it!=new_files.end();++it) + if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + f.file_hash.clear(); // hash needs recomputing + else + new_files.erase(f.file_name) ; + + ++i; + } + + for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { d.subfiles.push_back(mNodes.size()) ; - mNodes.push_back(new FileEntry(*it)); + mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); } return true; } @@ -166,7 +177,7 @@ class InternalFileHierarchyStorage static_cast(mNodes[file_index])->file_hash = hash ; return true; } - bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const uint32_t modf_time) + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) return false; @@ -273,15 +284,15 @@ private: std::string indent(2*depth,' '); DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << d.dir_name << std::endl; + std::cerr << indent << "dir:" << d.dir_name << std::endl; for(int i=0;i(mNodes[i])); - std::cerr << indent << " " << f.file_hash << " " << f.file_modtime << " " << f.file_name << std::endl; + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << std::fill(8) << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; } } @@ -369,7 +380,7 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std:: RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; } -bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) +bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9f8293963..bd0f5b8ce 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -83,10 +83,16 @@ class DirectoryStorage InternalFileHierarchyStorage *mStorage ; }; + struct FileTS + { + uint64_t size ; + time_t modtime; + }; + EntryIndex root() const ; // returns the index of the root directory entry. bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; - bool updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; + bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 9367e826e..f9a228e1b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -66,21 +66,21 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles - std::set subfiles ; + std::map subfiles ; std::set subdirs ; for(;dirIt.isValid();dirIt.next()) { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ; - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; - break; + case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; + subfiles[dirIt.file_name()].size = dirIt.file_size(); + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; - default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } @@ -89,7 +89,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - std::set new_files ; + std::map new_files ; mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; // now go through list of subfiles and request the hash to hashcache From aeb05953015d786997303abe55686f3255dc8cd0 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 10:49:49 +0200 Subject: [PATCH 14/92] fixed updated ts in hash cache files --- .../src/file_sharing/directory_storage.cc | 6 +++++- .../src/file_sharing/directory_updater.cc | 18 +++++++++++++++-- .../src/file_sharing/directory_updater.h | 14 +++++-------- libretroshare/src/file_sharing/hash_cache.cc | 20 ++++++++++++++++++- libretroshare/src/file_sharing/hash_cache.h | 1 + 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b178daf85..d94cb64f0 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -155,7 +155,11 @@ class InternalFileHierarchyStorage } if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + { f.file_hash.clear(); // hash needs recomputing + f.file_modtime = it->second.modtime; + f.file_size = it->second.size; + } else new_files.erase(f.file_name) ; @@ -292,7 +296,7 @@ private: for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << std::fill(8) << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; } } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index f9a228e1b..b4ebfb0f5 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -5,6 +5,8 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 10 ; // 10 seconds for testing. Should be much more!! + void RemoteDirectoryUpdater::tick() { // use the stored iterator @@ -13,11 +15,23 @@ void RemoteDirectoryUpdater::tick() LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { + mLastSweepTime = 0; } -void LocalDirectoryUpdater::tick() +void LocalDirectoryUpdater::data_tick() { - std::cerr << "LocalDirectoryUpdater::tick()" << std::endl; + time_t now = time(NULL) ; + + if(now > DELAY_BETWEEN_DIRECTORY_UPDATES + mLastSweepTime) + { + sweepSharedDirectories() ; + mLastSweepTime = now; + } +} + +void LocalDirectoryUpdater::sweepSharedDirectories() +{ + std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index f0317b36d..2d30793cd 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -11,30 +11,26 @@ class DirectoryUpdater public: DirectoryUpdater() {} virtual ~DirectoryUpdater(){} - - // Does some updating job. Crawls the existing directories and checks wether it has been updated - // recently enough. If not, calls the directry source. - // - virtual void tick() =0; - - // }; -class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient +class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread { public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void tick() ; + virtual void data_tick() ; protected: virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); + void sweepSharedDirectories(); private: HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; + + time_t mLastSweepTime; }; class RemoteDirectoryUpdater: public DirectoryUpdater diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 0acd323c2..eb86813b3 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -40,6 +40,16 @@ void HashStorage::data_tick() mRunning = false ; std::cerr << "done." << std::endl; } + + // store the result + + HashStorageInfo& info(mFiles[job.full_path]); + + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; } // call the client @@ -51,6 +61,9 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t { // check if the hash is up to date w.r.t. cache. +#ifdef HASHSTORAGE_DEBUG + std::cerr << "HASH Requested for file " << full_path << ": "; +#endif RS_STACK_MUTEX(mHashMtx) ; time_t now = time(NULL) ; @@ -59,11 +72,15 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) { it->second.time_stamp = now ; -#ifdef HASHCACHE_DEBUG + known_hash = it->second.hash; +#ifdef HASHSTORAGE_DEBUG std::cerr << "Found in cache." << std::endl ; #endif return true ; } +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Not in cache. Sceduling for re-hash." << std::endl ; +#endif // we need to schedule a re-hashing @@ -75,6 +92,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t job.client = c ; job.client_param = client_param ; job.full_path = full_path ; + job.ts = mod_time ; mFilesToHash[full_path] = job; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 88b4206f9..2d10c09ca 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -68,6 +68,7 @@ private: std::string full_path; HashStorageClient *client; uint32_t client_param ; + time_t ts; }; // current work From 5b9fd74d85b018a00fe7f16c6905e8577cc288c4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 30 Jul 2016 21:52:42 +0200 Subject: [PATCH 15/92] removed most of CacheStrapper stuff from ftServer, replaced fimonitor by p3FileDatabase --- libretroshare/src/dbase/fimonitor.cc | 6 +- libretroshare/src/dbase/fimonitor.h | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 149 ++++++-- libretroshare/src/file_sharing/p3filelists.h | 92 +++-- libretroshare/src/ft/ftcontroller.cc | 226 ++--------- libretroshare/src/ft/ftcontroller.h | 8 +- libretroshare/src/ft/ftdatamultiplex.cc | 2 - libretroshare/src/ft/ftserver.cc | 358 ++++-------------- libretroshare/src/ft/ftserver.h | 335 ++++++++-------- libretroshare/src/plugins/pluginmanager.cc | 3 +- libretroshare/src/retroshare/rsfiles.h | 25 +- libretroshare/src/rsserver/rsinit.cc | 8 - libretroshare/src/serialiser/rsserviceids.h | 1 + .../src/gui/FileTransfer/TransfersDialog.cpp | 15 - 14 files changed, 463 insertions(+), 769 deletions(-) diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index 56b1e7bc2..ec8140892 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -1594,9 +1594,9 @@ std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const int FileIndexMonitor::RequestDirDetails(const std::string& path, DirDetails& details) const { - /* lock it up */ - RsStackMutex mutex(fiMutex) ; - return fi.extractData(path,details) ; + /* lock it up */ + RsStackMutex mutex(fiMutex) ; + return fi.extractData(path,details) ; } uint32_t FileIndexMonitor::getType(void *ref) const diff --git a/libretroshare/src/dbase/fimonitor.h b/libretroshare/src/dbase/fimonitor.h index dc616625d..1ba0c327c 100644 --- a/libretroshare/src/dbase/fimonitor.h +++ b/libretroshare/src/dbase/fimonitor.h @@ -137,7 +137,7 @@ class FileIndexMonitor: public CacheSource, public RsTickingThread // Interface for browsing dir hirarchy int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ; uint32_t getType(void*) const ; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; + int RequestDirDetails(const std::string& path, DirDetails &details) const ; // set/update shared directories virtual void setSharedDirectories(const std::list& dirs); @@ -146,7 +146,7 @@ class FileIndexMonitor: public CacheSource, public RsTickingThread void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files. - bool inDirectoryCheck(); + bool inDirectoryCheck(); /* util fns */ diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 7c35fe332..0fd3a830c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,5 +1,9 @@ +#include "serialiser/rsserviceids.h" + #include "file_sharing/p3filelists.h" #include "file_sharing/directory_storage.h" +#include "file_sharing/directory_updater.h" + #include "retroshare/rsids.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " @@ -9,19 +13,36 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -p3FileLists::p3FileLists(p3LinkMgr *mpeers) - : mLinkMgr(mpeers), mFLSMtx("p3FileLists") +#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not implemented!" << std::endl; } + +p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) + : mServCtrl(mpeers), mFLSMtx("p3FileLists") { // loads existing indexes for friends. Some might be already present here. // - - // init the directory watchers - - // local variables/structures - // + mRemoteDirectories.clear() ; // we should load them! + + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin") ; + mHashCache = new HashStorage("hash_cache.bin") ; + + mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; + mRemoteDirWatcher = NULL; // not used yet + + mLocalDirWatcher->start(); + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -p3FileLists::~p3FileLists() + +void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) +{ + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; +} +void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) +{ + mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; +} + +p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; @@ -30,10 +51,28 @@ p3FileLists::~p3FileLists() mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around. - // delete all pointers + delete mLocalSharedDirs ; + delete mLocalDirWatcher ; + delete mRemoteDirWatcher ; + delete mHashCache ; } -int p3FileLists::tick() +const std::string FILE_DB_APP_NAME = "file_database"; +const uint16_t FILE_DB_APP_MAJOR_VERSION = 1; +const uint16_t FILE_DB_APP_MINOR_VERSION = 0; +const uint16_t FILE_DB_MIN_MAJOR_VERSION = 1; +const uint16_t FILE_DB_MIN_MINOR_VERSION = 0; + +RsServiceInfo p3FileDatabase::getServiceInfo() +{ + return RsServiceInfo(RS_SERVICE_TYPE_FILE_DATABASE, + FILE_DB_APP_NAME, + FILE_DB_APP_MAJOR_VERSION, + FILE_DB_APP_MINOR_VERSION, + FILE_DB_MIN_MAJOR_VERSION, + FILE_DB_MIN_MINOR_VERSION); +} +int p3FileDatabase::tick() { // tick the watchers, possibly create new ones if additional friends do connect. // @@ -53,29 +92,37 @@ int p3FileLists::tick() return 0; } -void p3FileLists::tickRecv() +void p3FileDatabase::tickWatchers() { -} -void p3FileLists::tickSend() -{ - // go through the list of out requests and send them to the corresponding friends, if they are online. + NOT_IMPLEMENTED(); } -bool p3FileLists::loadList(std::list& items) +void p3FileDatabase::tickRecv() +{ +} +void p3FileDatabase::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. + NOT_IMPLEMENTED(); +} + +bool p3FileDatabase::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + NOT_IMPLEMENTED(); return true ; } -bool p3FileLists::saveList(const std::list& items) +bool p3FileDatabase::saveList(bool &cleanup, std::list&) { + NOT_IMPLEMENTED(); return true ; } -void p3FileLists::cleanup() +void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; @@ -83,15 +130,8 @@ void p3FileLists::cleanup() // P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; - std::set friend_set ; - { -#warning we should use a std::set in the first place. Modify rsPeers? - std::list friend_list ; - mLinkMgr->getFriendList(friend_list) ; - - for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) - friend_set.insert(*it) ; - } + std::set friend_set ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, friend_set); for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) if(friend_set.find(it->first) == friend_set.end()) @@ -125,4 +165,59 @@ void p3FileLists::cleanup() IndicateConfigChanged(); } +int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const +{ + NOT_IMPLEMENTED(); + return 0; +} +int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const +{ + NOT_IMPLEMENTED(); + return 0; +} +uint32_t p3FileDatabase::getType(void *ref) const +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed +{ + NOT_IMPLEMENTED(); +} +bool p3FileDatabase::inDirectoryCheck() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::setWatchPeriod(uint32_t seconds) +{ + NOT_IMPLEMENTED(); +} +uint32_t p3FileDatabase::watchPeriod() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) +{ + NOT_IMPLEMENTED(); +} +uint32_t p3FileDatabase::rememberHashCacheDuration() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::clearHashCache() +{ + NOT_IMPLEMENTED(); +} +bool p3FileDatabase::rememberHashCache() +{ + NOT_IMPLEMENTED(); + return false; +} +void p3FileDatabase::setRememberHashCache(bool) +{ + NOT_IMPLEMENTED(); +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index e2c9d2016..7c0060183 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -21,23 +21,30 @@ // #pragma once +#include "ft/ftsearch.h" #include "retroshare/rsfiles.h" #include "services/p3service.h" +#include "file_sharing/hash_cache.h" + #include "pqi/p3cfgmgr.h" #include "pqi/p3linkmgr.h" -class RemoteDirectoryStorage ; -class RemoteSharedDirectoryWatcher ; -class LocalSharedDirectoryWatcher ; -class LocalSharedDirectoryMap ; -class HashCache ; +class RemoteDirectoryUpdater ; +class LocalDirectoryUpdater ; -class p3FileLists: public p3Service, public p3Config //, public RsSharedFileService +class RemoteDirectoryStorage ; +class LocalDirectoryStorage ; + +class HashStorage ; + +class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + virtual RsServiceInfo getServiceInfo(); + struct RsFileListSyncRequest { RsPeerId peerId ; @@ -45,12 +52,17 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ // [...] more to add here }; - p3FileLists(p3LinkMgr *mpeers) ; - ~p3FileLists(); + p3FileDatabase(p3ServiceControl *mpeers) ; + ~p3FileDatabase(); + + /*! + * \brief forceSyncWithPeers + * + * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new gorup of friend is created, or when + * a friend was added/removed from a group. + */ + void forceSyncWithPeers(); - /* - */ - // derived from p3Service // virtual int tick() ; @@ -62,32 +74,61 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + // ftSearch + virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; + // Interface for browsing dir hierarchy // - int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ; - uint32_t getType(const EntryIndex&) const ; + + void stopThreads() ; + void startThreads() ; + + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); int RequestDirDetails(const std::string& path, DirDetails &details) const ; - // set/update shared directories - void setSharedDirectories(const std::list& dirs); - void getSharedDirectories(std::list& dirs); - void updateShareFlags(const SharedDirInfo& info) ; + // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. + + int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; + uint32_t getType(void *) const ; + + // set/update shared directories + void setSharedDirectories(const std::list& dirs); + void getSharedDirectories(std::list& dirs); + void updateShareFlags(const SharedDirInfo& info) ; + bool convertSharedFilePath(const std::string& path,std::string& fullpath); + + // interface for hash caching + + void setWatchPeriod(uint32_t seconds); + uint32_t watchPeriod() ; + void setRememberHashCacheDuration(uint32_t days) ; + uint32_t rememberHashCacheDuration() ; + void clearHashCache() ; + bool rememberHashCache() ; + void setRememberHashCache(bool) ; + + // interfact for directory parsing void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed bool inDirectoryCheck(); - // Derived from p3Config - // + // debug + void full_print(); + protected: + // Derived from p3Config + // virtual bool loadList(std::list& items); - virtual bool saveList(const std::list& items); + virtual bool saveList(bool &cleanup, std::list&); + virtual RsSerialiser *setupSerialiser() ; + void cleanup(); void tickRecv(); void tickSend(); void tickWatchers(); private: - p3LinkMgr *mLinkMgr ; + p3ServiceControl *mServCtrl ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -98,15 +139,14 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ // Directory storage hierarchies // std::map mRemoteDirectories ; - - LocalSharedDirectoryMap *mLocalSharedDirs ; + LocalDirectoryStorage *mLocalSharedDirs ; - RemoteSharedDirectoryWatcher *mRemoteDirWatcher ; - LocalSharedDirectoryWatcher *mLocalDirWatcher ; + RemoteDirectoryUpdater *mRemoteDirWatcher ; + LocalDirectoryUpdater *mLocalDirWatcher ; // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // - HashCache *mHashCache ; + HashStorage *mHashCache ; // Local flags and mutexes diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index fc66c7f2a..ae90dfe50 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -73,10 +73,8 @@ static const int32_t SAVE_TRANSFERS_DELAY = 301 ; // save transfer progress every 301 seconds. static const int32_t INACTIVE_CHUNKS_CHECK_DELAY = 240 ; // time after which an inactive chunk is released static const int32_t MAX_TIME_INACTIVE_REQUEUED = 120 ; // time after which an inactive ftFileControl is bt-queued -static const int32_t TIMOUT_CACHE_FILE_TRANSFER = 800 ; // time after which cache transfer gets cancelled if inactive. static const int32_t FT_FILECONTROL_QUEUE_ADD_END = 0 ; -static const int32_t FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE = 1 ; const uint32_t FT_CNTRL_STANDARD_RATE = 10 * 1024 * 1024; const uint32_t FT_CNTRL_SLOW_RATE = 100 * 1024; @@ -99,8 +97,8 @@ ftFileControl::ftFileControl(std::string fname, return; } -ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId) - :CacheTransfer(cs), p3Config(), +ftController::ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId) + : p3Config(), last_save_time(0), last_clean_time(0), mDataplex(dm), @@ -231,7 +229,6 @@ void ftController::data_tick() time_t now = time(NULL) ; if(now > last_save_time + SAVE_TRANSFERS_DELAY) { - cleanCacheDownloads() ; searchForDirectSources() ; IndicateConfigChanged() ; @@ -282,16 +279,15 @@ void ftController::searchForDirectSources() for(std::map::iterator it(mDownloads.begin()); it != mDownloads.end(); ++it) if(it->second->mState != ftFileControl::QUEUED && it->second->mState != ftFileControl::PAUSED) - if(! (it->second->mFlags & RS_FILE_REQ_CACHE)) - { - FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there) + { + FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there) - if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info)) - for(std::list::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit) - if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL) - if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */ - setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId)); - } + if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info)) + for(std::list::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit) + if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL) + if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */ + setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId)); + } } void ftController::tickTransfers() @@ -335,37 +331,6 @@ void ftController::setPriority(const RsFileHash& hash,DwlSpeed p) it->second->mTransfer->setDownloadPriority(p) ; } -void ftController::cleanCacheDownloads() -{ - std::vector toCancel ; - time_t now = time(NULL) ; - - { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ - - for(std::map::iterator it(mDownloads.begin());it!=mDownloads.end();++it) - if (((it->second)->mFlags & RS_FILE_REQ_CACHE) && it->second->mState != ftFileControl::DOWNLOADING) - // check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second)->mCreateTime); - std::cerr << std::endl; -#endif - if (now > (it->second)->mCreator->creationTimeStamp() + TIMOUT_CACHE_FILE_TRANSFER) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second)->mHash << ", time=" << (it->second)->mCreateTime << ", now = " << time(NULL) ; - std::cerr << std::endl; -#endif - toCancel.push_back((it->second)->mHash); - } - } - } - - for(uint32_t i=0;imFlags & RS_FILE_REQ_CACHE) // cache file. add to potential move list - to_move_before.push_back(p) ; - else ++user_transfers ; // count one more user file in the prioritized range. else { if(to_move_after.size() + user_transfers >= _min_prioritized_transfers) // we caught enough transfers to move back to the top of the queue. break ; - if(!(_queue[p]->mFlags & RS_FILE_REQ_CACHE)) // non cache file. add to potential move list - to_move_after.push_back(p) ; + to_move_after.push_back(p) ; } } uint32_t to_move = (uint32_t)std::max(0,(int)_min_prioritized_transfers - (int)user_transfers) ; // we move as many transfers as needed to get _min_prioritized_transfers user transfers. @@ -482,27 +443,6 @@ void ftController::locked_addToQueue(ftFileControl* ftfc,int add_strategy) case FT_FILECONTROL_QUEUE_ADD_END: _queue.push_back(ftfc) ; locked_checkQueueElement(_queue.size()-1) ; break ; - case FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE: - { - // We add the transfer just before the first non cache transfer. - // This is costly, so only use this in case we really need it. - // - uint32_t pos =0; - while(pos < _queue.size() && (pos < _min_prioritized_transfers || (_queue[pos]->mFlags & RS_FILE_REQ_CACHE)>0) ) - ++pos ; - - _queue.push_back(NULL) ; - - for(int i=int(_queue.size())-1;i>(int)pos;--i) - { - _queue[i] = _queue[i-1] ; - locked_checkQueueElement(i) ; - } - - _queue[pos] = ftfc ; - locked_checkQueueElement(pos) ; - } - break ; } } @@ -787,8 +727,8 @@ bool ftController::completeFile(const RsFileHash& hash) std::string path; std::string name; uint64_t size = 0; - uint32_t state = 0; - uint32_t period = 0; + uint32_t state = 0; + uint32_t period = 0; TransferRequestFlags flags ; TransferRequestFlags extraflags ; uint32_t completeCount = 0; @@ -889,12 +829,8 @@ bool ftController::completeFile(const RsFileHash& hash) locked_queueRemove(it->second->mQueuePosition) ; /* switch map */ - if (!(fc->mFlags & RS_FILE_REQ_CACHE)) /* clean up completed cache files automatically */ - { - mCompleted[fc->mHash] = fc; - completeCount = mCompleted.size(); - } else - delete fc ; + mCompleted[fc->mHash] = fc; + completeCount = mCompleted.size(); mDownloads.erase(it); @@ -910,51 +846,14 @@ bool ftController::completeFile(const RsFileHash& hash) /* If it has a callback - do it now */ - if(flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA))// | RS_FILE_HINTS_MEDIA)) + if(flags & RS_FILE_REQ_EXTRA)// | RS_FILE_HINTS_MEDIA)) { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() doing Callback, callbackflags:" << (flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA ));//| RS_FILE_HINTS_MEDIA)) ; - std::cerr << std::endl; -#endif - if(flags & RS_FILE_REQ_CACHE) - { - /* callback */ - if (state == ftFileControl::COMPLETED) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() doing Callback : Success"; - std::cerr << std::endl; -#endif - - CompletedCache(hash); - } - else - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() Cache Callback : Failed"; - std::cerr << std::endl; -#endif - FailedCache(hash); - } - } - - if(flags & RS_FILE_REQ_EXTRA) - { #ifdef CONTROL_DEBUG std::cerr << "ftController::completeFile() adding to ExtraList"; std::cerr << std::endl; #endif mExtraList->addExtraFile(path, hash, size, period, extraflags); - } - -// if(flags & RS_FILE_HINTS_MEDIA) -// { -//#ifdef CONTROL_DEBUG -// std::cerr << "ftController::completeFile() NULL MEDIA callback"; -// std::cerr << std::endl; -//#endif -// } } else { @@ -965,14 +864,12 @@ bool ftController::completeFile(const RsFileHash& hash) } /* Notify GUI */ - if ((flags & RS_FILE_REQ_CACHE) == 0) { - RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, ""); + RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, ""); - RsServer::notify()->notifyDownloadComplete(hash.toStdString()); - RsServer::notify()->notifyDownloadCompleteCount(completeCount); + RsServer::notify()->notifyDownloadComplete(hash.toStdString()); + RsServer::notify()->notifyDownloadCompleteCount(completeCount); - rsFiles->ForceDirectoryCheck() ; - } + rsFiles->ForceDirectoryCheck() ; IndicateConfigChanged(); /* completed transfer -> save */ return true; @@ -1134,17 +1031,16 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files. // - if(!(flags & RS_FILE_REQ_CACHE)) - for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) - if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL)) - { - std::list::iterator tmp(it) ; - ++tmp ; - srcIds.erase(it) ; - it = tmp ; - } - else - ++it ; + for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) + if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL)) + { + std::list::iterator tmp(it) ; + ++tmp ; + srcIds.erase(it) ; + it = tmp ; + } + else + ++it ; std::list::const_iterator it; std::list::const_iterator pit; @@ -1280,7 +1176,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash if(flags & RS_FILE_REQ_ANONYMOUS_ROUTING) mTurtle->monitorTunnels(hash,mFtServer,true) ; - bool assume_availability = flags & RS_FILE_REQ_CACHE ; // assume availability for cache files + bool assume_availability = false; ftFileCreator *fc = new ftFileCreator(savepath, size, hash,assume_availability); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); @@ -1911,58 +1807,6 @@ void ftController::statusChange(const std::list &plist) } } - /* Cache Interface */ -bool ftController::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ -#ifdef CONTROL_DEBUG - std::cerr << "ftController::RequestCacheFile(" << id << ","; - std::cerr << path << "," << hash << "," << size << ")"; - std::cerr << std::endl; -#endif - - /* Request File */ - std::list ids; - ids.push_back(id); - - FileInfo info ; - if(mSearch->search(hash, RS_FILE_HINTS_CACHE, info)) - { -#ifdef CONTROL_DEBUG - std::cerr << "I already have this file:" << std::endl ; - std::cerr << " path: " << info.path << std::endl ; - std::cerr << " fname: " << info.fname << std::endl ; - std::cerr << " hash: " << info.hash << std::endl ; - - std::cerr << "Copying it !!" << std::endl ; -#endif - - if(info.size > 0 && copyFile(info.path,path+"/"+hash.toStdString())) - { - CompletedCache(hash); - return true ; - } - else - return false ; - } - - FileRequest(hash.toStdString(), hash, size, path, RS_FILE_REQ_CACHE | RS_FILE_REQ_NO_SEARCH, ids); - - return true; -} - - -bool ftController::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ - std::cerr << "ftController::CancelCacheFile(" << id << ","; - std::cerr << path << "," << hash << "," << size << ")"; - std::cerr << std::endl; -#ifdef CONTROL_DEBUG -#endif - - return FileCancel(hash); - -} - const std::string active_downloads_size_ss("MAX_ACTIVE_DOWNLOADS"); const std::string min_prioritized_downl_ss("MIN_PRORITIZED_DOWNLOADS"); const std::string download_dir_ss("DOWN_DIR"); @@ -2056,14 +1900,6 @@ bool ftController::saveList(bool &cleanup, std::list& saveData) /* ignore cache files. As this is small files, better download them again from scratch at restart.*/ - if (fit->second->mFlags & RS_FILE_REQ_CACHE) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftcontroller::saveList(): Not saving (callback) file entry " << fit->second->mName << ", " << fit->second->mHash << ", " << fit->second->mSize << std::endl ; -#endif - continue; - } - // Node: We still save finished transfers. This keeps transfers that are // in checking mode. Finished or checked transfers will restart and // immediately terminate/recheck at next startup. diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index fe20a7845..5ee84623b 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -113,12 +113,12 @@ class ftPendingRequest }; -class ftController: public CacheTransfer, public RsTickingThread, public pqiServiceMonitor, public p3Config +class ftController: public RsTickingThread, public pqiServiceMonitor, public p3Config { public: /* Setup */ - ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId); + ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId); void setFtSearchNExtra(ftSearch *, ftExtraList *); void setTurtleRouter(p3turtle *) ; @@ -195,10 +195,6 @@ class ftController: public CacheTransfer, public RsTickingThread, public pqiServ protected: - virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - - void cleanCacheDownloads() ; void searchForDirectSources() ; void tickTransfers() ; diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index a30545b38..ad16e0e4c 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -1104,8 +1104,6 @@ bool ftDataMultiplex::handleSearchRequest(const RsPeerId& peerId, const RsFileHa if(rsTurtle->isTurtlePeer(peerId)) hintflags |= RS_FILE_HINTS_NETWORK_WIDE ; - else - hintflags |= RS_FILE_HINTS_CACHE ; if(mSearch->search(hash, hintflags, info)) { diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index a861d79b7..f78784bee 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -31,6 +31,7 @@ #include "retroshare/rspeers.h" const int ftserverzone = 29539; +#include "file_sharing/p3filelists.h" #include "ft/ftturtlefiletransferitem.h" #include "ft/ftserver.h" #include "ft/ftextralist.h" @@ -64,15 +65,11 @@ static const time_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priorit /* Setup */ ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc) : p3Service(), - mPeerMgr(pm), - mServiceCtrl(sc), - mCacheStrapper(NULL), - mFiStore(NULL), mFiMon(NULL), + mPeerMgr(pm), mServiceCtrl(sc), + mFileDatabase(NULL), mFtController(NULL), mFtExtra(NULL), mFtDataplex(NULL), mFtSearch(NULL), srvMutex("ftServer") { - mCacheStrapper = new ftCacheStrapper(sc, getServiceInfo().mServiceType); - addSerialType(new RsFileTransferSerialiser()) ; } @@ -145,7 +142,7 @@ void ftServer::SetupFtServer() mFtDataplex = new ftDataMultiplex(ownId, this, mFtSearch); /* make Controller */ - mFtController = new ftController(mCacheStrapper, mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType); + mFtController = new ftController(mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType); mFtController -> setFtSearchNExtra(mFtSearch, mFtExtra); std::string tmppath = "."; mFtController->setPartialsDirectory(tmppath); @@ -153,21 +150,13 @@ void ftServer::SetupFtServer() /* Make Cache Source/Store */ - mFiStore = new ftFiStore(mCacheStrapper, mFtController, mPeerMgr, ownId, remotecachedir); - mFiMon = new ftFiMonitor(mCacheStrapper,localcachedir, ownId,mConfigPath); - - /* now add the set to the cachestrapper */ - CachePair cp(mFiMon, mFiStore, CacheId(RS_SERVICE_TYPE_FILE_INDEX, 0)); - mCacheStrapper -> addCachePair(cp); + mFileDatabase = new p3FileDatabase(mServiceCtrl) ; /* complete search setup */ - mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE); mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA); - mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL); - mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE); + mFtSearch->addSearchMode(mFileDatabase, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType); - mServiceCtrl->registerServiceMonitor(mCacheStrapper, getServiceInfo().mServiceType); return; } @@ -193,7 +182,7 @@ void ftServer::StartupThreads() /* startup Monitor Thread */ /* startup the FileMonitor (after cache load) */ /* start it up */ - mFiMon->start("ft monitor"); + mFileDatabase->startThreads(); /* Controller thread */ mFtController->start("ft ctrl"); @@ -210,9 +199,6 @@ void ftServer::StopThreads() /* stop Controller thread */ mFtController->join(); - /* stop Monitor Thread */ - mFiMon->join(); - /* self contained threads */ /* stop ExtraList Thread */ mFtExtra->join(); @@ -223,24 +209,15 @@ void ftServer::StopThreads() delete (mFtController); mFtController = NULL; - delete (mFiMon); - mFiMon = NULL; - delete (mFtExtra); mFtExtra = NULL; -} -CacheStrapper *ftServer::getCacheStrapper() -{ - return mCacheStrapper; + /* stop Monitor Thread */ + mFileDatabase->stopThreads(); + delete mFileDatabase; + mFileDatabase = NULL ; } -CacheTransfer *ftServer::getCacheTransfer() -{ - return mFtController; -} - - /***************************************************************/ /********************** RsFiles Interface **********************/ /***************************************************************/ @@ -256,11 +233,6 @@ bool ftServer::ResumeTransfers() return true; } -bool ftServer::checkHash(const RsFileHash& /*hash*/, std::string& /*error_string*/) -{ - return true ; -} - bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data) { return mFtDataplex->getFileData(hash, offset, requested_size,data); @@ -273,15 +245,7 @@ bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) { - std::string error_string ; - - if(!checkHash(hash,error_string)) - { - RsServer::notify()->notifyErrorMsg(0,0,"Error handling hash \""+hash.toStdString()+"\". This hash appears to be invalid(Error string=\""+error_string+"\"). This is probably due an bad handling of strings.") ; - return false ; - } - - std::cerr << "Requesting " << fname << std::endl ; + std::cerr << "Requesting " << fname << std::endl ; if(!mFtController->FileRequest(fname, hash, size, dest, flags, srcIds)) return false ; @@ -402,16 +366,18 @@ std::string ftServer::getPartialsDirectory() return mFtController->getPartialsDirectory(); } +/***************************************************************/ +/************************* Other Access ************************/ +/***************************************************************/ - /***************************************************************/ - /************************* Other Access ************************/ - /***************************************************************/ +bool ftServer::copyFile(const std::string& source, const std::string& dest) +{ + return mFtController->copyFile(source, dest); +} void ftServer::FileDownloads(std::list &hashs) { mFtController->FileDownloads(hashs); - /* this only contains downloads.... not completed */ - //return mFtDataplex->FileDownloads(hashs); } bool ftServer::FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& cmap) @@ -438,7 +404,7 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi // file, we skip the call to fileDetails() for efficiency reasons. // FileInfo info2 ; - if( (!(info.transfer_info_flags & RS_FILE_REQ_CACHE)) && mFtController->FileDetails(hash, info2)) + if(mFtController->FileDetails(hash, info2)) info.fname = info2.fname ; return true ; @@ -539,12 +505,11 @@ bool ftServer::handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_i return res ; } - /***************************************************************/ - /******************* ExtraFileList Access **********************/ - /***************************************************************/ +/***************************************************************/ +/******************* ExtraFileList Access **********************/ +/***************************************************************/ -bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, - uint32_t period, TransferRequestFlags flags) +bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags) { return mFtExtra->addExtraFile(fname, hash, size, period, flags); } @@ -564,73 +529,27 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info) return mFtExtra->hashExtraFileDone(localpath, info); } -bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, - std::string destpath) +bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath) { return mFtExtra->moveExtraFile(fname, hash, size, destpath); } - - /***************************************************************/ - /******************** Directory Listing ************************/ - /***************************************************************/ +/***************************************************************/ +/******************** Directory Listing ************************/ +/***************************************************************/ int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(uid:" << uid; - std::cerr << ", path:" << path << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } -#endif - if(uid == mServiceCtrl->getOwnId()) - return mFiMon->RequestDirDetails(path, details); - else - return mFiStore->RequestDirDetails(uid, path, details); + return mFileDatabase->RequestDirDetails(uid, path, details); } int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(ref:" << ref; - std::cerr << ", flags:" << flags << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->RequestDirDetails(ref, details, flags); - else - return mFiStore->RequestDirDetails(ref, details, flags); + return mFileDatabase->RequestDirDetails(ref,details,flags) ; } uint32_t ftServer::getType(void *ref, FileSearchFlags flags) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(ref:" << ref; - std::cerr << ", flags:" << flags << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->getType(ref); - else - return mFiStore->getType(ref); + return mFileDatabase->getType(ref) ; } /***************************************************************/ /******************** Search Interface *************************/ @@ -639,86 +558,55 @@ uint32_t ftServer::getType(void *ref, FileSearchFlags flags) int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchKeywords(keywords, results,flags,RsPeerId()); - else - return mFiStore->SearchKeywords(keywords, results,flags); - return 0 ; + return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId()); } int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::SearchKeywords()"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchKeywords(keywords, results,flags,peer_id); - else - return mFiStore->SearchKeywords(keywords, results,flags); + return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id); } int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchBoolExp(exp,results,flags,RsPeerId()) ; - else - return mFiStore->searchBoolExp(exp, results); - return 0 ; + return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId()); } int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchBoolExp(exp,results,flags,peer_id) ; - else - return mFiStore->searchBoolExp(exp, results); + return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ; } - /***************************************************************/ /*************** Local Shared Dir Interface ********************/ /***************************************************************/ -bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath) +bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath) { - return mFiMon->convertSharedFilePath(path, fullpath); + return mFileDatabase->convertSharedFilePath(path, fullpath); } void ftServer::updateSinceGroupPermissionsChanged() { - mFiMon->forceDirListsRebuildAndSend(); + mFileDatabase->forceSyncWithPeers(); } void ftServer::ForceDirectoryCheck() { - mFiMon->forceDirectoryCheck(); + mFileDatabase->forceDirectoryCheck(); return; } bool ftServer::InDirectoryCheck() { - return mFiMon->inDirectoryCheck(); -} - -bool ftServer::copyFile(const std::string& source, const std::string& dest) -{ - return mFtController->copyFile(source, dest); + return mFileDatabase->inDirectoryCheck(); } bool ftServer::getSharedDirectories(std::list &dirs) { - mFiMon->getSharedDirectories(dirs); + mFileDatabase->getSharedDirectories(dirs); return true; } bool ftServer::setSharedDirectories(std::list &dirs) { - mFiMon->setSharedDirectories(dirs); + mFileDatabase->setSharedDirectories(dirs); return true; } @@ -728,7 +616,7 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir) _dir.filename = RsDirUtil::convertPathToUnix(_dir.filename); std::list dirList; - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); // check that the directory is not already in the list. for(std::list::const_iterator it(dirList.begin());it!=dirList.end();++it) @@ -738,13 +626,13 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir) // ok then, add the shared directory. dirList.push_back(_dir); - mFiMon->setSharedDirectories(dirList); + mFileDatabase->setSharedDirectories(dirList); return true; } bool ftServer::updateShareFlags(const SharedDirInfo& info) { - mFiMon->updateShareFlags(info); + mFileDatabase->updateShareFlags(info); return true ; } @@ -761,7 +649,7 @@ bool ftServer::removeSharedDirectory(std::string dir) std::cerr << std::endl; #endif - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); #ifdef SERVER_DEBUG for(it = dirList.begin(); it != dirList.end(); ++it) @@ -776,12 +664,7 @@ bool ftServer::removeSharedDirectory(std::string dir) if(it == dirList.end()) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::removeSharedDirectory()"; - std::cerr << " Cannot Find Directory... Fail"; - std::cerr << std::endl; -#endif - + std::cerr << "(EE) ftServer::removeSharedDirectory(): Cannot Find Directory... Fail" << std::endl; return false; } @@ -793,44 +676,44 @@ bool ftServer::removeSharedDirectory(std::string dir) #endif dirList.erase(it); - mFiMon->setSharedDirectories(dirList); + mFileDatabase->setSharedDirectories(dirList); return true; } void ftServer::setWatchPeriod(int minutes) { - mFiMon->setWatchPeriod(minutes*60) ; + mFileDatabase->setWatchPeriod(minutes*60) ; } int ftServer::watchPeriod() const { - return mFiMon->watchPeriod()/60 ; + return mFileDatabase->watchPeriod()/60 ; } void ftServer::setRememberHashFiles(bool b) { - mFiMon->setRememberHashCache(b) ; + mFileDatabase->setRememberHashCache(b) ; } bool ftServer::rememberHashFiles() const { - return mFiMon->rememberHashCache() ; + return mFileDatabase->rememberHashCache() ; } void ftServer::setRememberHashFilesDuration(uint32_t days) { - mFiMon->setRememberHashCacheDuration(days) ; + mFileDatabase->setRememberHashCacheDuration(days) ; } uint32_t ftServer::rememberHashFilesDuration() const { - return mFiMon->rememberHashCacheDuration() ; + return mFileDatabase->rememberHashCacheDuration() ; } void ftServer::clearHashCache() { - mFiMon->clearHashCache() ; + mFileDatabase->clearHashCache() ; } bool ftServer::getShareDownloadDirectory() { std::list dirList; - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); std::string dir = mFtController->getDownloadDirectory(); @@ -844,7 +727,8 @@ bool ftServer::getShareDownloadDirectory() bool ftServer::shareDownloadDirectory(bool share) { - if (share) { + if (share) + { /* Share */ SharedDirInfo inf ; inf.filename = mFtController->getDownloadDirectory(); @@ -852,48 +736,22 @@ bool ftServer::shareDownloadDirectory(bool share) return addSharedDirectory(inf); } - - /* Unshare */ - std::string dir = mFtController->getDownloadDirectory(); - return removeSharedDirectory(dir); + else + { + /* Unshare */ + std::string dir = mFtController->getDownloadDirectory(); + return removeSharedDirectory(dir); + } } /***************************************************************/ /****************** End of RsFiles Interface *******************/ /***************************************************************/ - - /***************************************************************/ - /**************** Config Interface *****************************/ - /***************************************************************/ - -/* Key Functions to be overloaded for Full Configuration */ -RsSerialiser *ftServer::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser ; - rss->addSerialType(new RsFileTransferSerialiser) ; - - //rss->addSerialType(new RsGeneralConfigSerialiser()); - - return rss ; -} - -bool ftServer::saveList(bool &/*cleanup*/, std::list& /*list*/) -{ - - return true; -} - -bool ftServer::loadList(std::list& /*load*/) -{ - return true; -} - -bool ftServer::loadConfigMap(std::map &/*configMap*/) -{ - return true; -} - +//bool ftServer::loadConfigMap(std::map &/*configMap*/) +//{ +// return true; +//} /***************************************************************/ /********************** Data Flow **********************/ @@ -1257,9 +1115,6 @@ int ftServer::tick() if(handleIncoming()) moreToTick = true; - if(handleCacheData()) - moreToTick = true; - static time_t last_law_priority_tasks_handling_time = 0 ; time_t now = time(NULL) ; @@ -1275,45 +1130,6 @@ int ftServer::tick() return moreToTick; } -bool ftServer::handleCacheData() -{ - std::list > cacheUpdates; - std::list >::iterator it; - int i=0 ; - -#ifdef SERVER_DEBUG_CACHE - std::cerr << "handleCacheData() " << std::endl; -#endif - mCacheStrapper->getCacheUpdates(cacheUpdates); - for(it = cacheUpdates.begin(); it != cacheUpdates.end(); ++it) - { - /* construct reply */ - RsFileTransferCacheItem *ci = new RsFileTransferCacheItem(); - - /* id from incoming */ - ci -> PeerId(it->first); - - ci -> file.hash = (it->second).hash; - ci -> file.name = (it->second).name; - ci -> file.path = ""; // (it->second).path; - ci -> file.filesize = (it->second).size; - ci -> cacheType = (it->second).cid.type; - ci -> cacheSubId = (it->second).cid.subid; - -#ifdef SERVER_DEBUG_CACHE - std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n"; - ci -> print_string(out2); - std::cerr << out2 << std::endl; -#endif - - sendItem(ci); - - i++; - } - - return i>0 ; -} - int ftServer::handleIncoming() { // now File Input. @@ -1411,42 +1227,6 @@ int ftServer::handleIncoming() } } break ; - - case RS_PKT_SUBTYPE_FT_CACHE_ITEM: - { - RsFileTransferCacheItem *ci = dynamic_cast(item) ; - if (ci) - { -#ifdef SERVER_DEBUG_CACHE - std::cerr << "ftServer::handleIncoming: received cache item hash=" << ci->file.hash << ". from peer " << ci->PeerId() << std::endl; -#endif - /* these go to the CacheStrapper! */ - RsCacheData data; - data.pid = ci->PeerId(); - data.cid = CacheId(ci->cacheType, ci->cacheSubId); - data.path = ci->file.path; - data.name = ci->file.name; - data.hash = ci->file.hash; - data.size = ci->file.filesize; - data.recvd = time(NULL) ; - - mCacheStrapper->recvCacheResponse(data, time(NULL)); - } - } - break ; - -// case RS_PKT_SUBTYPE_FT_CACHE_REQUEST: -// { -// // do nothing -// RsFileTransferCacheRequestItem *cr = dynamic_cast(item) ; -// if (cr) -// { -//#ifdef SERVER_DEBUG_CACHE -// std::cerr << "ftServer::handleIncoming: received cache request from peer " << cr->PeerId() << std::endl; -//#endif -// } -// } -// break ; } delete item ; @@ -1465,7 +1245,7 @@ int ftServer::handleIncoming() bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr) { /* add all the subbits to config mgr */ - cfgmgr->addConfiguration("ft_shared.cfg", mFiMon); + cfgmgr->addConfiguration("ft_database.cfg", mFileDatabase); cfgmgr->addConfiguration("ft_extra.cfg", mFtExtra); cfgmgr->addConfiguration("ft_transfers.cfg", mFtController); diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 0c5b12aa7..1b66db710 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -70,230 +70,203 @@ class p3turtle; class p3PeerMgr; class p3ServiceControl; +class p3FileDatabase; class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService { - public: +public: - /***************************************************************/ - /******************** Setup ************************************/ - /***************************************************************/ + /***************************************************************/ + /******************** Setup ************************************/ + /***************************************************************/ - ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl); - virtual RsServiceInfo getServiceInfo(); + ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl); + virtual RsServiceInfo getServiceInfo(); - /* Assign important variables */ - void setConfigDirectory(std::string path); + /* Assign important variables */ + void setConfigDirectory(std::string path); - /* add Config Items (Extra, Controller) */ - void addConfigComponents(p3ConfigMgr *mgr); + /* add Config Items (Extra, Controller) */ + void addConfigComponents(p3ConfigMgr *mgr); - virtual CacheStrapper *getCacheStrapper(); - virtual CacheTransfer *getCacheTransfer(); + const RsPeerId& OwnId(); - const RsPeerId& OwnId(); + /* Final Setup (once everything is assigned) */ + void SetupFtServer() ; + virtual void connectToTurtleRouter(p3turtle *p) ; - /* Final Setup (once everything is assigned) */ - void SetupFtServer() ; - virtual void connectToTurtleRouter(p3turtle *p) ; + // Implements RsTurtleClientService + // + virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; + virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ; - // Checks that the given hash is well formed. Used to chase - // string bugs. - static bool checkHash(const RsFileHash& hash,std::string& error_string) ; + void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; + void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; - // Implements RsTurtleClientService - // - virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; - virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ; + /***************************************************************/ + /*************** Control Interface *****************************/ + /************** (Implements RsFiles) ***************************/ + /***************************************************************/ - void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; - void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; + void StartupThreads(); + void StopThreads(); - /***************************************************************/ - /*************** Control Interface *****************************/ - /************** (Implements RsFiles) ***************************/ - /***************************************************************/ + // member access - void StartupThreads(); - void StopThreads(); + ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } + ftController *getController() const { return mFtController ; } - // member access - - ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } - ftController *getController() const { return mFtController ; } - - /** + /** * @see RsFiles::getFileData */ - bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data); + bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data); - /*** - * Control of Downloads - ***/ - virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info); - virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds); - virtual bool FileCancel(const RsFileHash& hash); - virtual bool FileControl(const RsFileHash& hash, uint32_t flags); - virtual bool FileClearCompleted(); - virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ; - virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ; - virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ; - virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ; - virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ; - virtual uint32_t freeDiskSpaceLimit() const ; - virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ; + /*** + * Control of Downloads + ***/ + virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info); + virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds); + virtual bool FileCancel(const RsFileHash& hash); + virtual bool FileControl(const RsFileHash& hash, uint32_t flags); + virtual bool FileClearCompleted(); + virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ; + virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ; + virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ; + virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ; + virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ; + virtual uint32_t freeDiskSpaceLimit() const ; + virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ; - /*** - * Control of Downloads Priority. - ***/ - virtual uint32_t getMinPrioritizedTransfers() ; - virtual void setMinPrioritizedTransfers(uint32_t s) ; - virtual uint32_t getQueueSize() ; - virtual void setQueueSize(uint32_t s) ; - virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); - virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed); - virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed); - virtual bool clearDownload(const RsFileHash& hash); - //virtual void getDwlDetails(std::list & details); + /*** + * Control of Downloads Priority. + ***/ + virtual uint32_t getMinPrioritizedTransfers() ; + virtual void setMinPrioritizedTransfers(uint32_t s) ; + virtual uint32_t getQueueSize() ; + virtual void setQueueSize(uint32_t s) ; + virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); + virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed); + virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed); + virtual bool clearDownload(const RsFileHash& hash); + //virtual void getDwlDetails(std::list & details); - /*** - * Download/Upload Details - ***/ - virtual void FileDownloads(std::list &hashs); - virtual bool FileUploads(std::list &hashs); - virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info); - virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ; - virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ; + /*** + * Download/Upload Details + ***/ + virtual void FileDownloads(std::list &hashs); + virtual bool FileUploads(std::list &hashs); + virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info); + virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ; + virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ; - /*** - * Extra List Access - ***/ - virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags); - virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags); - virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags); - virtual bool ExtraFileStatus(std::string localpath, FileInfo &info); - virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath); + /*** + * Extra List Access + ***/ + virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags); + virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags); + virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags); + virtual bool ExtraFileStatus(std::string localpath, FileInfo &info); + virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath); - /*** - * Directory Listing / Search Interface - ***/ - virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); - virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); - virtual uint32_t getType(void *ref,FileSearchFlags flags) ; + /*** + * Directory Listing / Search Interface + ***/ + virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); + virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); + virtual uint32_t getType(void *ref,FileSearchFlags flags) ; - virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); - virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); + virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); + virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - /*** - * Utility Functions - ***/ - virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath); - virtual void ForceDirectoryCheck(); - virtual void updateSinceGroupPermissionsChanged() ; - virtual bool InDirectoryCheck(); - virtual bool copyFile(const std::string& source, const std::string& dest); + /*** + * Utility Functions + ***/ + virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath); + virtual void ForceDirectoryCheck(); + virtual void updateSinceGroupPermissionsChanged() ; + virtual bool InDirectoryCheck(); + virtual bool copyFile(const std::string& source, const std::string& dest); - /*** - * Directory Handling - ***/ - virtual void setDownloadDirectory(std::string path); - virtual void setPartialsDirectory(std::string path); - virtual std::string getDownloadDirectory(); - virtual std::string getPartialsDirectory(); + /*** + * Directory Handling + ***/ + virtual void setDownloadDirectory(std::string path); + virtual void setPartialsDirectory(std::string path); + virtual std::string getDownloadDirectory(); + virtual std::string getPartialsDirectory(); - virtual bool getSharedDirectories(std::list &dirs); - virtual bool setSharedDirectories(std::list &dirs); - virtual bool addSharedDirectory(const SharedDirInfo& dir); - virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist ! - virtual bool removeSharedDirectory(std::string dir); + virtual bool getSharedDirectories(std::list &dirs); + virtual bool setSharedDirectories(std::list &dirs); + virtual bool addSharedDirectory(const SharedDirInfo& dir); + virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist ! + virtual bool removeSharedDirectory(std::string dir); - virtual bool getShareDownloadDirectory(); - virtual bool shareDownloadDirectory(bool share); + virtual bool getShareDownloadDirectory(); + virtual bool shareDownloadDirectory(bool share); - virtual void setRememberHashFilesDuration(uint32_t days) ; - virtual uint32_t rememberHashFilesDuration() const ; - virtual bool rememberHashFiles() const ; - virtual void setRememberHashFiles(bool) ; - virtual void clearHashCache() ; - virtual void setWatchPeriod(int minutes) ; - virtual int watchPeriod() const ; + virtual void setRememberHashFilesDuration(uint32_t days) ; + virtual uint32_t rememberHashFilesDuration() const ; + virtual bool rememberHashFiles() const ; + virtual void setRememberHashFiles(bool) ; + virtual void clearHashCache() ; + virtual void setWatchPeriod(int minutes) ; + virtual int watchPeriod() const ; - /***************************************************************/ - /*************** Control Interface *****************************/ - /***************************************************************/ + /***************************************************************/ + /*************** Data Transfer Interface ***********************/ + /***************************************************************/ +public: + virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); + virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize); + virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ; + virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ; + virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ; + virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ; - /***************************************************************/ - /*************** Data Transfer Interface ***********************/ - /***************************************************************/ - public: - virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); - virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize); - virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ; - virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ; - virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ; - virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ; + /*************** Internal Transfer Fns *************************/ + virtual int tick(); - /*************** Internal Transfer Fns *************************/ - virtual int tick(); + /* Configuration */ + bool addConfiguration(p3ConfigMgr *cfgmgr); + bool ResumeTransfers(); - /* Configuration */ - bool addConfiguration(p3ConfigMgr *cfgmgr); - bool ResumeTransfers(); + /*************************** p3 Config Overload ********************/ - /******************* p3 Config Overload ************************/ - protected: - /* Key Functions to be overloaded for Full Configuration */ - virtual RsSerialiser *setupSerialiser(); - virtual bool saveList(bool &cleanup, std::list&); - virtual bool loadList(std::list& load); +protected: + int handleIncoming() ; + bool handleCacheData() ; - private: - bool loadConfigMap(std::map &configMap); - /******************* p3 Config Overload ************************/ +private: - /*************************** p3 Config Overload ********************/ + /**** INTERNAL FUNCTIONS ***/ + //virtual int reScanDirs(); + //virtual int check_dBUpdate(); - protected: - int handleIncoming() ; - bool handleCacheData() ; +private: - private: + /* no need for Mutex protection - + * as each component is protected independently. + */ + p3PeerMgr *mPeerMgr; + p3ServiceControl *mServiceCtrl; + p3FileDatabase *mFileDatabase ; + ftController *mFtController; + ftExtraList *mFtExtra; + ftDataMultiplex *mFtDataplex; + p3turtle *mTurtleRouter ; + ftFileSearch *mFtSearch; - /**** INTERNAL FUNCTIONS ***/ - //virtual int reScanDirs(); - //virtual int check_dBUpdate(); - - private: - - /* no need for Mutex protection - - * as each component is protected independently. - */ - p3PeerMgr *mPeerMgr; - p3ServiceControl *mServiceCtrl; - - ftCacheStrapper *mCacheStrapper; - ftFiStore *mFiStore; - ftFiMonitor *mFiMon; - - ftController *mFtController; - ftExtraList *mFtExtra; - - ftDataMultiplex *mFtDataplex; - p3turtle *mTurtleRouter ; - - ftFileSearch *mFtSearch; - - RsMutex srvMutex; - std::string mConfigPath; - std::string mDownloadPath; - std::string mPartialsPath; + RsMutex srvMutex; + std::string mConfigPath; + std::string mDownloadPath; + std::string mPartialsPath; }; diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 31ee2a39f..af1bfbb65 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -410,12 +410,13 @@ void RsPluginManager::slowTickPlugins(time_t seconds) void RsPluginManager::registerCacheServices() { +#warning this code should go std::cerr << " Registering cache services." << std::endl; for(uint32_t i=0;i<_plugins.size();++i) if(_plugins[i].plugin != NULL && _plugins[i].plugin->rs_cache_service() != NULL) { - rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; + //rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ; } } diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index c6480b540..6fd1967d7 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -65,26 +65,26 @@ const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000; // Flags used when requesting info about transfers, mostly to filter out the result. // -const FileSearchFlags RS_FILE_HINTS_CACHE ( 0x00000001 ); -const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 ); -const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 ); -const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 ); +const FileSearchFlags RS_FILE_HINTS_CACHE_deprecated ( 0x00000001 ); +const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 ); +const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 ); +const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 ); const FileSearchFlags RS_FILE_HINTS_DOWNLOAD ( 0x00000010 ); -const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 ); -const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 ); +const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 ); +const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 ); -const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network -const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends -const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out. +const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network +const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends +const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out. // Flags used when requesting a transfer // const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING ( 0x00000040 ); // Use to ask turtle router to download the file. const TransferRequestFlags RS_FILE_REQ_ASSUME_AVAILABILITY ( 0x00000200 ); // Assume full source availability. Used for cache files. -const TransferRequestFlags RS_FILE_REQ_CACHE ( 0x00000400 ); // Assume full source availability. Used for cache files. +const TransferRequestFlags RS_FILE_REQ_CACHE_deprecated ( 0x00000400 ); // Assume full source availability. Used for cache files. const TransferRequestFlags RS_FILE_REQ_EXTRA ( 0x00000800 ); const TransferRequestFlags RS_FILE_REQ_MEDIA ( 0x00001000 ); -const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly. +const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly. const TransferRequestFlags RS_FILE_REQ_NO_SEARCH ( 0x02000000 ); // disable searching for potential direct sources. // const uint32_t RS_FILE_HINTS_SHARE_FLAGS_MASK = RS_FILE_HINTS_NETWORK_WIDE_OTHERS | RS_FILE_HINTS_BROWSABLE_OTHERS @@ -222,9 +222,6 @@ class RsFiles virtual void setWatchPeriod(int minutes) =0; virtual int watchPeriod() const =0; - virtual CacheStrapper *getCacheStrapper() =0; - virtual CacheTransfer *getCacheTransfer() =0; - virtual bool getShareDownloadDirectory() = 0; virtual bool shareDownloadDirectory(bool share) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 733009932..40c3417ff 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1228,8 +1228,6 @@ int RsServer::StartupRetroShare() ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); ftserver->SetupFtServer() ; - CacheStrapper *mCacheStrapper = ftserver->getCacheStrapper(); - //CacheTransfer *mCacheTransfer = ftserver->getCacheTransfer(); /* setup any extra bits (Default Paths) */ ftserver->setPartialsDirectory(emergencyPartialsDir); @@ -1273,7 +1271,6 @@ int RsServer::StartupRetroShare() // These are needed to load plugins: plugin devs might want to know the place of // cache directories, get pointers to cache strapper, or access ownId() // - mPluginsManager->setCacheDirectories(localcachedir,remotecachedir) ; mPluginsManager->setServiceControl(serviceCtrl) ; // std::cerr << "rsinitconf (core 1) = " << (void*)rsInitConfig<addMonitor(serviceCtrl); mLinkMgr->addMonitor(serviceInfo); - // NOTE these were added in ftServer (was added twice). - //mLinkMgr->addMonitor(mCacheStrapper); - //mLinkMgr->addMonitor(((ftController *) mCacheTransfer)); - // Services that have been changed to pqiServiceMonitor serviceCtrl->registerServiceMonitor(msgSrv, msgSrv->getServiceInfo().mServiceType); serviceCtrl->registerServiceMonitor(mDisc, mDisc->getServiceInfo().mServiceType); @@ -1668,7 +1661,6 @@ int RsServer::StartupRetroShare() * after all the other configurations have happened. */ - mConfigMgr->addConfiguration("cache.cfg", mCacheStrapper); mConfigMgr->loadConfiguration(); /**************************************************************************/ diff --git a/libretroshare/src/serialiser/rsserviceids.h b/libretroshare/src/serialiser/rsserviceids.h index 7aadd3dea..475d92595 100644 --- a/libretroshare/src/serialiser/rsserviceids.h +++ b/libretroshare/src/serialiser/rsserviceids.h @@ -49,6 +49,7 @@ const uint16_t RS_SERVICE_TYPE_TUNNEL = 0x0015; const uint16_t RS_SERVICE_TYPE_HEARTBEAT = 0x0016; const uint16_t RS_SERVICE_TYPE_FILE_TRANSFER = 0x0017; const uint16_t RS_SERVICE_TYPE_GROUTER = 0x0018; +const uint16_t RS_SERVICE_TYPE_FILE_DATABASE = 0x0019; const uint16_t RS_SERVICE_TYPE_SERVICEINFO = 0x0020; /* Bandwidth Control */ diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index 3a7f5cf61..895b082c7 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -1269,13 +1269,6 @@ void TransfersDialog::insertTransfers() continue; } - if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) { - // if file transfer is a cache file index file, don't show it - DLListModel->removeRow(row); - rowCount = DLListModel->rowCount(); - continue; - } - hashs.erase(hashIt); if (addItem(row, fileInfo) < 0) { @@ -1295,11 +1288,6 @@ void TransfersDialog::insertTransfers() continue; } - if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) { - //if file transfer is a cache file index file, don't show it - continue; - } - addItem(-1, fileInfo); } @@ -1322,9 +1310,6 @@ void TransfersDialog::insertTransfers() if (!rsFiles->FileDetails(*it, RS_FILE_HINTS_UPLOAD, info)) continue; - if((info.transfer_info_flags & RS_FILE_REQ_CACHE) && _show_cache_transfers) - continue ; - std::list::iterator pit; for(pit = info.peers.begin(); pit != info.peers.end(); ++pit) { From 538a11fd14c3686aa6cad6a5d72c53381bb263ba Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 31 Jul 2016 15:59:58 +0200 Subject: [PATCH 16/92] fixed compilation. Added missing functions, left blank --- .../src/file_sharing/directory_storage.cc | 68 +++++++++- .../src/file_sharing/directory_storage.h | 27 ++-- libretroshare/src/file_sharing/p3filelists.cc | 128 +++++++++++++++++- libretroshare/src/file_sharing/p3filelists.h | 14 +- libretroshare/src/ft/ftserver.cc | 7 +- 5 files changed, 221 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d94cb64f0..8663dddda 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,4 +1,5 @@ #include +#include "util/rsdir.h" #include "directory_storage.h" /******************************************************************************************************************/ @@ -321,6 +322,8 @@ private: return true ; } + + std::map mHashes ; // used for fast search access }; /******************************************************************************************************************/ @@ -442,10 +445,71 @@ void DirectoryStorage::print() void LocalDirectoryStorage::setSharedDirectoryList(const std::list& lst) { RS_STACK_MUTEX(mDirStorageMtx) ; - mLocalDirs = lst ; + + for(std::list::const_iterator it(lst.begin());it!= lst.end();++it) + mLocalDirs[(*it).filename] = *it ; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { RS_STACK_MUTEX(mDirStorageMtx) ; - lst = mLocalDirs ; + + lst.clear(); + + for(std::map::iterator it(mLocalDirs.begin());it!=mLocalDirs.end();++it) + lst.push_back(it->second) ; } + +void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::map::iterator it = mLocalDirs.find(info.virtualname) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; + return ; + } + it->second = info; +} + +bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) +{ + std::string shpath = RsDirUtil::removeRootDir(path); + std::string basedir = RsDirUtil::getRootDir(path); + std::string realroot ; + { + RS_STACK_MUTEX(mDirStorageMtx) ; + realroot = locked_findRealRootFromVirtualFilename(basedir); + } + + if (realroot.empty()) + return false; + + /* construct full name */ + fullpath = realroot + "/"; + fullpath += shpath; + + return true; +} + +std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const +{ + /**** MUST ALREADY BE LOCKED ****/ + + std::map::const_iterator cit = mLocalDirs.find(virtual_rootdir) ; + + if (cit == mLocalDirs.end()) + { + std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: " << virtual_rootdir << std::endl; + return std::string(); + } + return cit->second.filename; +} + +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + NOT_IMPLEMENTED() ; + return false; +} + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index bd0f5b8ce..28ba00401 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -16,6 +16,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; +#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } + class InternalFileHierarchyStorage ; class DirectoryStorage @@ -29,9 +31,9 @@ class DirectoryStorage void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const { return 0;} - virtual int searchHash(const RsFileHash& hash, std::list &results) const { return 0;} - virtual int searchBoolExp(Expression * exp, std::list &results) const { return 0; } + virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} + virtual int searchHash(const RsFileHash& hash, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } + virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } void getFileDetails(EntryIndex i) ; @@ -112,7 +114,6 @@ class DirectoryStorage InternalFileHierarchyStorage *mFileHierarchy ; std::string mFileName; - protected: RsMutex mDirStorageMtx ; }; @@ -120,7 +121,7 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid) ; + RemoteDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} virtual ~RemoteDirectoryStorage() {} }; @@ -133,10 +134,20 @@ public: void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; -private: - std::list mLocalDirs ; + void updateShareFlags(const SharedDirInfo& info) ; + bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; - std::map mHashes ; // used for fast search access + /*! + * \brief getFileInfo Converts an index info a full file info structure. + * \param i index in the directory structure + * \param info structure to be filled in + * \return false if the file does not exist, or is a directory,... + */ + bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; +private: + std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; + + std::map mLocalDirs ; // map is better for search. it->first=it->second.filename }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0fd3a830c..ffc3d401f 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -5,6 +5,7 @@ #include "file_sharing/directory_updater.h" #include "retroshare/rsids.h" +#include "retroshare/rspeers.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " @@ -13,8 +14,6 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not implemented!" << std::endl; } - p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { @@ -41,6 +40,10 @@ void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; } +void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) +{ + mLocalSharedDirs->updateShareFlags(info) ; +} p3FileDatabase::~p3FileDatabase() { @@ -86,12 +89,26 @@ int p3FileDatabase::tick() // cleanup // - remove/delete shared file lists for people who are not friend anymore // - - cleanup(); return 0; } +void p3FileDatabase::startThreads() +{ + std::cerr << "Starting hash cache thread..." ; + + mHashCache->start(); +} +void p3FileDatabase::stopThreads() +{ + std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; + + mHashCache->fullstop(); + + std::cerr << "Done." << std::endl; +} + void p3FileDatabase::tickWatchers() { NOT_IMPLEMENTED(); @@ -156,7 +173,7 @@ void p3FileDatabase::cleanup() { P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; - mRemoteDirectories[*it] = new RemoteDirectoryStorage(*it); + mRemoteDirectories[*it] = new RemoteDirectoryStorage(makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -165,11 +182,22 @@ void p3FileDatabase::cleanup() IndicateConfigChanged(); } +std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const +{ +#warning we should use the default paths here. Ask p3config + return "dirlist_"+pid.toStdString()+".txt" ; +} + int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const { NOT_IMPLEMENTED(); return 0; } +int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) +{ + NOT_IMPLEMENTED(); + return 0; +} int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); @@ -189,7 +217,6 @@ bool p3FileDatabase::inDirectoryCheck() NOT_IMPLEMENTED(); return 0; } - void p3FileDatabase::setWatchPeriod(uint32_t seconds) { NOT_IMPLEMENTED(); @@ -221,3 +248,94 @@ void p3FileDatabase::setRememberHashCache(bool) { NOT_IMPLEMENTED(); } + +bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const +{ + std::list firesults; + mLocalSharedDirs->searchHash(hash,firesults) ; + + NOT_IMPLEMENTED(); + return false; +} + +int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) +{ + std::list firesults; + mLocalSharedDirs->searchTerms(keywords,firesults) ; + + return filterResults(firesults,results,flags,client_peer_id) ; +} +int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const +{ + std::list firesults; + mLocalSharedDirs->searchBoolExp(exp,firesults) ; + + return filterResults(firesults,results,flags,client_peer_id) ; +} +bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const +{ + if(hintflags & RS_FILE_HINTS_LOCAL) + { + std::list res; + mLocalSharedDirs->searchHash(hash,res) ; + + if(res.empty()) + return false; + + EntryIndex indx = *res.begin() ; // no need to report dupicates + + mLocalSharedDirs->getFileInfo(indx,info) ; + } + else + { + NOT_IMPLEMENTED(); + return 0; + } +} + +int p3FileDatabase::filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const +{ + results.clear(); + +#ifdef P3FILELISTS_DEBUG + if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) + std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; +#endif + /* translate/filter results */ + + for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) + { + DirDetails cdetails ; + RequestDirDetails ((void*)*rit,cdetails,FileSearchFlags(0u)); +#ifdef P3FILELISTS_DEBUG + std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; +#endif + + if(!peer_id.isNull()) + { + FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ; + + if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags )) + { + cdetails.id.clear() ; + results.push_back(cdetails); +#ifdef P3FILELISTS_DEBUG + std::cerr << ": kept" << std::endl ; +#endif + } +#ifdef P3FILELISTS_DEBUG + else + std::cerr << ": discarded" << std::endl ; +#endif + } + else + results.push_back(cdetails); + } + + return !results.empty() ; +} + +bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath) +{ + return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 7c0060183..0d631cdbd 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -26,6 +26,7 @@ #include "services/p3service.h" #include "file_sharing/hash_cache.h" +#include "file_sharing/directory_storage.h" #include "pqi/p3cfgmgr.h" #include "pqi/p3linkmgr.h" @@ -41,7 +42,7 @@ class HashStorage ; class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService { public: - typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + typedef DirectoryStorage::EntryIndex EntryIndex; // this should probably be defined elsewhere virtual RsServiceInfo getServiceInfo(); @@ -58,10 +59,10 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub /*! * \brief forceSyncWithPeers * - * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new gorup of friend is created, or when + * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new group of friend is created, or when * a friend was added/removed from a group. */ - void forceSyncWithPeers(); + void forceSyncWithPeers() { NOT_IMPLEMENTED() ; } // derived from p3Service // @@ -92,6 +93,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t getType(void *) const ; // set/update shared directories + void setSharedDirectories(const std::list& dirs); void getSharedDirectories(std::list& dirs); void updateShareFlags(const SharedDirInfo& info) ; @@ -116,11 +118,15 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void full_print(); protected: + + int filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const; + std::string makeRemoteFileName(const RsPeerId& pid) const; + // Derived from p3Config // virtual bool loadList(std::list& items); virtual bool saveList(bool &cleanup, std::list&); - virtual RsSerialiser *setupSerialiser() ; + virtual RsSerialiser *setupSerialiser() { return NULL;} void cleanup(); void tickRecv(); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index f78784bee..44b361384 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -551,10 +551,9 @@ uint32_t ftServer::getType(void *ref, FileSearchFlags flags) { return mFileDatabase->getType(ref) ; } - /***************************************************************/ - /******************** Search Interface *************************/ - /***************************************************************/ - +/***************************************************************/ +/******************** Search Interface *************************/ +/***************************************************************/ int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) { From 067d4231ae9eaf410f6c89c33e88e69728600e7a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 5 Aug 2016 22:00:25 +0200 Subject: [PATCH 17/92] fixed compilation --- .../src/file_sharing/directory_storage.h | 7 +- libretroshare/src/file_sharing/p3filelists.cc | 151 +++++++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 19 ++- 3 files changed, 149 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 28ba00401..a3a084612 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -121,8 +121,13 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname),mPeerId(pid) {} virtual ~RemoteDirectoryStorage() {} + + const RsPeerId& peerId() const { return mPeerId ; } + +private: + RsPeerId mPeerId; }; class LocalDirectoryStorage: public DirectoryStorage diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ffc3d401f..597c9292e 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -14,6 +14,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; +static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; +static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; +static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. + p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { @@ -49,8 +53,8 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();++it) - delete it->second ; + for(uint32_t i=0;i friend_set ; - mServCtrl->getPeersConnected(getServiceInfo().mServiceType, friend_set); + { + std::list friend_lst ; - for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) - if(friend_set.find(it->first) == friend_set.end()) + rsPeers->getFriendList(friend_lst); + + for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) + friend_set.insert(*it) ; + } + + for(uint32_t i=0;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; - delete it->second ; - std::map::iterator tmp(it) ; - ++tmp ; - mRemoteDirectories.erase(it) ; - it=tmp ; + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - } - else - ++it ; - // look through the list of friends, and add a directory storage when it's missing + friend_set.erase(mRemoteDirectories[i]->peerId()); + + mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); + mFriendIndexTab[i].clear(); + } + + // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. // for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) - if(mRemoteDirectories.find(*it) == mRemoteDirectories.end()) - { - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + { + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; - mRemoteDirectories[*it] = new RemoteDirectoryStorage(makeRemoteFileName(*it)); + uint32_t i; + for(i=0;i::const_iterator it = mFriendIndexMap.find(pid) ; + + if(it == mFriendIndexMap.end()) + { + // allocate a new index for that friend, and tell that we should save. + uint32_t found = 0 ; + for(uint32_t i=1;isecond; +} + +const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const +{ + static const RsPeerId null_id ; + + if(indx >= mFriendIndexTab.size()) + return null_id ; + + if(mFriendIndexTab[indx].isNull()) + { + std::cerr << "(EE) null friend id requested from index " << indx << ": this is a bug, most likely" << std::endl; + return null_id ; + } + + return mFriendIndexTab[indx]; +} +bool p3FileDatabase::convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) +{ + // trust me, I can do this ;-) + + e = EntryIndex( *reinterpret_cast(&p) & ENTRY_INDEX_BIT_MASK ) ; + friend_index = (*reinterpret_cast(&p)) >> NB_ENTRY_INDEX_BITS ; + + return true; +} +bool p3FileDatabase::convertEntryIndexToPointer(EntryIndex& e, uint32_t fi, void *& p) +{ + // the pointer is formed the following way: + // + // [ 10 bits | 22 bits ] + // + // This means that the whoel software has the following build-in limitation: + // * 1024 friends + // * 4M shared files. + + uint32_t fe = (uint32_t)e ; + + if(fi >= (1<= (1<< NB_ENTRY_INDEX_BITS)) + { + std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; + return false ; + } + + p = reinterpret_cast( (fi << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; + + return true; +} int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const { - NOT_IMPLEMENTED(); + uint32_t fi; + EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi) ; +#warning code needed here + return 0; } -int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) +int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); return 0; @@ -285,11 +386,13 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F EntryIndex indx = *res.begin() ; // no need to report dupicates mLocalSharedDirs->getFileInfo(indx,info) ; + + return true; } else { NOT_IMPLEMENTED(); - return 0; + return false; } } diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 0d631cdbd..d7c8e2250 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -84,7 +84,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void stopThreads() ; void startThreads() ; - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; int RequestDirDetails(const std::string& path, DirDetails &details) const ; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. @@ -144,13 +144,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // Directory storage hierarchies // - std::map mRemoteDirectories ; + // The remote one is the reference for the PeerId index below: + // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) + + std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; RemoteDirectoryUpdater *mRemoteDirWatcher ; LocalDirectoryUpdater *mLocalDirWatcher ; - // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. + // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc + + static bool convertEntryIndexToPointer(EntryIndex& e,uint32_t friend_index,void *& p); + static bool convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) ; + uint32_t getFriendIndex(const RsPeerId& pid); + const RsPeerId& getFriendFromIndex(uint32_t indx) const; + + std::map mFriendIndexMap ; + std::vector mFriendIndexTab; + + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // HashStorage *mHashCache ; From 43f4f5d2d9ed928b5a5b082ddcdf0f57d0cfaa15 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 5 Aug 2016 22:37:40 +0200 Subject: [PATCH 18/92] fixed threading issues in directory updater --- .../src/file_sharing/directory_updater.cc | 4 +++- libretroshare/src/file_sharing/hash_cache.cc | 16 ++++++++-------- libretroshare/src/file_sharing/p3filelists.cc | 12 ++++++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index b4ebfb0f5..bccae26cf 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -5,7 +5,7 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! void RemoteDirectoryUpdater::tick() { @@ -27,6 +27,8 @@ void LocalDirectoryUpdater::data_tick() sweepSharedDirectories() ; mLastSweepTime = now; } + else + usleep(10*1000*1000); } void LocalDirectoryUpdater::sweepSharedDirectories() diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index eb86813b3..58ca5d80a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -19,7 +19,15 @@ void HashStorage::data_tick() RS_STACK_MUTEX(mHashMtx) ; if(mFilesToHash.empty()) + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + + usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. return ; + } job = mFilesToHash.begin()->second ; @@ -33,14 +41,6 @@ void HashStorage::data_tick() mFilesToHash.erase(mFilesToHash.begin()) ; - if(mFilesToHash.empty()) - { - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; - } - // store the result HashStorageInfo& info(mFiles[job.full_path]); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 597c9292e..302bc778d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -31,8 +31,6 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; mRemoteDirWatcher = NULL; // not used yet - mLocalDirWatcher->start(); - mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } @@ -101,15 +99,21 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { std::cerr << "Starting hash cache thread..." ; - mHashCache->start(); + std::cerr << "Done." << std::endl; + + std::cerr << "Starting directory watcher thread..." ; + mLocalDirWatcher->start(); + std::cerr << "Done." << std::endl; } void p3FileDatabase::stopThreads() { std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; - mHashCache->fullstop(); + std::cerr << "Done." << std::endl; + std::cerr << "Stopping directory watcher thread..." ; std::cerr.flush() ; + mLocalDirWatcher->fullstop(); std::cerr << "Done." << std::endl; } From 2d1824b01d2b091a68802b4d850a1b3800df67f7 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 6 Aug 2016 19:04:54 +0200 Subject: [PATCH 19/92] fixed virtual names --- .../src/file_sharing/directory_storage.cc | 30 ++++++++++++++++++- .../src/file_sharing/directory_updater.cc | 9 ++++++ .../src/file_sharing/directory_updater.h | 4 ++- libretroshare/src/file_sharing/hash_cache.cc | 7 +++++ libretroshare/src/file_sharing/p3filelists.cc | 7 +++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8663dddda..b91bbe4cb 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,6 @@ #include #include "util/rsdir.h" +#include "util/rsstring.h" #include "directory_storage.h" /******************************************************************************************************************/ @@ -374,6 +375,7 @@ time_t DirectoryStorage::FileIterator::modtime() const { const InternalFil DirectoryStorage::DirectoryStorage(const std::string& local_file_name) : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) { + RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); } @@ -446,8 +448,33 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list virtual_names ; // maps virtual to real name + std::list processed_list ; + for(std::list::const_iterator it(lst.begin());it!= lst.end();++it) - mLocalDirs[(*it).filename] = *it ; + { + int i=0; + std::string candidate_virtual_name = it->virtualname ; + + if(candidate_virtual_name.empty()) + candidate_virtual_name = RsDirUtil::getTopDir(it->filename); + + while(virtual_names.find(candidate_virtual_name) != virtual_names.end()) + rs_sprintf_append(candidate_virtual_name, "-%d", ++i); + + SharedDirInfo d(*it); + d.virtualname = candidate_virtual_name ; + processed_list.push_back(d) ; + + virtual_names.insert(candidate_virtual_name) ; + } + + mLocalDirs.clear(); + + for(std::list::const_iterator it(processed_list.begin());it!=processed_list.end();++it) + mLocalDirs[it->filename] = *it; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { @@ -509,6 +536,7 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { + RS_STACK_MUTEX(mDirStorageMtx) ; NOT_IMPLEMENTED() ; return false; } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index bccae26cf..149937901 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,4 +1,5 @@ #include "util/folderiterator.h" +#include "rsserver/p3face.h" #include "directory_storage.h" #include "directory_updater.h" @@ -31,8 +32,15 @@ void LocalDirectoryUpdater::data_tick() usleep(10*1000*1000); } +void LocalDirectoryUpdater::forceUpdate() +{ + mLastSweepTime = 0; +} + void LocalDirectoryUpdater::sweepSharedDirectories() { + RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; // recursive update algorithm works that way: @@ -62,6 +70,7 @@ void LocalDirectoryUpdater::sweepSharedDirectories() std::cerr << " recursing into " << real_dir_it->filename << std::endl; recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; } + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 2d30793cd..d146a61b5 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -19,9 +19,11 @@ public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void data_tick() ; + virtual void forceUpdate(); protected: + virtual void data_tick() ; + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); void sweepSharedDirectories(); diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 58ca5d80a..32401ca4b 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,4 +1,5 @@ #include "util/rsdir.h" +#include "rsserver/p3face.h" #include "hash_cache.h" #define HASHSTORAGE_DEBUG 1 @@ -25,6 +26,8 @@ void HashStorage::data_tick() mRunning = false ; std::cerr << "done." << std::endl; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. return ; } @@ -33,6 +36,10 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + std::string tmpout; + //rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", cnt+1, n_files, friendlyUnit(size).c_str(), int(size/double(total_size)*100.0), fe.name.c_str()) ; + + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, job.full_path) ; if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 302bc778d..299d90029 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -7,6 +7,8 @@ #include "retroshare/rsids.h" #include "retroshare/rspeers.h" +#include "rsserver/p3face.h" + #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; @@ -36,14 +38,19 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; + mLocalDirWatcher->forceUpdate(); } void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; } void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->updateShareFlags(info) ; } From 82c18bbb868093da319d51b4a1ab2346ed70a770 Mon Sep 17 00:00:00 2001 From: MrAlice Date: Tue, 9 Aug 2016 15:07:02 +0200 Subject: [PATCH 20/92] added load/save for file hash cache --- .../src/file_sharing/directory_storage.h | 9 - libretroshare/src/file_sharing/filelist_io.cc | 2 + libretroshare/src/file_sharing/filelist_io.h | 58 ++++++ libretroshare/src/file_sharing/hash_cache.cc | 175 +++++++++++++++++- libretroshare/src/file_sharing/hash_cache.h | 8 +- libretroshare/src/libretroshare.pro | 6 +- 6 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 libretroshare/src/file_sharing/filelist_io.cc create mode 100644 libretroshare/src/file_sharing/filelist_io.h diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index a3a084612..9e7872611 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -7,15 +7,6 @@ #include "retroshare/rsids.h" #include "retroshare/rsfiles.h" -static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; - -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; -static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; -static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; -static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; - #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } class InternalFileHierarchyStorage ; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc new file mode 100644 index 000000000..e8e1207f7 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -0,0 +1,2 @@ +#include "filelist_io.h" + diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h new file mode 100644 index 000000000..d8f5f0d10 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +// This file implements load/save of various fields used for file lists and directory content. +// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. + +static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; + +class FileListIO +{ +public: + template + static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val) + { + if(!checkSectionSize(buff,buff_size,offset,sizeof(T))) + return false; + + if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T))) + return false; + + memcpy(&buff[offset],reinterpret_cast(&val),sizeof(T)) ; + offset += sizeof(T) ; + + return true; + } + +private: + static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) + { + if(offset + S > buff_size) + { + buff = (unsigned char *)realloc(buff,offset + S) ; + buff_size = offset + S ; + + if(!buff) + return false ; + } + return true ; + } + + static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S) + { + buff[offset++] = section_tag ; + if(!write125Size(buff,offset,buff_size,S)) return false ; + + return true; + } +}; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 32401ca4b..c86e6e8cc 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,13 +1,19 @@ #include "util/rsdir.h" #include "rsserver/p3face.h" +#include "pqi/authssl.h" #include "hash_cache.h" +#include "filelist_io.h" #define HASHSTORAGE_DEBUG 1 +static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000; +static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; + HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { mRunning = false ; + load() ; } void HashStorage::data_tick() @@ -28,9 +34,15 @@ void HashStorage::data_tick() RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + mInactivitySleepTime *= 2; + + if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME) + mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + return ; } + mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; job = mFilesToHash.begin()->second ; @@ -145,3 +157,164 @@ void HashStorage::clean() #endif } +void HashStorage::load() +{ + uint64_t file_size ; + + if(!RsDirUtil::checkFile( mFilePath,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file not present." << std::endl; + return ; + } + + // read the binary stream into memory. + // + void *buffer = rs_malloc(file_size) ; + + if(buffer == NULL) + return ; + + FILE *F = fopen( mFilePath.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl; + free(buffer); + return; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl; + free(buffer) ; + fclose(F) ; + return ; + } + fclose(F) ; + + // now decrypt + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + free(buffer) ; + return ; + } + free(buffer) ; + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Loading HashCache from file " << mFilePath << std::endl ; + int n=0 ; +#endif + unsigned char *data = (unsigned char*)decrypted_data ; + uint32_t offset = 0 ; + HashStorageInfo info ; + + while(offset < decrypted_data_size) + if(readHashStorageInfo(data,decrypted_data_size,offset,info)) + { +#ifdef HASHSTORAGE_DEBUG + std::cerr << info << std::endl; + ++n ; +#endif + mFiles[info.filename] = info ; + } + + free(decrypted_data) ; +#ifdef HASHSTORAGE_DEBUG + std::cerr << n << " entries loaded." << std::endl ; +#endif +} + +void HashStorage::save() +{ +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ; +#endif + + unsigned char *data = NULL ; + uint32_t offset = 0 ; + uint32_t total_size = 0; + + for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) + writeHashStorageInfo(it->second,data,total_size,offset) ; + + void *encryptedData = NULL ; + int encDataLen = 0 ; + + if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId())) + { + std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + return; + } + + FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ; + + if(!F) + { + std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl; + goto save_free; + } + if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) + { + std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + fclose(F) ; + goto save_free; + } + + fclose(F) ; + + RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ; +#ifdef FIM_DEBUG + std::cerr << "done." << std::endl ; +#endif + +save_free: + free(encryptedData); +} + +bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_size = 0; + + if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) + return false; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + free(section_data); + return true; +} + +bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_offset = 0 ; + uint32_t section_size = 0; + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; + + free(section_data) ; + + return true; +} + +std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info) +{ + return o << info.hash << " " << info.size << " " << info.filename ; +} + + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 2d10c09ca..cf6b12a9f 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -34,7 +34,7 @@ public: struct HashStorageInfo { - std::string filename ; + std::string filename ; // full path of the file uint64_t size ; uint32_t time_stamp ; // last time the hash was tested/requested uint32_t modf_stamp ; @@ -50,12 +50,17 @@ public: // Functions called by the thread virtual void data_tick() ; + + friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ; private: void clean() ; void save() ; void load() ; + bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; + bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; + // Local configuration and storage uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries @@ -79,5 +84,6 @@ private: RsMutex mHashMtx ; bool mRunning; + uint32_t mInactivitySleepTime = 50*1000 ; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index b7a113fd7..70994b8c7 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -38,12 +38,14 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ file_sharing/hash_cache.h \ - file_sharing/directory_storage.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ file_sharing/directory_updater.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ - file_sharing/directory_storage.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc } From d8dfce4fff4e5a5e14d867eeb4dd85ed61cef509 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 11 Aug 2016 14:07:45 +0200 Subject: [PATCH 21/92] added parent row and friend status to shared dirs --- .../src/file_sharing/directory_storage.cc | 154 ++++++++++++++++-- .../src/file_sharing/directory_storage.h | 32 ++-- libretroshare/src/file_sharing/p3filelists.cc | 109 ++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 8 +- libretroshare/src/retroshare/rstypes.h | 29 ++-- 5 files changed, 267 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b91bbe4cb..5230ec2fd 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -30,6 +30,9 @@ class InternalFileHierarchyStorage virtual ~FileStorageNode() {} virtual uint32_t type() const =0; + + DirectoryStorage::EntryIndex parent_index; + uint32_t row ; }; class FileEntry: public FileStorageNode { @@ -48,23 +51,35 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + DirEntry(const std::string& name) : dir_name(name) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } // local stuff std::string dir_name ; - DirectoryStorage::EntryIndex parent_index; + std::string dir_parent_path ; std::vector subdirs ; std::vector subfiles ; + + time_t most_recent_time; }; // class stuff InternalFileHierarchyStorage() : mRoot(0) { - mNodes.push_back(new DirEntry("",0)) ; + mNodes.push_back(new DirEntry("")) ; + mNodes.back()->row=0; + mNodes.back()->parent_index=-1; + } + + int parentRow(DirectoryStorage::EntryIndex e) + { + if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) + return -1 ; + + return mNodes[mNodes[e]->parent_index]->row; } // high level modification routines @@ -95,7 +110,9 @@ class InternalFileHierarchyStorage for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) { d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(new DirEntry(*it,indx)); + mNodes.push_back(new DirEntry(*it)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; } return true; @@ -127,7 +144,7 @@ class InternalFileHierarchyStorage if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) return nodeAccessError("checkIndex(): Node does not exist") ; - if(mNodes[indx]->type() != type) + if(! mNodes[indx]->type() & type) return nodeAccessError("checkIndex(): Node is of wrong type") ; return true; @@ -172,6 +189,8 @@ class InternalFileHierarchyStorage { d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; } return true; } @@ -208,6 +227,7 @@ class InternalFileHierarchyStorage // are not referenced. friend class DirectoryStorage ; // only class that can use this. + friend class LocalDirectoryStorage ; // only class that can use this. // Low level stuff. Should normally not be used externally. @@ -225,6 +245,16 @@ class InternalFileHierarchyStorage return static_cast(mNodes[indx]) ; } + uint32_t getType(DirectoryStorage::EntryIndex indx) const + { + if(checkIndex(indx,FileStorageNode::TYPE_FILE)) + return FileStorageNode::TYPE_FILE; + else if(checkIndex(indx,FileStorageNode::TYPE_DIR)) + return FileStorageNode::TYPE_DIR; + else + return FileStorageNode::TYPE_UNKNOWN; + } + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) { if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) @@ -368,12 +398,14 @@ uint64_t DirectoryStorage::FileIterator::size() const { const InternalFil std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } +std::string DirectoryStorage::DirIterator::name() const { const InternalFileHierarchyStorage::DirEntry *d = mStorage->getDirEntry(**this) ; return d?(d->dir_name):std::string(); } + /******************************************************************************************************************/ /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirectoryStorage(const std::string& local_file_name) - : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) +DirectoryStorage::DirectoryStorage(const std::string& local_file_name, const RsPeerId &pid) + : mFileName(local_file_name),mPeerId(pid), mDirStorageMtx("Directory storage "+local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); @@ -383,6 +415,25 @@ DirectoryStorage::EntryIndex DirectoryStorage::root() const { return EntryIndex(0) ; } +int DirectoryStorage::parentRow(EntryIndex e) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + return mFileHierarchy->parentRow(e) ; +} + +uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + switch(mFileHierarchy->getType(indx)) + { + case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR: return DIR_TYPE_DIR ; + case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE: return DIR_TYPE_FILE ; + default: + return DIR_TYPE_UNKNOWN; + } +} bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { @@ -440,6 +491,15 @@ void DirectoryStorage::print() std::cerr << "LocalDirectoryStorage:" << std::endl; mFileHierarchy->print(); } + +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; +#warning todo + + return true; +} + /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -534,10 +594,80 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const return cit->second.filename; } -bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { - RS_STACK_MUTEX(mDirStorageMtx) ; - NOT_IMPLEMENTED() ; - return false; -} + d.children.clear() ; + time_t now = time(NULL) ; + const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + + if (dir_entry != NULL) /* has children --- fill */ + { +#ifdef FI_DEBUG + std::cerr << "FileIndex::extractData() ref=dir" << std::endl; +#endif + /* extract all the entries */ + + for(DirectoryStorage::DirIterator it(this,indx);it;++it) + { + DirStub stub; + stub.type = DIR_TYPE_DIR; + stub.name = it.name(); + stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with + + d.children.push_back(stub); + } + + for(DirectoryStorage::FileIterator it(this,indx);it;++it) + { + DirStub stub; + stub.type = DIR_TYPE_FILE; + stub.name = it.name(); + stub.ref = (void*)(intptr_t)*it; + + d.children.push_back(stub); + } + + if(dir_entry->parent_index == 0) + d.type = DIR_TYPE_PERSON ; + else + d.type = DIR_TYPE_DIR; + + d.hash.clear() ; + d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); + d.min_age = now - dir_entry->most_recent_time ; + d.name = dir_entry->dir_name; + d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; + d.parent = (void*)(intptr_t)dir_entry->parent_index ; + } + else + { + const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; +#ifdef FI_DEBUG + std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; +#endif + d.type = DIR_TYPE_FILE; + d.count = file_entry->file_size; + d.min_age = now - file_entry->file_modtime ; + d.name = file_entry->file_name; + d.hash = file_entry->file_hash; + d.age = now - file_entry->file_modtime; + d.parent = (void*)(intptr_t)file_entry->parent_index ; + + const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); + + if(parent_dir_entry != NULL) + d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; + else + d.path = "" ; + } + +#ifdef FI_DEBUG + std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; +#endif + d.flags.clear() ; + + /* find parent pointer, and row */ + + return true; +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9e7872611..0bdbb9dd6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -14,10 +14,10 @@ class InternalFileHierarchyStorage ; class DirectoryStorage { public: - DirectoryStorage(const std::string& local_file_name) ; + DirectoryStorage(const std::string& local_file_name,const RsPeerId& pid) ; virtual ~DirectoryStorage() {} - typedef uint32_t EntryIndex ; + typedef int32_t EntryIndex ; static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; @@ -27,6 +27,8 @@ class DirectoryStorage virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } void getFileDetails(EntryIndex i) ; + uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. + virtual bool extractData(const EntryIndex& indx,DirDetails& d) =0; // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. // It gives access to sub-files and sub-directories below. @@ -44,7 +46,7 @@ class DirectoryStorage // info about the directory that is pointed by the iterator - const std::string& name() const ; + std::string name() const ; private: EntryIndex mParentIndex ; // index of the parent dir. uint32_t mDirTabIndex ; // index in the vector of subdirs. @@ -83,6 +85,8 @@ class DirectoryStorage }; EntryIndex root() const ; // returns the index of the root directory entry. + const RsPeerId& peerId() const { return mPeerId ; } + int parentRow(EntryIndex e) const ; bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; @@ -103,28 +107,31 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. - InternalFileHierarchyStorage *mFileHierarchy ; std::string mFileName; + RsPeerId mPeerId; + protected: - RsMutex mDirStorageMtx ; + mutable RsMutex mDirStorageMtx ; + + InternalFileHierarchyStorage *mFileHierarchy ; }; class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname),mPeerId(pid) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} - const RsPeerId& peerId() const { return mPeerId ; } - -private: - RsPeerId mPeerId; + virtual bool extractData(const EntryIndex& indx,DirDetails& d) + { + NOT_IMPLEMENTED(); + } }; class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} + LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(fname,own_id) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; @@ -140,6 +147,9 @@ public: * \return false if the file does not exist, or is a directory,... */ bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; + + virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; + private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 299d90029..1c6f50802 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -25,9 +25,11 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) { // loads existing indexes for friends. Some might be already present here. // - mRemoteDirectories.clear() ; // we should load them! + mDirectories.clear() ; // we should load them! + + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ; + mDirectories.push_back(mLocalSharedDirs) ; - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin") ; mHashCache = new HashStorage("hash_cache.bin") ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; @@ -58,10 +60,10 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(uint32_t i=0;ipeerId()) == friend_set.end()) + for(uint32_t i=1;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - delete mRemoteDirectories[i]; - mRemoteDirectories[i] = NULL ; + delete mDirectories[i]; + mDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - friend_set.erase(mRemoteDirectories[i]->peerId()); + friend_set.erase(mDirectories[i]->peerId()); - mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); + mFriendIndexMap.erase(mDirectories[i]->peerId()); mFriendIndexTab[i].clear(); } @@ -195,12 +197,12 @@ void p3FileDatabase::cleanup() P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; uint32_t i; - for(i=0;iroot(),i,p); + + DirStub stub; + stub.type = DIR_TYPE_PERSON; + stub.name = mDirectories[i]->peerId().toStdString(); + stub.ref = p; + d.children.push_back(stub); + } + d.count = mDirectories.size(); + + d.parent = NULL; + d.prow = -1; + d.ref = NULL; + d.type = DIR_TYPE_ROOT; + d.name = "root"; + d.hash.clear() ; + d.path = "root"; + d.age = 0; + d.flags.clear() ; + d.min_age = 0 ; + + return true ; + } + uint32_t fi; EntryIndex e ; - convertPointerToEntryIndex(ref,e,fi) ; -#warning code needed here + convertPointerToEntryIndex(ref,e,fi); - return 0; + // Case where the index is the top of a single person. Can be us, or a friend. + + bool res = mDirectories[fi]->extractData(e,d) ; + + // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert + // a pointer into an int. + + convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; + + for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) + convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); + + d.prow = mDirectories[fi]->parentRow(e) ; + d.id = mDirectories[fi]->peerId(); } + int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); @@ -317,9 +367,20 @@ int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &detai } uint32_t p3FileDatabase::getType(void *ref) const { - NOT_IMPLEMENTED(); - return 0; + EntryIndex e ; + uint32_t fi; + + if(ref == NULL) + return DIR_TYPE_ROOT ; + + convertPointerToEntryIndex(ref,e,fi); + + if(e == 0) + return DIR_TYPE_PERSON ; + + return mDirectories[fi]->getEntryType(e) ; } + void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { NOT_IMPLEMENTED(); diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index d7c8e2250..cd090af61 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -147,16 +147,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // The remote one is the reference for the PeerId index below: // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) - std::vector mRemoteDirectories ; + std::vector mDirectories ; // mDirectories[0]=mLocalSharedDirs LocalDirectoryStorage *mLocalSharedDirs ; - RemoteDirectoryUpdater *mRemoteDirWatcher ; + RemoteDirectoryUpdater *mRemoteDirWatcher ; // not used yet. LocalDirectoryUpdater *mLocalDirWatcher ; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc - static bool convertEntryIndexToPointer(EntryIndex& e,uint32_t friend_index,void *& p); - static bool convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) ; + static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); + static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; uint32_t getFriendIndex(const RsPeerId& pid); const RsPeerId& getFriendFromIndex(uint32_t indx) const; diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 37975d605..d30b156eb 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -138,6 +138,7 @@ public: /********************** For FileCache Interface *****************/ +#define DIR_TYPE_UNKNOWN 0x00 #define DIR_TYPE_ROOT 0x01 #define DIR_TYPE_PERSON 0x02 #define DIR_TYPE_DIR 0x04 @@ -159,7 +160,7 @@ const FileStorageFlags DIR_FLAGS_BROWSABLE_OTHERS ( 0x0100 ); // one should const FileStorageFlags DIR_FLAGS_NETWORK_WIDE_GROUPS ( 0x0200 ); const FileStorageFlags DIR_FLAGS_BROWSABLE_GROUPS ( 0x0400 ); const FileStorageFlags DIR_FLAGS_PERMISSIONS_MASK ( DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_OTHERS - | DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS ); + | DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS ); const FileStorageFlags DIR_FLAGS_LOCAL ( 0x1000 ); const FileStorageFlags DIR_FLAGS_REMOTE ( 0x2000 ); @@ -227,23 +228,23 @@ class DirStub class DirDetails { - public: - void *parent; - int prow; /* parent row */ +public: + void *parent; + int prow; /* parent row */ - void *ref; - uint8_t type; + void *ref; + uint8_t type; RsPeerId id; - std::string name; + std::string name; RsFileHash hash; - std::string path; - uint64_t count; - uint32_t age; - FileStorageFlags flags; - uint32_t min_age ; // minimum age of files in this subtree + std::string path; + uint64_t count; + uint32_t age; + FileStorageFlags flags; + uint32_t min_age ; // minimum age of files in this subtree - std::list children; - std::list parent_groups; // parent groups for the shared directory + std::list children; + std::list parent_groups; // parent groups for the shared directory }; class FileDetail From 733ad438fbfc9cc8a04e75174ab0c190e8532a40 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 12 Aug 2016 15:20:23 +0200 Subject: [PATCH 22/92] added IO for hash cache --- libretroshare/src/file_sharing/filelist_io.cc | 99 +++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 45 ++++++++- libretroshare/src/file_sharing/hash_cache.cc | 5 +- 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index e8e1207f7..3cd153ae8 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,2 +1,101 @@ #include "filelist_io.h" +bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) +{ + if(!checkSectionSize(buff,buff_size,offset,size)) + return false; + + if(!writeSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + return false; + + memcpy(&buff[offset],val,size) ; + offset += size ; + + return true; +} + +bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) +{ + if(!readSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + return false; + + val = (unsigned char *)rs_malloc(size) ; + + if(!val) + return false; + + memcpy(val,&buff[offset],size); + offset += size ; + + return true ; +} + +bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t S) +{ + if(S < 192) + { + if(offset+1 > data_size) + return false; + + data[offset++] = (uint8_t)S ; + return true; + } + else if(S < 8384) + { + if(offset+2 > data_size) + return false; + + data[offset++] = (uint8_t)((S >> 8) + 192) ; + data[offset++] = (uint8_t)((S & 255) - 192) ; + + return true; + } + else + { + if(offset+5 > data_size) + return false; + + data[offset++] = 0xff ; + data[offset++] = (uint8_t)((S >> 24) & 255) ; + data[offset++] = (uint8_t)((S >> 16) & 255) ; + data[offset++] = (uint8_t)((S >> 8) & 255) ; + data[offset++] = (uint8_t)((S ) & 255) ; + + return true ; + } +} + +bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t& S) +{ + if(offset + 1 >= data_size) return false; + + uint8_t b1 = data[offset++] ; + + if(b1 < 192) + { + S = b1; + return true ; + } + if(offset + 1 >= data_size) return false; + + uint8_t b2 = data[offset++] ; + + if(b1 < 224) + { + S = ((b1-192) << 8) + b2 + 192 ; + return true; + } + + if(b1 != 0xff) + return false; + + if(offset + 3 >= data_size) return false; + + uint8_t b3 = data[offset++]; + uint8_t b4 = data[offset++]; + uint8_t b5 = data[offset++]; + + S = (b2 << 24) | (b3 << 16) | (b4 << 8) | b5 ; + return true; +} + diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index d8f5f0d10..9c5818025 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -1,6 +1,10 @@ #pragma once +#include #include +#include + +#include "util/rsmemory.h" // This file implements load/save of various fields used for file lists and directory content. // WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. @@ -15,6 +19,7 @@ static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; class FileListIO { @@ -34,7 +39,30 @@ public: return true; } + template + static bool readField(const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,T& val) + { + uint32_t section_size ; + + if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size)) + return false; + + if(section_size != sizeof(T)) + return false ; + + memcpy(reinterpret_cast(&val),&buff[offset],sizeof(T)) ; + offset += sizeof(T) ; + + return true; + } + + static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; + static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ; + private: + static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; + static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; + static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) { if(offset + S > buff_size) @@ -48,11 +76,24 @@ private: return true ; } - static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S) + static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,uint32_t S) { buff[offset++] = section_tag ; - if(!write125Size(buff,offset,buff_size,S)) return false ; + if(!write125Size(buff,buff_size,offset,S)) return false ; return true; } + + static bool readSectionHeader(const unsigned char *& buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,uint32_t& S) + { + if(offset + 1 > buff_size) + return false ; + + uint8_t section_tag = buff[offset++] ; + + if(section_tag != check_section_tag) + return false; + + return read125Size(buff,buff_size,offset,S) ; + } }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index c86e6e8cc..743e959dd 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -237,7 +237,7 @@ void HashStorage::save() uint32_t total_size = 0; for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) - writeHashStorageInfo(it->second,data,total_size,offset) ; + writeHashStorageInfo(data,total_size,offset,it->second) ; void *encryptedData = NULL ; int encDataLen = 0 ; @@ -277,6 +277,7 @@ bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_s { unsigned char *section_data = NULL ; uint32_t section_size = 0; + uint32_t section_offset = 0; if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) return false; @@ -316,5 +317,3 @@ std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& inf { return o << info.hash << " " << info.size << " " << info.filename ; } - - From df14b41ba6663dc472cfa0848fa481743c66d032 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 12 Aug 2016 19:30:19 +0200 Subject: [PATCH 23/92] fixed sleeping when hashing strategy --- libretroshare/src/file_sharing/hash_cache.cc | 44 ++++++++++++++----- libretroshare/src/file_sharing/p3filelists.cc | 11 +++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 743e959dd..60eec0cd9 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -23,22 +23,41 @@ void HashStorage::data_tick() uint64_t size ; { - RS_STACK_MUTEX(mHashMtx) ; + bool empty ; + uint32_t st ; - if(mFilesToHash.empty()) { - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; + RS_STACK_MUTEX(mHashMtx) ; - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + empty = mFilesToHash.empty(); + st = mInactivitySleepTime ; + } - usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. - mInactivitySleepTime *= 2; + // sleep off mutex! + if(empty) + { + std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl; - if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME) - mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + + if(st > MAX_INACTIVITY_SLEEP_TIME) + { + RS_STACK_MUTEX(mHashMtx) ; + + mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + } + else + { + RS_STACK_MUTEX(mHashMtx) ; + mInactivitySleepTime = 2*st ; + } return ; } @@ -279,6 +298,9 @@ bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_s uint32_t section_size = 0; uint32_t section_offset = 0; + // This way, the entire section is either read or skipped. That avoids the risk of being stuck somewhere in the middle + // of a section because of some unknown field, etc. + if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) return false; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 1c6f50802..0db5ad01a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -102,6 +102,15 @@ int p3FileDatabase::tick() // - cleanup(); + static time_t last_print_time = 0; + time_t now = time(NULL) ; + + if(last_print_time + 60 < now) + { + mLocalSharedDirs->print(); + last_print_time = now ; + } + return 0; } @@ -353,6 +362,8 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.prow = mDirectories[fi]->parentRow(e) ; d.id = mDirectories[fi]->peerId(); + + return true; } int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const From 2668b6da1cd90e37cb2a8f2fb779afdecf00de1e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 13 Aug 2016 10:15:02 +0200 Subject: [PATCH 24/92] added p3fileDatabase as a service in rsinit --- .../src/file_sharing/directory_storage.cc | 5 ++++- libretroshare/src/file_sharing/directory_storage.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 6 +++++- libretroshare/src/ft/ftserver.cc | 10 +++++----- libretroshare/src/ft/ftserver.h | 4 +++- libretroshare/src/rsserver/rsinit.cc | 14 +++++++++----- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 5230ec2fd..e112d6651 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -100,7 +100,10 @@ class InternalFileHierarchyStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) - removeDirectory(d.subdirs[i]) ; + { + if( !removeDirectory(d.subdirs[i])) + i++ ; + } else { should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0bdbb9dd6..a96790684 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -125,6 +125,7 @@ public: virtual bool extractData(const EntryIndex& indx,DirDetails& d) { NOT_IMPLEMENTED(); + return false; } }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0db5ad01a..2d27c9788 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -105,7 +105,7 @@ int p3FileDatabase::tick() static time_t last_print_time = 0; time_t now = time(NULL) ; - if(last_print_time + 60 < now) + if(last_print_time + 20 < now) { mLocalSharedDirs->print(); last_print_time = now ; @@ -212,6 +212,10 @@ void p3FileDatabase::cleanup() mDirectories.push_back(NULL) ; mDirectories[i] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + + if(mFriendIndexTab.size() <= i) + mFriendIndexTab.resize(i+1) ; + mFriendIndexTab[i] = *it ; mFriendIndexMap[*it] = i; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 44b361384..8a0424a5c 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -148,19 +148,19 @@ void ftServer::SetupFtServer() mFtController->setPartialsDirectory(tmppath); mFtController->setDownloadDirectory(tmppath); - - /* Make Cache Source/Store */ - mFileDatabase = new p3FileDatabase(mServiceCtrl) ; - /* complete search setup */ mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA); - mFtSearch->addSearchMode(mFileDatabase, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType); return; } +void ftServer::connectToFileDatabase(p3FileDatabase *fdb) +{ + mFileDatabase = fdb ; + mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); +} void ftServer::connectToTurtleRouter(p3turtle *fts) { mTurtleRouter = fts ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 1b66db710..4e5ae3a61 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -53,6 +53,7 @@ #include "pqi/p3cfgmgr.h" class p3ConnectMgr; +class p3FileDatabase; class CacheStrapper; class CacheTransfer; @@ -94,7 +95,8 @@ public: /* Final Setup (once everything is assigned) */ void SetupFtServer() ; - virtual void connectToTurtleRouter(p3turtle *p) ; + virtual void connectToTurtleRouter(p3turtle *p) ; + virtual void connectToFileDatabase(p3FileDatabase *b); // Implements RsTurtleClientService // diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 40c3417ff..0f484fae3 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -56,7 +56,8 @@ #include #include -#include +#include "gxstunnel/p3gxstunnel.h" +#include "file_sharing/p3filelists.h" #define ENABLE_GROUTER @@ -1481,22 +1482,25 @@ int RsServer::StartupRetroShare() pqih->addService(gr,true) ; #endif - p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ; + p3FileDatabase *fdb = new p3FileDatabase(serviceCtrl) ; + p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ; rsTurtle = tr ; pqih -> addService(tr,true); - pqih -> addService(ftserver,true); + pqih -> addService(fdb,true); + pqih -> addService(ftserver,true); mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ; mGxsTunnels->connectToTurtleRouter(tr) ; rsGxsTunnel = mGxsTunnels; - + rsDisc = mDisc; rsMsgs = new p3Msgs(msgSrv, chatSrv); // connect components to turtle router. ftserver->connectToTurtleRouter(tr) ; - chatSrv->connectToGxsTunnelService(mGxsTunnels) ; + ftserver->connectToFileDatabase(fdb) ; + chatSrv->connectToGxsTunnelService(mGxsTunnels) ; gr->connectToTurtleRouter(tr) ; #ifdef ENABLE_GROUTER msgSrv->connectToGlobalRouter(gr) ; From ca8f95c7f74b16c208c25f2f1a25037987c69978 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 16 Aug 2016 13:46:55 +0200 Subject: [PATCH 25/92] fixed a few bugs in internal directory storage structure --- .../src/file_sharing/directory_storage.cc | 30 +++++++++++++++- .../src/file_sharing/directory_updater.cc | 6 ---- libretroshare/src/file_sharing/hash_cache.cc | 10 ++++-- libretroshare/src/file_sharing/hash_cache.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 35 ++++++++++++------- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e112d6651..7b913b4c3 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -101,17 +101,23 @@ class InternalFileHierarchyStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) { + std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + if( !removeDirectory(d.subdirs[i])) i++ ; } else { + std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; ++i; } for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) { + std::cerr << "[directory storage] adding new subdirectory " << *it << " at index " << mNodes.size() << std::endl; + d.subdirs.push_back(mNodes.size()) ; mNodes.push_back(new DirEntry(*it)); mNodes.back()->row = mNodes.size()-1; @@ -170,9 +176,13 @@ class InternalFileHierarchyStorage if(it == subfiles.end()) // file does not exist anymore => delete { + std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; + + delete mNodes[d.subfiles[i]] ; + mNodes[d.subfiles[i]] = NULL ; + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; d.subfiles.pop_back(); - continue; } @@ -190,6 +200,8 @@ class InternalFileHierarchyStorage for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { + std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; + d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); mNodes.back()->row = mNodes.size()-1; @@ -200,7 +212,12 @@ class InternalFileHierarchyStorage bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; + } + + std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; static_cast(mNodes[file_index])->file_hash = hash ; return true; @@ -208,10 +225,15 @@ class InternalFileHierarchyStorage bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; + } FileEntry& fe(*static_cast(mNodes[file_index])) ; + std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; + fe.file_hash = hash; fe.file_size = size; fe.file_modtime = modf_time; @@ -321,6 +343,12 @@ private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const { std::string indent(2*depth,' '); + + if(mNodes[node] == NULL) + { + std::cerr << "EMPTY NODE !!" << std::endl; + return ; + } DirEntry& d(*static_cast(mNodes[node])); std::cerr << indent << "dir:" << d.dir_name << std::endl; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 149937901..e0f0438a0 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -83,12 +83,6 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p librs::util::FolderIterator dirIt(cumulated_path); - if(!dirIt.isValid()) - { - mSharedDirectories->removeDirectory(indx) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. - return ; - } - // collect subdirs and subfiles std::map subfiles ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 60eec0cd9..1e7cf0fe7 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -12,7 +12,9 @@ static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { + mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; + load() ; } @@ -63,7 +65,11 @@ void HashStorage::data_tick() } mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; - job = mFilesToHash.begin()->second ; + { + RS_STACK_MUTEX(mHashMtx) ; + job = mFilesToHash.begin()->second ; + mFilesToHash.erase(mFilesToHash.begin()) ; + } std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); @@ -77,8 +83,6 @@ void HashStorage::data_tick() else std::cerr << "done."<< std::endl; - mFilesToHash.erase(mFilesToHash.begin()) ; - // store the result HashStorageInfo& info(mFiles[job.full_path]); diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index cf6b12a9f..6b2dbf8f3 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -84,6 +84,6 @@ private: RsMutex mHashMtx ; bool mRunning; - uint32_t mInactivitySleepTime = 50*1000 ; + uint32_t mInactivitySleepTime ; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 2d27c9788..09cf31e53 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -203,21 +203,16 @@ void p3FileDatabase::cleanup() // for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) { - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + // Check if a remote directory exists for that friend, possibly creating the index. - uint32_t i; - for(i=0;i friend_index && mDirectories[friend_index] != NULL) + continue ; - mDirectories[i] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; - if(mFriendIndexTab.size() <= i) - mFriendIndexTab.resize(i+1) ; - - mFriendIndexTab[i] = *it ; - mFriendIndexMap[*it] = i; + mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -249,19 +244,33 @@ uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) if(!found) { - std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than 1024 friends??" << std::endl; - return 1024 ; + found = mFriendIndexTab.size(); + mFriendIndexTab.push_back(pid); + } + + if(mFriendIndexTab.size() >= (1 << NB_FRIEND_INDEX_BITS) ) + { + std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<second) + mDirectories.resize(it->second+1,NULL) ; + return it->second; + } } const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const From ac242ce58bde9bfe22eb49b323968ff771a5fd5d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 16 Aug 2016 23:44:48 +0200 Subject: [PATCH 26/92] fixed display of own files --- .../src/file_sharing/directory_storage.cc | 160 ++++++++++-------- .../src/file_sharing/directory_storage.h | 8 +- libretroshare/src/file_sharing/p3filelists.cc | 57 +++++-- libretroshare/src/ft/ftserver.cc | 2 +- libretroshare/src/retroshare/rstypes.h | 5 +- libretroshare/src/rsserver/rstypes.cc | 21 ++- 6 files changed, 155 insertions(+), 98 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7b913b4c3..77b9e70c9 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -3,6 +3,8 @@ #include "util/rsstring.h" #include "directory_storage.h" +#define DEBUG_DIRECTORY_STORAGE 1 + /******************************************************************************************************************/ /* Internal File Hierarchy Storage */ /******************************************************************************************************************/ @@ -71,7 +73,7 @@ class InternalFileHierarchyStorage { mNodes.push_back(new DirEntry("")) ; mNodes.back()->row=0; - mNodes.back()->parent_index=-1; + mNodes.back()->parent_index=0; } int parentRow(DirectoryStorage::EntryIndex e) @@ -153,7 +155,7 @@ class InternalFileHierarchyStorage if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) return nodeAccessError("checkIndex(): Node does not exist") ; - if(! mNodes[indx]->type() & type) + if(! (mNodes[indx]->type() & type)) return nodeAccessError("checkIndex(): Node is of wrong type") ; return true; @@ -351,7 +353,10 @@ private: } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << std::endl; + std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ; + for(int i=0;isubdirs.size() + dir_entry->subfiles.size(); + d.min_age = now - dir_entry->most_recent_time ; + d.name = dir_entry->dir_name; + d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; + d.parent = (void*)(intptr_t)dir_entry->parent_index ; + + if(indx == 0) + { + d.type = DIR_TYPE_PERSON ; + d.name = mPeerId.toStdString(); + } + } + else + { + const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; +#endif + d.type = DIR_TYPE_FILE; + d.count = file_entry->file_size; + d.min_age = now - file_entry->file_modtime ; + d.name = file_entry->file_name; + d.hash = file_entry->file_hash; + d.age = now - file_entry->file_modtime; + d.parent = (void*)(intptr_t)file_entry->parent_index ; + + const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); + + if(parent_dir_entry != NULL) + d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; + else + d.path = "" ; + } + + d.flags.clear() ; + + return true; +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -627,78 +708,19 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { - d.children.clear() ; - time_t now = time(NULL) ; + bool res = DirectoryStorage::extractData(indx,d) ; - const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + if(!res) + return false; - if (dir_entry != NULL) /* has children --- fill */ - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif - /* extract all the entries */ + // here we should update the file sharing flags - for(DirectoryStorage::DirIterator it(this,indx);it;++it) - { - DirStub stub; - stub.type = DIR_TYPE_DIR; - stub.name = it.name(); - stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with - - d.children.push_back(stub); - } - - for(DirectoryStorage::FileIterator it(this,indx);it;++it) - { - DirStub stub; - stub.type = DIR_TYPE_FILE; - stub.name = it.name(); - stub.ref = (void*)(intptr_t)*it; - - d.children.push_back(stub); - } - - if(dir_entry->parent_index == 0) - d.type = DIR_TYPE_PERSON ; - else - d.type = DIR_TYPE_DIR; - - d.hash.clear() ; - d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); - d.min_age = now - dir_entry->most_recent_time ; - d.name = dir_entry->dir_name; - d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; - d.parent = (void*)(intptr_t)dir_entry->parent_index ; - } - else - { - const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif - d.type = DIR_TYPE_FILE; - d.count = file_entry->file_size; - d.min_age = now - file_entry->file_modtime ; - d.name = file_entry->file_name; - d.hash = file_entry->file_hash; - d.age = now - file_entry->file_modtime; - d.parent = (void*)(intptr_t)file_entry->parent_index ; - - const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); - - if(parent_dir_entry != NULL) - d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; - else - d.path = "" ; - } - -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; -#endif d.flags.clear() ; /* find parent pointer, and row */ + std::cerr << "LocalDirectoryStorage::extractData(): Returning this:" << std::endl; + std::cerr << d << std::endl; + return true; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index a96790684..9f2b63be9 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -28,7 +28,7 @@ class DirectoryStorage void getFileDetails(EntryIndex i) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. - virtual bool extractData(const EntryIndex& indx,DirDetails& d) =0; + virtual bool extractData(const EntryIndex& indx,DirDetails& d); // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. // It gives access to sub-files and sub-directories below. @@ -121,12 +121,6 @@ class RemoteDirectoryStorage: public DirectoryStorage public: RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} - - virtual bool extractData(const EntryIndex& indx,DirDetails& d) - { - NOT_IMPLEMENTED(); - return false; - } }; class LocalDirectoryStorage: public DirectoryStorage diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 09cf31e53..20e2c5c6c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -137,7 +137,6 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { - NOT_IMPLEMENTED(); } void p3FileDatabase::tickRecv() @@ -146,7 +145,6 @@ void p3FileDatabase::tickRecv() void p3FileDatabase::tickSend() { // go through the list of out requests and send them to the corresponding friends, if they are online. - NOT_IMPLEMENTED(); } bool p3FileDatabase::loadList(std::list& items) @@ -295,6 +293,13 @@ bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, ui e = EntryIndex( *reinterpret_cast(&p) & ENTRY_INDEX_BIT_MASK ) ; friend_index = (*reinterpret_cast(&p)) >> NB_ENTRY_INDEX_BITS ; + if(friend_index == 0) + { + std::cerr << "(EE) Cannot find friend index in pointer. Encoded value is zero!" << std::endl; + return false; + } + friend_index--; + return true; } bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi, void *& p) @@ -309,13 +314,13 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi uint32_t fe = (uint32_t)e ; - if(fi >= (1<= (1<< NB_ENTRY_INDEX_BITS)) + if(fi+1 >= (1<= (1<< NB_ENTRY_INDEX_BITS)) { std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; return false ; } - p = reinterpret_cast( (fi << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; + p = reinterpret_cast( ( (1+fi) << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; return true; } @@ -325,10 +330,36 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) + // or at the top of our own list of shred directories, depending on the flags. if (ref == NULL) { - for(uint32_t i=0;igetOwnId().toStdString(); + stub.ref = p; + d.children.push_back(stub); + } + else for(uint32_t i=1;iroot(),i,p); @@ -339,24 +370,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags stub.ref = p; d.children.push_back(stub); } - d.count = mDirectories.size(); - d.parent = NULL; - d.prow = -1; - d.ref = NULL; - d.type = DIR_TYPE_ROOT; - d.name = "root"; - d.hash.clear() ; - d.path = "root"; - d.age = 0; - d.flags.clear() ; - d.min_age = 0 ; + d.count = d.children.size(); return true ; } uint32_t fi; - EntryIndex e ; + DirectoryStorage::EntryIndex e ; convertPointerToEntryIndex(ref,e,fi); @@ -505,7 +526,7 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) { DirDetails cdetails ; - RequestDirDetails ((void*)*rit,cdetails,FileSearchFlags(0u)); + RequestDirDetails ((void*)(intptr_t)*rit,cdetails,FileSearchFlags(0u)); #ifdef P3FILELISTS_DEBUG std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 8a0424a5c..1268ea831 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -240,7 +240,7 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) { - return mFtController->alreadyHaveFile(hash, info); + return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info); } bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index d30b156eb..9ab56c70b 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -215,8 +215,7 @@ class FileInfo std::list parent_groups ; }; -std::ostream &operator<<(std::ostream &out, const FileInfo &info); - +std::ostream &operator<<(std::ostream &out, const FileInfo& info); class DirStub { @@ -247,6 +246,8 @@ public: std::list parent_groups; // parent groups for the shared directory }; +std::ostream &operator<<(std::ostream &out, const DirDetails& details); + class FileDetail { public: diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index c5c3c8b02..128f08005 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -44,7 +44,26 @@ * **********************************************************************/ - +std::ostream &operator<<(std::ostream &out, const DirDetails& d) +{ + std::cerr << "====DIR DETAILS====" << std::endl; + std::cerr << " parent pointer: " << (intptr_t)d.parent << std::endl; + std::cerr << " current pointer: " << (intptr_t)d.ref << std::endl; + std::cerr << " parent row : " << d.prow << std::endl; + std::cerr << " type : " << (int)d.type << std::endl; + std::cerr << " PeerId : " << d.id << std::endl; + std::cerr << " Name : " << d.name << std::endl; + std::cerr << " Hash : " << d.hash << std::endl; + std::cerr << " Path : " << d.path << std::endl; + std::cerr << " Count : " << d.count << std::endl; + std::cerr << " Age : " << d.age << std::endl; + std::cerr << " Min age : " << d.min_age << std::endl; + std::cerr << " Flags : " << d.flags << std::endl; + std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; + std::cerr << " Children : " ; for(std::list::const_iterator it(d.children.begin());it!=d.children.end();++it) std::cerr << (intptr_t)(*it).ref << " "; std::cerr << std::endl; + std::cerr << "===================" << std::endl; + return out; +} std::ostream &operator<<(std::ostream &out, const FileInfo &info) { out << "FileInfo: path: " << info.path; From 78b87441838a4a16de9082136a0983156f020a24 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 17 Aug 2016 14:48:54 +0200 Subject: [PATCH 27/92] fixed display of remote directories --- .../src/file_sharing/directory_storage.cc | 2 +- .../src/file_sharing/directory_updater.cc | 15 +- libretroshare/src/file_sharing/p3filelists.cc | 133 +++++++++++------- libretroshare/src/file_sharing/p3filelists.h | 6 +- libretroshare/src/rsserver/rstypes.cc | 4 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 +- 6 files changed, 100 insertions(+), 62 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 77b9e70c9..d0bd333b6 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -719,7 +719,7 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) /* find parent pointer, and row */ - std::cerr << "LocalDirectoryStorage::extractData(): Returning this:" << std::endl; + std::cerr << "LocalDirectoryStorage::extractData(): indx=" << indx << " Returning this:" << std::endl; std::cerr << d << std::endl; return true; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index e0f0438a0..7f427ff7d 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -63,12 +63,12 @@ void LocalDirectoryUpdater::sweepSharedDirectories() // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; - - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it) { - std::cerr << " recursing into " << real_dir_it->filename << std::endl; - recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; + std::cerr << " recursing into " << stored_dir_it.name() << std::endl; + + recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two + // are not necessarily in the same order. } RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } @@ -128,7 +128,10 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) - recursUpdateSharedDir(cumulated_path + "/" + *real_dir_it, *stored_dir_it) ; + { + std::cerr << " recursing into " << stored_dir_it.name() << std::endl; + recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; + } } void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 20e2c5c6c..64bc38924 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -107,25 +107,39 @@ int p3FileDatabase::tick() if(last_print_time + 20 < now) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->print(); last_print_time = now ; + +#warning this should be removed, but it's necessary atm for updating the GUI + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } + if(mUpdateFlags) + { + IndicateConfigChanged(); + + if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED) + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + + if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED) + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + } return 0; } void p3FileDatabase::startThreads() { - std::cerr << "Starting hash cache thread..." ; - mHashCache->start(); - std::cerr << "Done." << std::endl; - + RS_STACK_MUTEX(mFLSMtx) ; std::cerr << "Starting directory watcher thread..." ; mLocalDirWatcher->start(); std::cerr << "Done." << std::endl; } void p3FileDatabase::stopThreads() { + RS_STACK_MUTEX(mFLSMtx) ; std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; mHashCache->fullstop(); std::cerr << "Done." << std::endl; @@ -138,7 +152,6 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } - void p3FileDatabase::tickRecv() { } @@ -165,58 +178,55 @@ bool p3FileDatabase::saveList(bool &cleanup, std::list&) void p3FileDatabase::cleanup() { - RS_STACK_MUTEX(mFLSMtx) ; - - // look through the list of friend directories. Remove those who are not our friends anymore. - // - P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; - - std::set friend_set ; { - std::list friend_lst ; + RS_STACK_MUTEX(mFLSMtx) ; - rsPeers->getFriendList(friend_lst); + // look through the list of friend directories. Remove those who are not our friends anymore. + // + std::set friend_set ; + { + std::list friend_lst ; - for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) - friend_set.insert(*it) ; - } + rsPeers->getFriendList(friend_lst); - for(uint32_t i=1;ipeerId()) == friend_set.end()) - { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - - delete mDirectories[i]; - mDirectories[i] = NULL ; - - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - - friend_set.erase(mDirectories[i]->peerId()); - - mFriendIndexMap.erase(mDirectories[i]->peerId()); - mFriendIndexTab[i].clear(); + for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) + friend_set.insert(*it) ; } - // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. - // - for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) - { - // Check if a remote directory exists for that friend, possibly creating the index. + for(uint32_t i=1;ipeerId()) == friend_set.end()) + { + P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - uint32_t friend_index = getFriendIndex(*it) ; + delete mDirectories[i]; + mDirectories[i] = NULL ; - if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) - continue ; + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + friend_set.erase(mDirectories[i]->peerId()); - mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mFriendIndexMap.erase(mDirectories[i]->peerId()); + mFriendIndexTab[i].clear(); + } - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. + // + for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) + { + // Check if a remote directory exists for that friend, possibly creating the index. + + uint32_t friend_index = locked_getFriendIndex(*it) ; + + if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) + continue ; + + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + + mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } } - - if(mUpdateFlags) - IndicateConfigChanged(); } std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const @@ -225,7 +235,7 @@ std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const return "dirlist_"+pid.toStdString()+".txt" ; } -uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) +uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { std::map::const_iterator it = mFriendIndexMap.find(pid) ; @@ -271,7 +281,7 @@ uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) } } -const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const +const RsPeerId& p3FileDatabase::locked_getFriendFromIndex(uint32_t indx) const { static const RsPeerId null_id ; @@ -329,6 +339,8 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { + RS_STACK_MUTEX(mFLSMtx) ; + // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) // or at the top of our own list of shred directories, depending on the flags. @@ -389,14 +401,26 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // a pointer into an int. convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; - convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); - d.prow = mDirectories[fi]->parentRow(e) ; + if(e == 0) + { + d.prow = fi-1 ; + d.parent = NULL ; + } + else + { + d.prow = mDirectories[fi]->parentRow(e) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; + } + d.id = mDirectories[fi]->peerId(); + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; + return true; } @@ -412,6 +436,8 @@ int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &detai } uint32_t p3FileDatabase::getType(void *ref) const { + RS_STACK_MUTEX(mFLSMtx) ; + EntryIndex e ; uint32_t fi; @@ -469,6 +495,8 @@ void p3FileDatabase::setRememberHashCache(bool) bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchHash(hash,firesults) ; @@ -478,6 +506,8 @@ bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags, int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchTerms(keywords,firesults) ; @@ -485,6 +515,8 @@ int p3FileDatabase::SearchKeywords(const std::list& keywords, std:: } int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchBoolExp(exp,firesults) ; @@ -492,6 +524,8 @@ int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& result } bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const { + RS_STACK_MUTEX(mFLSMtx) ; + if(hintflags & RS_FILE_HINTS_LOCAL) { std::list res; @@ -557,5 +591,6 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath) { + RS_STACK_MUTEX(mFLSMtx) ; return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; } diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index cd090af61..0fa36088e 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -157,8 +157,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; - uint32_t getFriendIndex(const RsPeerId& pid); - const RsPeerId& getFriendFromIndex(uint32_t indx) const; + uint32_t locked_getFriendIndex(const RsPeerId& pid); + const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; std::map mFriendIndexMap ; std::vector mFriendIndexTab; @@ -169,7 +169,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // Local flags and mutexes - RsMutex mFLSMtx ; + mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index 128f08005..0c88cb97a 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -47,8 +47,8 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) { std::cerr << "====DIR DETAILS====" << std::endl; - std::cerr << " parent pointer: " << (intptr_t)d.parent << std::endl; - std::cerr << " current pointer: " << (intptr_t)d.ref << std::endl; + std::cerr << " parent pointer: " << d.parent << std::endl; + std::cerr << " current pointer: " << d.ref << std::endl; std::cerr << " parent row : " << d.prow << std::endl; std::cerr << " type : " << (int)d.type << std::endl; std::cerr << " PeerId : " << d.id << std::endl; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index c3bf94d5a..fcaec5d09 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -825,7 +825,7 @@ void SharedFilesDialog::preModDirectories(bool local) void SharedFilesDialog::postModDirectories(bool local) { - if (isRemote() == local) { + if (isRemote() == local) { return; } From f8ed1d3fb707bc578f2842ed8c0ed85143865659 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 19 Aug 2016 18:49:42 +0200 Subject: [PATCH 28/92] changed DirDetails::children to be a vector, removed costly caching of DirDetailVector --- .../src/file_sharing/directory_storage.cc | 2 + libretroshare/src/file_sharing/p3filelists.cc | 12 +- libretroshare/src/retroshare/rstypes.h | 2 +- libretroshare/src/rsserver/rstypes.cc | 2 +- .../src/gui/FileTransfer/SearchDialog.cpp | 5 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 203 +++++++++--------- retroshare-gui/src/gui/RemoteDirModel.h | 2 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 4 +- .../src/gui/common/RsCollectionFile.cpp | 6 +- retroshare-gui/src/main.cpp | 2 +- 10 files changed, 124 insertions(+), 116 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d0bd333b6..72aaf6f52 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -545,6 +545,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + d.ref = (void*)(intptr_t)indx ; + if (dir_entry != NULL) /* has children --- fill */ { #ifdef DEBUG_DIRECTORY_STORAGE diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 64bc38924..9a8496470 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -108,11 +108,12 @@ int p3FileDatabase::tick() if(last_print_time + 20 < now) { RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->print(); last_print_time = now ; -#warning this should be removed, but it's necessary atm for updating the GUI - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); +//#warning this should be removed, but it's necessary atm for updating the GUI +// RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } if(mUpdateFlags) @@ -402,12 +403,13 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; - for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) - convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); + for(uint32_t i=0;i children; + std::vector children; std::list parent_groups; // parent groups for the shared directory }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index 0c88cb97a..be83f7d7a 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -60,7 +60,7 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) std::cerr << " Min age : " << d.min_age << std::endl; std::cerr << " Flags : " << d.flags << std::endl; std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; - std::cerr << " Children : " ; for(std::list::const_iterator it(d.children.begin());it!=d.children.end();++it) std::cerr << (intptr_t)(*it).ref << " "; std::cerr << std::endl; + std::cerr << " Children : " ; for(uint32_t i=0;i::const_iterator it(dir.children.begin()); it != dir.children.end(); ++it) { + for (uint32_t i=0;iRequestDirDetails(it->ref, details, FileSearchFlags(0u)); + rsFiles->RequestDirDetails(dir.children[i].ref, details, FileSearchFlags(0u)); insertDirectory(txt, searchId, details, child); } } diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index a2220f2ff..295ebbf83 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -99,10 +99,9 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const void *ref = parent.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); - - if (!details) - { + DirDetails details ; + if (!requestDirDetails(ref, RemoteMode,details)) + { /* error */ #ifdef RDM_DEBUG std::cerr << "lookup failed -> false"; @@ -111,7 +110,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const return false; } - if (details->type == DIR_TYPE_FILE) + if (details.type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE -> false"; @@ -124,7 +123,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return (details->count > 0); /* do we have children? */ + return (details.count > 0); /* do we have children? */ } bool FlatStyle_RDM::hasChildren(const QModelIndex &parent) const { @@ -155,9 +154,9 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const void *ref = (parent.isValid())? parent.internalPointer() : NULL ; - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + if (! requestDirDetails(ref, RemoteMode,details)) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> 0"; @@ -165,7 +164,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const #endif return 0; } - if (details->type == DIR_TYPE_FILE) + if (details.type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE: 0"; @@ -179,7 +178,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return details->count; + return details.count; } int FlatStyle_RDM::rowCount(const QModelIndex &parent) const @@ -381,16 +380,16 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const QString FlatStyle_RDM::computeDirectoryPath(const DirDetails& details) const { QString dir ; - const DirDetails *det = requestDirDetails(details.parent,RemoteMode); + DirDetails det ; - if(!det) - return QString(); + if(!requestDirDetails(details.parent,RemoteMode,det)) + return QString(); #ifdef SHOW_TOTAL_PATH do { #endif - dir = QString::fromUtf8(det->name.c_str())+"/"+dir ; + dir = QString::fromUtf8(det.name.c_str())+"/"+dir ; #ifdef SHOW_TOTAL_PATH if(!requestDirDetails(det.parent,det,flags)) @@ -521,28 +520,28 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const void *ref = index.internalPointer(); int coln = index.column(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - return QVariant(); + if (!requestDirDetails(ref, RemoteMode,details)) + return QVariant(); if (role == RetroshareDirModel::FileNameRole) /* end of FileNameRole */ - return QString::fromUtf8(details->name.c_str()); + return QString::fromUtf8(details.name.c_str()); if (role == Qt::TextColorRole) { - if(details->min_age > ageIndicator) + if(details.min_age > ageIndicator) return QVariant(QColor(Qt::gray)) ; else if(RemoteMode) { FileInfo info; QVariant local_file_color = QVariant(QColor(Qt::red)); - if(rsFiles->alreadyHaveFile(details->hash, info)) + if(rsFiles->alreadyHaveFile(details.hash, info)) return local_file_color; std::list downloads; rsFiles->FileDownloads(downloads); - if(std::find(downloads.begin(), downloads.end(), details->hash) != downloads.end()) + if(std::find(downloads.begin(), downloads.end(), details.hash) != downloads.end()) return local_file_color; else return QVariant(); @@ -553,7 +552,7 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if(role == Qt::DecorationRole) - return decorationRole(*details,coln) ; + return decorationRole(details,coln) ; /***************** Qt::EditRole @@ -578,10 +577,10 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const } /* end of TextAlignmentRole */ if (role == Qt::DisplayRole) - return displayRole(*details,coln) ; + return displayRole(details,coln) ; if (role == SortRole) - return sortRole(index,*details,coln) ; + return sortRole(index,details,coln) ; return QVariant(); } @@ -712,9 +711,10 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } ********/ - const DirDetailsVector *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + + if (! requestDirDetails(ref, RemoteMode,details)) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> invalid"; @@ -728,7 +728,7 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent * get the reference number */ - if (row >= (int) details->childrenVector.size()) + if (row >= (int) details.children.size()) { #ifdef RDM_DEBUG std::cerr << "wrong number of children -> invalid"; @@ -744,7 +744,7 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent /* we can just grab the reference now */ - return createIndex(row, column, details->childrenVector[row].ref); + return createIndex(row, column, details.children[row].ref); } QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const { @@ -786,10 +786,10 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const } void *ref = index.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - { + if (! requestDirDetails(ref, RemoteMode,details)) + { #ifdef RDM_DEBUG std::cerr << "Failed Lookup -> invalid"; std::cerr << std::endl; @@ -797,7 +797,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const return QModelIndex(); } - if (!(details->parent)) + if (!(details.parent)) { #ifdef RDM_DEBUG std::cerr << "success. parent is Root/NULL --> invalid"; @@ -811,7 +811,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const std::cerr << std::endl; #endif - return createIndex(details->prow, 0, details->parent); + return createIndex(details.prow, 0, details.parent); } QModelIndex FlatStyle_RDM::parent( const QModelIndex & index ) const { @@ -836,12 +836,12 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const void *ref = index.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - return Qt::ItemIsSelectable; // Error. + if (! requestDirDetails(ref, RemoteMode,details)) + return Qt::ItemIsSelectable; // Error. - switch(details->type) + switch(details.type) { case DIR_TYPE_PERSON: return Qt::ItemIsEnabled; case DIR_TYPE_DIR: return Qt::ItemIsSelectable | Qt::ItemIsEnabled; @@ -902,32 +902,36 @@ void RetroshareDirModel::postMods() emit layoutChanged(); } -const DirDetailsVector *RetroshareDirModel::requestDirDetails(void *ref, bool remote) const +bool RetroshareDirModel::requestDirDetails(void *ref, bool remote,DirDetails& d) const { - const QMap::const_iterator it = mCache.constFind(ref); - if (it != mCache.constEnd()) { - /* Details found in cache */ - return &it.value(); - } + FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; - /* Get details from the lib */ - DirDetailsVector details; - FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; - if (rsFiles->RequestDirDetails(ref, details, flags)) { - /* Convert std::list to std::vector for fast access with index */ - std::list::const_iterator childIt; - for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) { - details.childrenVector.push_back(*childIt); - } - - /* Add to cache, must cast to none const */ - const QMap::iterator it1 = ((QMap*) &mCache)->insert(ref, details); - return &it1.value(); - } - - /* No details found */ - return NULL; + return rsFiles->RequestDirDetails(ref, d, flags) ; } +// const QMap::const_iterator it = mCache.constFind(ref); +// if (it != mCache.constEnd()) { +// /* Details found in cache */ +// return &it.value(); +// } +// +//#warning this is terrible! A vector in a std::map will keep being copied and create a lot of CPU overload. Use a pointer instead. +// /* Get details from the lib */ +// +// if (rsFiles->RequestDirDetails(ref, details, flags)) { +// /* Convert std::list to std::vector for fast access with index */ +// std::list::const_iterator childIt; +// for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) { +// details.childrenVector.push_back(*childIt); +// } +// +// /* Add to cache, must cast to none const */ +// const QMap::iterator it1 = ((QMap*) &mCache)->insert(ref, details); +// return &it1.value(); +// } +// +// /* No details found */ +// return NULL; +//} void RetroshareDirModel::createCollectionFile(QWidget *parent, const QModelIndexList &list) { @@ -1010,15 +1014,16 @@ void RetroshareDirModel::downloadDirectory(const DirDetails & dirDetails, int pr if (!dwlDir.mkpath(cleanPath)) return; - for (it = dirDetails.children.begin(); it != dirDetails.children.end(); ++it) + for(uint32_t i=0;iref) continue; + if (!dirDetails.children[i].ref) continue; - const DirDetails *subDirDetails = requestDirDetails(it->ref, true); + DirDetails subDirDetails ; - if (!subDirDetails) continue; + if(!requestDirDetails(dirDetails.children[i].ref, true,subDirDetails)) + continue; - downloadDirectory(*subDirDetails, prefixLen); + downloadDirectory(subDirDetails, prefixLen); } } } @@ -1035,12 +1040,12 @@ void RetroshareDirModel::getDirDetailsFromSelect (const QModelIndexList &list, s { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + if(!requestDirDetails(ref, RemoteMode,details)) continue; - dirVec.push_back(*details); + dirVec.push_back(details); } } } @@ -1072,12 +1077,12 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details; - if (!details) + if (!requestDirDetails(ref, RemoteMode,details)) continue; - if(details->type == DIR_TYPE_PERSON) + if(details.type == DIR_TYPE_PERSON) continue ; #ifdef RDM_DEBUG @@ -1090,10 +1095,10 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s // Note: for directories, the returned hash, is the peer id, so if we collect // dirs, we need to be a bit more conservative for the - if(already_in.find(details->hash.toStdString()+details->name) == already_in.end()) + if(already_in.find(details.hash.toStdString()+details.name) == already_in.end()) { - file_details.push_back(*details) ; - already_in.insert(details->hash.toStdString()+details->name) ; + file_details.push_back(details) ; + already_in.insert(details.hash.toStdString()+details.name) ; } } #ifdef RDM_DEBUG @@ -1167,17 +1172,17 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list internalPointer(); - const DirDetails *details = requestDirDetails(ref, false); + DirDetails details ; - if (!details) - { + if (!requestDirDetails(ref, false,details) ) + { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Bad Request" << std::endl; #endif continue; } - if (details->type != DIR_TYPE_FILE) + if (details.type != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Not File" << std::endl; @@ -1187,14 +1192,14 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::listname << std::endl; - std::cerr << "Hash: " << details->hash << std::endl; - std::cerr << "Size: " << details->count << std::endl; - std::cerr << "Path: " << details->path << std::endl; + std::cerr << "Name: " << details.name << std::endl; + std::cerr << "Hash: " << details.hash << std::endl; + std::cerr << "Size: " << details.count << std::endl; + std::cerr << "Path: " << details.path << std::endl; #endif - std::string filepath = details->path + "/"; - filepath += details->name; + std::string filepath = details.path + "/"; + filepath += details.name; #ifdef RDM_DEBUG std::cerr << "Constructed FilePath: " << filepath << std::endl; @@ -1222,12 +1227,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); - - if (!details) - { - continue; - } + DirDetails details ; + if (!requestDirDetails(ref, RemoteMode,details)) + continue; #ifdef RDM_DEBUG std::cerr << "::::::::::::FileDrag:::: " << std::endl; @@ -1237,7 +1239,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con std::cerr << "Path: " << details->path << std::endl; #endif - if (details->type != DIR_TYPE_FILE) + if (details.type != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Not File" << std::endl; @@ -1245,7 +1247,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* not file! */ } - if (drags.end() != (dit = drags.find(details->hash))) + if (drags.end() != (dit = drags.find(details.hash))) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Duplicate" << std::endl; @@ -1253,9 +1255,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* duplicate */ } - drags[details->hash] = details->count; + drags[details.hash] = details.count; - QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details->name.c_str()), QString::fromStdString(details->hash.toStdString()), QString::number(details->count)); + QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details.name.c_str()), QString::fromStdString(details.hash.toStdString()), QString::number(details.count)); if (RemoteMode) { @@ -1348,17 +1350,18 @@ void FlatStyle_RDM::updateRefs() std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; #endif _ref_stack.pop_back() ; - const DirDetails *details = requestDirDetails(ref, RemoteMode) ; - if (details) + DirDetails details ; + + if (requestDirDetails(ref, RemoteMode,details)) { - if(details->type == DIR_TYPE_FILE) // only push files, not directories nor persons. - _ref_entries.push_back(std::pair(ref,computeDirectoryPath(*details))); + if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. + _ref_entries.push_back(std::pair(ref,computeDirectoryPath(details))); #ifdef RDM_DEBUG std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; #endif - for(std::list::const_iterator it = details->children.begin(); it != details->children.end(); ++it) - _ref_stack.push_back(it->ref) ; + for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to { // the user and setup a timer to finish the job later. diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index a16dcfc3f..dd81e2c84 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -74,7 +74,7 @@ class RetroshareDirModel : public QAbstractItemModel void getFilePaths(const QModelIndexList &list, std::list &fullpaths); void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } - const DirDetailsVector *requestDirDetails(void *ref, bool remote) const; + bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} public: diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index fcaec5d09..67a5e3d0e 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -544,9 +544,9 @@ void SharedFilesDialog::copyLink (const QModelIndexList& lst, bool remote) if (details.type == DIR_TYPE_DIR) { - for (std::list::const_iterator cit = details.children.begin();cit != details.children.end(); ++cit) + for(uint32_t j=0;j::const_iterator it = details.children.begin(); it != details.children.end(); ++it) + for(uint32_t i=0;iref) + if (!details.children[i].ref) continue; DirDetails subDirDetails; FileSearchFlags flags = RS_FILE_HINTS_LOCAL; - if (!rsFiles->RequestDirDetails(it->ref, subDirDetails, flags)) + if (!rsFiles->RequestDirDetails(details.children[i].ref, subDirDetails, flags)) continue; recursAddElements(doc,subDirDetails,d) ; diff --git a/retroshare-gui/src/main.cpp b/retroshare-gui/src/main.cpp index f7e7c835d..c64b6fa54 100644 --- a/retroshare-gui/src/main.cpp +++ b/retroshare-gui/src/main.cpp @@ -368,7 +368,7 @@ feenableexcept(FE_INVALID | FE_DIVBYZERO); QObject::connect(notify,SIGNAL(deferredSignatureHandlingRequested()),notify,SLOT(handleSignatureEvent()),Qt::QueuedConnection) ; QObject::connect(notify,SIGNAL(chatLobbyTimeShift(int)),notify,SLOT(handleChatLobbyTimeShift(int)),Qt::QueuedConnection) ; QObject::connect(notify,SIGNAL(diskFull(int,int)) ,w ,SLOT(displayDiskSpaceWarning(int,int))) ; - QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool) )) ; + QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool)) ,Qt::QueuedConnection ) ; QObject::connect(notify,SIGNAL(transfersChanged()) ,w->transfersDialog ,SLOT(insertTransfers() )) ; QObject::connect(notify,SIGNAL(publicChatChanged(int)) ,w->friendsDialog ,SLOT(publicChatChanged(int) )); QObject::connect(notify,SIGNAL(neighboursChanged()) ,w->friendsDialog->networkDialog ,SLOT(securedUpdateDisplay())) ; From edc602f68f67e5549af3288d5b4bb7a137a6f71b Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 20 Aug 2016 16:23:11 +0200 Subject: [PATCH 29/92] added infrastructure for synchronisation of file lists --- .../src/file_sharing/directory_storage.cc | 22 +- .../src/file_sharing/directory_storage.h | 6 +- .../src/file_sharing/directory_updater.cc | 11 +- .../src/file_sharing/directory_updater.h | 10 +- libretroshare/src/file_sharing/p3filelists.cc | 198 ++++++++--- libretroshare/src/file_sharing/p3filelists.h | 23 +- .../src/file_sharing/rsfilelistitems.cc | 336 ++++++++++++++++++ .../src/file_sharing/rsfilelistitems.h | 132 +++++++ libretroshare/src/libretroshare.pro | 18 +- libretroshare/src/serialiser/itempriorities.h | 5 + 10 files changed, 692 insertions(+), 69 deletions(-) create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.cc create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.h diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 72aaf6f52..7e8cea686 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -303,6 +303,20 @@ class InternalFileHierarchyStorage return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } + bool searchHash(const RsFileHash& hash,std::list& results) + { + std::map::const_iterator it = mHashes.find(hash); + + if( it != mHashes.end() ) + { + results.clear(); + results.push_back(it->second) ; + return true ; + } + else + return false; + } + bool check() // checks consistency of storage. { return true; @@ -390,7 +404,7 @@ private: return true ; } - std::map mHashes ; // used for fast search access + std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? }; /******************************************************************************************************************/ @@ -498,6 +512,12 @@ bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash return mFileHierarchy->updateHash(index,hash); } +int DirectoryStorage::searchHash(const RsFileHash& hash, std::list &results) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchHash(hash,results); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9f2b63be9..637a8e88e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -23,10 +23,10 @@ class DirectoryStorage void save() const ; virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} - virtual int searchHash(const RsFileHash& hash, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } + virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - void getFileDetails(EntryIndex i) ; + bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -47,6 +47,8 @@ class DirectoryStorage // info about the directory that is pointed by the iterator std::string name() const ; + time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified. + time_t last_update_time() const ; // last time this directory was updated private: EntryIndex mParentIndex ; // index of the parent dir. uint32_t mDirTabIndex ; // index in the vector of subdirs. diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 7f427ff7d..731d92902 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,12 +6,12 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +//=============================================================================================================// +// Local Directory Updater // +//=============================================================================================================// -void RemoteDirectoryUpdater::tick() -{ - // use the stored iterator -} +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) @@ -139,4 +139,3 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } - diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index d146a61b5..1248aeefb 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -13,6 +13,8 @@ class DirectoryUpdater virtual ~DirectoryUpdater(){} }; +#warning: simplify, if we don't keep the remote directory updater + class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread { public: @@ -35,11 +37,3 @@ private: time_t mLastSweepTime; }; -class RemoteDirectoryUpdater: public DirectoryUpdater -{ - public: - RemoteDirectoryUpdater() {} - virtual ~RemoteDirectoryUpdater() {} - - virtual void tick() ; -}; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9a8496470..7772af5c6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -3,6 +3,7 @@ #include "file_sharing/p3filelists.h" #include "file_sharing/directory_storage.h" #include "file_sharing/directory_updater.h" +#include "file_sharing/rsfilelistitems.h" #include "retroshare/rsids.h" #include "retroshare/rspeers.h" @@ -25,17 +26,16 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) { // loads existing indexes for friends. Some might be already present here. // - mDirectories.clear() ; // we should load them! - - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ; - mDirectories.push_back(mLocalSharedDirs) ; + mRemoteDirectories.clear() ; // we should load them! + mOwnId = mpeers->getOwnId() ; + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); mHashCache = new HashStorage("hash_cache.bin") ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; - mRemoteDirWatcher = NULL; // not used yet mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + mLastRemoteDirSweepTS = 0 ; } void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) @@ -60,14 +60,13 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(uint32_t i=0;iroot()) ; + + mLastRemoteDirSweepTS = now; + } + return 0; } @@ -153,19 +163,12 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } -void p3FileDatabase::tickRecv() -{ -} -void p3FileDatabase::tickSend() -{ - // go through the list of out requests and send them to the corresponding friends, if they are online. -} - bool p3FileDatabase::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + NOT_IMPLEMENTED(); return true ; @@ -194,19 +197,19 @@ void p3FileDatabase::cleanup() friend_set.insert(*it) ; } - for(uint32_t i=1;ipeerId()) == friend_set.end()) + for(uint32_t i=0;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; - delete mDirectories[i]; - mDirectories[i] = NULL ; + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - friend_set.erase(mDirectories[i]->peerId()); + friend_set.erase(mRemoteDirectories[i]->peerId()); - mFriendIndexMap.erase(mDirectories[i]->peerId()); + mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); mFriendIndexTab[i].clear(); } @@ -218,12 +221,12 @@ void p3FileDatabase::cleanup() uint32_t friend_index = locked_getFriendIndex(*it) ; - if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) + if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) continue ; P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; - mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -244,7 +247,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { // allocate a new index for that friend, and tell that we should save. uint32_t found = 0 ; - for(uint32_t i=1;isecond) - mDirectories.resize(it->second+1,NULL) ; + if(mRemoteDirectories.size() <= it->second) + mRemoteDirectories.resize(it->second,NULL) ; return it->second; } @@ -372,14 +375,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags stub.ref = p; d.children.push_back(stub); } - else for(uint32_t i=1;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i,p); DirStub stub; stub.type = DIR_TYPE_PERSON; - stub.name = mDirectories[i]->peerId().toStdString(); + stub.name = mRemoteDirectories[i]->peerId().toStdString(); stub.ref = p; d.children.push_back(stub); } @@ -396,7 +399,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - bool res = mDirectories[fi]->extractData(e,d) ; + bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -414,11 +417,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags } else { - d.prow = mDirectories[fi]->parentRow(e) ; + d.prow = mRemoteDirectories[fi]->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - d.id = mDirectories[fi]->peerId(); + if(flags & RS_FILE_HINTS_LOCAL) + d.id = mOwnId ; + else + d.id = mRemoteDirectories[fi]->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -451,7 +457,7 @@ uint32_t p3FileDatabase::getType(void *ref) const if(e == 0) return DIR_TYPE_PERSON ; - return mDirectories[fi]->getEntryType(e) ; + return mRemoteDirectories[fi]->getEntryType(e) ; } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed @@ -536,13 +542,14 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F if(res.empty()) return false; - EntryIndex indx = *res.begin() ; // no need to report dupicates + EntryIndex indx = *res.begin() ; // no need to report duplicates mLocalSharedDirs->getFileInfo(indx,info) ; return true; } - else + + if(hintflags & RS_FILE_HINTS_REMOTE) { NOT_IMPLEMENTED(); return false; @@ -596,3 +603,114 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& RS_STACK_MUTEX(mFLSMtx) ; return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; } + +//==============================================================================================================================// +// Update of remote directories // +//==============================================================================================================================// + +void p3FileDatabase::tickRecv() +{ + RsItem *item ; + + while( NULL != (item = recvItem()) ) + { + switch(item->PacketSubType()) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; + break ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast(item) ) ; + break ; + default: + std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; + } + + delete item ; + } +} + +void p3FileDatabase::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. +} + +void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) +{ + // look at item TS. If local is newer, send the full directory content. + + time_t recurs_max_modf_TS; + + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + { + // send full update of directory content + } + else + { + // send last recurs update TS. + } +} + +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +{ + // update the directory content for the specified friend. + + // set the update TS, and the remote modif TS accordingly + + // notify the GUI if the hierarchy has changed +} + +void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) +{ + time_t now = time(NULL) ; + + // get the info for this entry + + // compare TS + + // if not up to date, request update, and return (content is not certified, so no need to recurs yet). + // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. + + time_t recurs_max_modf_TS,last_update_TS ; + + if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + return ; + } + + if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) + { + // check if a request already exists and is not too old either: no need to re-ask. + + DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ; + + std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; + + if(it != mPendingSyncRequests.end()) + { + std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + return ; + } + + std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl; + + DirSyncRequestData data ; + + data.request_TS = now ; + + mPendingSyncRequests[sync_req_id] = data ; + + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + item->entry_index = e ; + item->known_TS = recurs_max_modf_TS ; + + sendItem(item) ; + + return ; + } +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 0fa36088e..8160095aa 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -37,6 +37,9 @@ class LocalDirectoryUpdater ; class RemoteDirectoryStorage ; class LocalDirectoryStorage ; +class RsFileListsSyncReqItem ; +class RsFileListsSyncDirItem ; + class HashStorage ; class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService @@ -135,22 +138,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub private: p3ServiceControl *mServCtrl ; + RsPeerId mOwnId ; + + typedef uint64_t DirSyncRequestId ; + + static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. // - std::list mFastRequestQueue ; - std::list mSlowRequestQueue ; + std::map mFastRequestQueue ; + std::map mSlowRequestQueue ; // Directory storage hierarchies // // The remote one is the reference for the PeerId index below: // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) - std::vector mDirectories ; // mDirectories[0]=mLocalSharedDirs + std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; - RemoteDirectoryUpdater *mRemoteDirWatcher ; // not used yet. LocalDirectoryUpdater *mLocalDirWatcher ; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc @@ -160,9 +167,17 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t locked_getFriendIndex(const RsPeerId& pid); const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; + void handleDirSyncRequest(RsFileListsSyncReqItem *) ; + void handleDirSyncContent(RsFileListsSyncDirItem *) ; + std::map mFriendIndexMap ; std::vector mFriendIndexTab; + // TS for friend list update + time_t mLastRemoteDirSweepTS ; + + void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // HashStorage *mHashCache ; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc new file mode 100644 index 000000000..986b4284c --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -0,0 +1,336 @@ +#include "file_sharing/rsfilelistitems.h" + +RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ + + switch(getRsItemSubType(rstype)) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncReqItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: return deserialFileListsSyncDirItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size); + + default: + { + std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl; + return NULL; + + } + } +} + +uint32_t RsFileListsSerialiser::size(RsItem *item) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serial_size() ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const +{ + tlvsize = serial_size() ; + offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize)) + { + std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl; + return false ; + } +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl; +#endif + offset += 8; + + return true ; +} + +bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + ok &= directory_content_data.SetTlv(data,size,&offset) ; + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +//============================================================================================================================// +// Deserialisation // +//============================================================================================================================// + +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + ok &= item->directory_content_data.GetTlv(data,*size,&offset) ; + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; +#endif + return false; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl; +#endif + return false; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + return true ; +} + +//============================================================================================================================// +// Sizes // +//============================================================================================================================// + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + + return s; +} + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + s += directory_content_data.TlvSize(); + + return s; +} + +void RsFileListsSyncReqItem::clear() +{ +} +void RsFileListsSyncDirItem::clear() +{ + directory_content_data.TlvClear(); +} +std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncReqItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncReqItem", indent); + + return out; +} + +std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncDirItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + printIndent(out , int_Indent); out << "Data size: " << directory_content_data.bin_len << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncDirItem", indent); + + return out; +} diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h new file mode 100644 index 000000000..6eb463b7d --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -0,0 +1,132 @@ +#pragma once + +/* + * libretroshare/src/serialiser: rsnxssitems.h + * + * RetroShare Serialiser. + * + * Copyright 2012 Christopher Evi-Parker, Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include +#include + +#include "serialiser/rsserviceids.h" +#include "serialiser/rsserial.h" +#include "serialiser/rstlvbase.h" +#include "serialiser/rstlvitem.h" +#include "serialiser/rstlvkeys.h" +#include "gxs/rsgxsdata.h" + +// These items have "flag type" numbers, but this is not used. + +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM = 0x02; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03; + +/*! + * Base class for filelist sync items + */ +class RsFileListsItem : public RsItem +{ +public: + RsFileListsItem(uint8_t subtype) + : RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype) + { + setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST); // this is the default. Someitems may be faster, on demand. + return; + } + virtual ~RsFileListsItem(){} + + virtual bool serialise(void *data,uint32_t& size) const = 0 ; + virtual uint32_t serial_size() const = 0 ; + virtual void clear() = 0; + virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0; + + bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const; + + static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ; + static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ; + static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0004 ; + static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ; +}; + +/*! + * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date. + */ +class RsFileListsSyncReqItem : public RsFileListsItem +{ +public: + + RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash +}; + +class RsFileListsSyncDirItem : public RsFileListsItem +{ +public: + + RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. +}; + +class RsFileListsSerialiser : public RsSerialType +{ +public: + + RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} + + virtual ~RsFileListsSerialiser() {} + + virtual uint32_t size(RsItem *item); + virtual bool serialise(RsItem *item, void *data, uint32_t *size); + virtual RsItem* deserialise(void *data, uint32_t *size); + +private: + RsFileListsSyncReqItem *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + + bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type); +}; + + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 70994b8c7..ea7344976 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -37,16 +37,18 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ - file_sharing/hash_cache.h \ - file_sharing/filelist_io.h \ - file_sharing/directory_storage.h \ - file_sharing/directory_updater.h + file_sharing/hash_cache.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ + file_sharing/directory_updater.h \ + file_sharing/filelistitems.h SOURCES *= file_sharing/p3filelists.cc \ - file_sharing/hash_cache.cc \ - file_sharing/filelist_io.cc \ - file_sharing/directory_storage.cc \ - file_sharing/directory_updater.cc + file_sharing/hash_cache.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ + file_sharing/directory_updater.cc \ + file_sharing/filelistitems.cc } dsdv { diff --git a/libretroshare/src/serialiser/itempriorities.h b/libretroshare/src/serialiser/itempriorities.h index 8809754a9..f5008eef0 100644 --- a/libretroshare/src/serialiser/itempriorities.h +++ b/libretroshare/src/serialiser/itempriorities.h @@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST = 2 ; // same priority. const uint8_t QOS_PRIORITY_RS_DISC_SERVICES = 2 ; const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT = 1 ; +// File database +// +const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST = 7 ; +const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST = 3 ; + // Heartbeat. // const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE = 8 ; From 76831348e63452dcec59d8c57cc39d49d92adc5a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 22 Aug 2016 07:49:45 +0200 Subject: [PATCH 30/92] added update of last modif LS, and rsitems to update directories between friends --- .../src/file_sharing/directory_storage.cc | 40 +++++++++++++++-- .../src/file_sharing/directory_storage.h | 1 + .../src/file_sharing/directory_updater.cc | 31 +++++++++---- .../src/file_sharing/directory_updater.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 45 ++++++++++++------- libretroshare/src/file_sharing/p3filelists.h | 12 ++++- .../src/file_sharing/rsfilelistitems.cc | 40 +++++++---------- .../src/file_sharing/rsfilelistitems.h | 22 ++++----- libretroshare/src/libretroshare.pro | 4 +- 9 files changed, 129 insertions(+), 67 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7e8cea686..8ea522b6f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -53,7 +53,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -65,7 +65,8 @@ class InternalFileHierarchyStorage std::vector subdirs ; std::vector subfiles ; - time_t most_recent_time; + time_t dir_modtime ; // this accounts for deleted files, etc. + time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. }; // class stuff @@ -221,7 +222,10 @@ class InternalFileHierarchyStorage std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; - static_cast(mNodes[file_index])->file_hash = hash ; + RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + + old_hash = hash ; + return true; } bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) @@ -243,6 +247,25 @@ class InternalFileHierarchyStorage return true; } + // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + + time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) + { + DirEntry& d(*static_cast(mNodes[dir_index])) ; + + time_t largest_modf_time = d.dir_modtime ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + + for(uint32_t i=0;i(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ; for(int i=0;irecursUpdateLastModfTime(EntryIndex(0)) ; + + std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +} + std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const { /**** MUST ALREADY BE LOCKED ****/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 637a8e88e..fbc6b3297 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -137,6 +137,7 @@ public: void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; + void updateTimeStamps(); /*! * \brief getFileInfo Converts an index info a full file info structure. * \param i index in the directory structure diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 731d92902..f9fada262 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,13 +10,15 @@ // Local Directory Updater // //=============================================================================================================// -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { mLastSweepTime = 0; + mLastTSUpdateTime = 0; } void LocalDirectoryUpdater::data_tick() @@ -28,8 +30,13 @@ void LocalDirectoryUpdater::data_tick() sweepSharedDirectories() ; mLastSweepTime = now; } - else - usleep(10*1000*1000); + + if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime) + { + mSharedDirectories->updateTimeStamps() ; + mLastTSUpdateTime = now ; + } + usleep(10*1000*1000); } void LocalDirectoryUpdater::forceUpdate() @@ -93,13 +100,13 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p switch(dirIt.file_type()) { case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + subfiles[dirIt.file_name()].size = dirIt.file_size(); + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; - break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + break; default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } @@ -139,3 +146,9 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 1248aeefb..94b9a2080 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -35,5 +35,6 @@ private: LocalDirectoryStorage *mSharedDirectories ; time_t mLastSweepTime; + time_t mLastTSUpdateTime; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 7772af5c6..5ef9af42a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -17,9 +17,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; -static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; -static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; +static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; +static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") @@ -608,6 +609,17 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // Update of remote directories // //==============================================================================================================================// +// Algorithm: +// +// Local dirs store the last modif time of the file, in local time +// - the max time is computed upward until the root of the hierarchy +// - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. +// +// Remote dirs store the last modif time of the files/dir in the friend's time +// - local node sends the last known modf time to friends, +// - friends respond with either a full directory content, or an acknowledge that the time is right +// + void p3FileDatabase::tickRecv() { RsItem *item ; @@ -637,15 +649,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) { // look at item TS. If local is newer, send the full directory content. - time_t recurs_max_modf_TS; + time_t recurs_max_modf_TS, last_update_TS; - if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) - { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; - return ; - } + P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->last_known_recurs_modf_TS < recurs_max_modf_TS) { // send full update of directory content } @@ -655,7 +669,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) } } -void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item) { // update the directory content for the specified friend. @@ -677,9 +691,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS,last_update_TS ; - if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl; return ; } @@ -705,9 +719,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r mPendingSyncRequests[sync_req_id] = data ; - RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ; item->entry_index = e ; - item->known_TS = recurs_max_modf_TS ; + item->last_known_recurs_modf_TS = recurs_max_modf_TS ; + item->PeerId(rds->peerId()) ; sendItem(item) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 8160095aa..58a635d8d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -173,8 +173,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub std::map mFriendIndexMap ; std::vector mFriendIndexTab; - // TS for friend list update - time_t mLastRemoteDirSweepTS ; + // Directory synchronization + // + struct DirSyncRequestData + { + time_t request_TS ; + uint32_t flags ; + }; + + time_t mLastRemoteDirSweepTS ; // TS for friend list update + std::map mPendingSyncRequests ; // pending requests, waiting for an answer void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 986b4284c..198bcc9db 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -1,3 +1,5 @@ +#include "serialiser/rsbaseserial.h" + #include "file_sharing/rsfilelistitems.h" RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) @@ -8,8 +10,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) /* get the type and size */ uint32_t rstype = getRsItemId(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) - return NULL; /* wrong type */ + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ switch(getRsItemSubType(rstype)) { @@ -28,26 +30,26 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) uint32_t RsFileListsSerialiser::size(RsItem *item) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serial_size() ; + if(flst_item != NULL) + return flst_item->serial_size() ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serialise(data,*size) ; + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } @@ -185,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void return item; } -RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem(); uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. @@ -234,7 +236,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su uint32_t rstype = getRsItemId(data); uint32_t rssize = getRsItemSize(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) { #ifdef RSSERIAL_DEBUG std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; @@ -265,11 +267,6 @@ uint32_t RsFileListsSyncReqItem::serial_size()const uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS @@ -278,16 +275,11 @@ uint32_t RsFileListsSyncReqItem::serial_size()const return s; } -uint32_t RsFileListsSyncReqItem::serial_size()const +uint32_t RsFileListsSyncDirItem::serial_size()const { uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 6eb463b7d..5ddd1f063 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -75,7 +75,7 @@ class RsFileListsSyncReqItem : public RsFileListsItem { public: - RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -93,20 +93,20 @@ class RsFileListsSyncDirItem : public RsFileListsItem { public: - RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} - virtual void clear(); - virtual std::ostream &print(std::ostream &out, uint16_t indent); + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); - virtual bool serialise(void *data,uint32_t& size) const; - virtual uint32_t serial_size() const ; + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash - RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. }; class RsFileListsSerialiser : public RsSerialType diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index ea7344976..82b0ea27b 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -41,14 +41,14 @@ file_lists { file_sharing/filelist_io.h \ file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ - file_sharing/filelistitems.h + file_sharing/rsfilelistitems.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ file_sharing/filelist_io.cc \ file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ - file_sharing/filelistitems.cc + file_sharing/rsfilelistitems.cc } dsdv { From 99ea8af015df77f20d4aeb8e908478ba89605f7b Mon Sep 17 00:00:00 2001 From: MrAlice Date: Tue, 23 Aug 2016 21:23:58 +0200 Subject: [PATCH 31/92] added logic for file list item exchange requests --- .../src/file_sharing/directory_storage.h | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 144 +++++++++++++----- libretroshare/src/file_sharing/p3filelists.h | 8 +- .../src/file_sharing/rsfilelistitems.cc | 28 ++-- .../src/file_sharing/rsfilelistitems.h | 21 +-- 5 files changed, 140 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fbc6b3297..309bff328 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -26,7 +26,9 @@ class DirectoryStorage virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ; + bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; + bool setUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 5ef9af42a..6fa89bd35 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -555,6 +555,7 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F NOT_IMPLEMENTED(); return false; } + return false; } int p3FileDatabase::filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const @@ -628,9 +629,9 @@ void p3FileDatabase::tickRecv() { switch(item->PacketSubType()) { - case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; break ; - case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast(item) ) ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: handleDirSyncResponse( dynamic_cast(item) ) ; break ; default: std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; @@ -645,59 +646,119 @@ void p3FileDatabase::tickSend() // go through the list of out requests and send them to the corresponding friends, if they are online. } -void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) +void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { + RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem; + // look at item TS. If local is newer, send the full directory content. - - time_t recurs_max_modf_TS, last_update_TS; - - P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - - if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl; - return ; + RS_STACK_MUTEX(mFLSMtx) ; + + P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + + uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ; + ritem->PeerId(item->PeerId()) ; + ritem->request_id = item->request_id; + ritem->entry_index = item->entry_index ; + + if(entry_type != DIR_TYPE_DIR) + { + P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory. Answering with proper flags." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; + } + else + { + time_t local_recurs_max_time,local_update_time; + mLocalSharedDirs->getUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + + if(item->last_known_recurs_modf_TS < local_recurs_max_time) + { + P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; + ritem->last_known_recurs_modf_TS = local_recurs_max_time; + + mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data) ; + } + else + { + P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; + ritem->last_known_recurs_modf_TS = local_update_time ; + } + } } - if(item->last_known_recurs_modf_TS < recurs_max_modf_TS) - { - // send full update of directory content - } - else - { - // send last recurs update TS. - } + // sends the response. + + sendItem(ritem); } -void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item) +void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { - // update the directory content for the specified friend. + // find the correct friend entry - // set the update TS, and the remote modif TS accordingly + uint32_t fi = 0 ; - // notify the GUI if the hierarchy has changed + P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + { + RS_STACK_MUTEX(mFLSMtx) ; + fi = locked_getFriendIndex(item->PeerId()); + + // make sure we have a remote directory for that friend. + + if(mRemoteDirectories.size() <= fi) + mRemoteDirectories.resize(fi+1,NULL) ; + + if(mRemoteDirectories[fi] == NULL) + mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); + } + + if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) + { + P3FILELISTS_DEBUG() << " removing directory with index " << item->entry_index << " because it does not exist." << std::endl; + mRemoteDirectories[fi]->removeDirectory(item->entry_index); + } + else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) + { + P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; + + mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + } + else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) + { + P3FILELISTS_DEBUG() << " Item contains directory data. Uptating." << std::endl; + + mRemoteDirectories[fi]->deserialiseUpdateEntry(item->entry_index,item->directory_content_data) ; + mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + +#warning should notify the GUI here + // notify the GUI if the hierarchy has changed + } } void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) { time_t now = time(NULL) ; - // get the info for this entry - - // compare TS - // if not up to date, request update, and return (content is not certified, so no need to recurs yet). // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. - time_t recurs_max_modf_TS,last_update_TS ; + // get the info for this entry - if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS)) + time_t recurs_max_modf_TS_remote_time,local_update_TS; + + if(!rds->getUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl; - return ; + std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + return; } - if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) + // compare TS + + if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. { // check if a request already exists and is not too old either: no need to re-ask. @@ -707,11 +768,11 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(it != mPendingSyncRequests.end()) { - std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + P3FILELISTS_DEBUG() << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; return ; } - std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl; + P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; DirSyncRequestData data ; @@ -719,13 +780,24 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r mPendingSyncRequests[sync_req_id] = data ; - RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ; + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; item->entry_index = e ; - item->last_known_recurs_modf_TS = recurs_max_modf_TS ; + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; + item->request_id = sync_req_id ; + item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; item->PeerId(rds->peerId()) ; sendItem(item) ; + // Dont recurs into sub-directories, since we dont know yet were to go. + return ; } + + for(DirectoryStorage::DirIterator it(rds,e);it;++it) + locked_recursSweepRemoteDirectory(rds,*it); } + + + + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 58a635d8d..a15333ace 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -37,8 +37,8 @@ class LocalDirectoryUpdater ; class RemoteDirectoryStorage ; class LocalDirectoryStorage ; -class RsFileListsSyncReqItem ; -class RsFileListsSyncDirItem ; +class RsFileListsSyncRequestItem ; +class RsFileListsSyncResponseItem ; class HashStorage ; @@ -167,8 +167,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t locked_getFriendIndex(const RsPeerId& pid); const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; - void handleDirSyncRequest(RsFileListsSyncReqItem *) ; - void handleDirSyncContent(RsFileListsSyncDirItem *) ; + void handleDirSyncRequest (RsFileListsSyncRequestItem *) ; + void handleDirSyncResponse (RsFileListsSyncResponseItem *) ; std::map mFriendIndexMap ; std::vector mFriendIndexTab; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 198bcc9db..ae2d1dc5f 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -15,8 +15,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) switch(getRsItemSubType(rstype)) { - case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncReqItem(data, size); - case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: return deserialFileListsSyncDirItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncRequestItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return deserialFileListsSyncResponseItem(data, size); case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size); default: @@ -77,7 +77,7 @@ bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tl return true ; } -bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const +bool RsFileListsSyncRequestItem::serialise(void *data, uint32_t& size) const { uint32_t tlvsize,offset=0; bool ok = true; @@ -113,7 +113,7 @@ bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const return ok; } -bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const +bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const { uint32_t tlvsize,offset=0; bool ok = true; @@ -154,12 +154,12 @@ bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const // Deserialisation // //============================================================================================================================// -RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size) +RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + RsFileListsSyncRequestItem* item = new RsFileListsSyncRequestItem(); ok &= getRawUInt32(data, *size, &offset, &item->entry_index); ok &= getRawUInt32(data, *size, &offset, &item->flags); @@ -187,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void return item; } -RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem(); + RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem(); uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. @@ -262,7 +262,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su // Sizes // //============================================================================================================================// -uint32_t RsFileListsSyncReqItem::serial_size()const +uint32_t RsFileListsSyncRequestItem::serial_size()const { uint32_t s = 8; //header size @@ -275,7 +275,7 @@ uint32_t RsFileListsSyncReqItem::serial_size()const return s; } -uint32_t RsFileListsSyncDirItem::serial_size()const +uint32_t RsFileListsSyncResponseItem::serial_size()const { uint32_t s = 8; //header size @@ -289,14 +289,14 @@ uint32_t RsFileListsSyncDirItem::serial_size()const return s; } -void RsFileListsSyncReqItem::clear() +void RsFileListsSyncRequestItem::clear() { } -void RsFileListsSyncDirItem::clear() +void RsFileListsSyncResponseItem::clear() { directory_content_data.TlvClear(); } -std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) +std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsFileListsSyncReqItem", indent); uint16_t int_Indent = indent + 2; @@ -311,7 +311,7 @@ std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) return out; } -std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent) +std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsFileListsSyncDirItem", indent); uint16_t int_Indent = indent + 2; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 5ddd1f063..a396574d1 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -38,7 +38,7 @@ // These items have "flag type" numbers, but this is not used. const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01; -const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM = 0x02; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02; const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03; /*! @@ -64,18 +64,19 @@ public: static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ; static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ; - static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0004 ; - static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ; + static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ; + static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ; + static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ; }; /*! * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date. */ -class RsFileListsSyncReqItem : public RsFileListsItem +class RsFileListsSyncRequestItem : public RsFileListsItem { public: - RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + RsFileListsSyncRequestItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -89,11 +90,11 @@ public: uint64_t request_id; // use to determine if changes that have occured since last hash }; -class RsFileListsSyncDirItem : public RsFileListsItem +class RsFileListsSyncResponseItem : public RsFileListsItem { public: - RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + RsFileListsSyncResponseItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -122,9 +123,9 @@ public: virtual RsItem* deserialise(void *data, uint32_t *size); private: - RsFileListsSyncReqItem *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ - RsFileListsSyncDirItem *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ - RsFileListsSyncDirItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncRequestItem *deserialFileListsSyncRequestItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncResponseItem *deserialFileListsSyncResponseItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncResponseItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type); }; From d8a24c8c3a5714923d62181c2e9a28bcdb488cf4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 26 Aug 2016 16:29:02 +0200 Subject: [PATCH 32/92] added serialisation of dir entries for exchange with friends. Fixed a few bugs in row and ref calculation --- .../src/file_sharing/directory_storage.cc | 211 +++++++++++++++++- .../src/file_sharing/directory_storage.h | 30 ++- libretroshare/src/file_sharing/filelist_io.h | 3 + libretroshare/src/file_sharing/p3filelists.cc | 76 +++++-- .../src/file_sharing/rsfilelistitems.cc | 9 +- .../src/file_sharing/rsfilelistitems.h | 2 +- libretroshare/src/rsserver/rstypes.cc | 2 +- 7 files changed, 298 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8ea522b6f..f63f0df6a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -2,6 +2,7 @@ #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" +#include "filelist_io.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -53,7 +54,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -67,6 +68,8 @@ class InternalFileHierarchyStorage time_t dir_modtime ; // this accounts for deleted files, etc. time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + + time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; // class stuff @@ -246,6 +249,36 @@ class InternalFileHierarchyStorage return true; } + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + recurs_max_modf_TS = d.most_recent_time ; + local_update_TS = d.dir_update_time ; + + return true; + } + bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + d.most_recent_time = recurs_max_modf_TS ; + d.dir_update_time = local_update_TS ; + + return true; + } // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. @@ -340,9 +373,58 @@ class InternalFileHierarchyStorage return false; } - bool check() // checks consistency of storage. + bool check(std::string& error_string) const // checks consistency of storage. { - return true; + // recurs go through all entries, check that all + + std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. + hits[0] = 1 ; // because 0 is never the child of anyone + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + // stamp the kids + const DirEntry& de = *static_cast(mNodes[i]) ; + + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subdirs[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subdirs[j]] = 1; + } + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subfiles[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subfiles[j]] = 1; + } + + } + + for(uint32_t i=0;igetDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} +bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + locked_check() ; + return res ; } bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + locked_check() ; + return res ; } bool DirectoryStorage::removeDirectory(const EntryIndex& indx) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->removeDirectory(indx); + bool res = mFileHierarchy->removeDirectory(indx); + + locked_check(); + return res ; +} + +void DirectoryStorage::locked_check() +{ + std::string error ; + if(!mFileHierarchy->check(error)) + std::cerr << "Check error: " << error << std::endl; } bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) @@ -773,8 +879,95 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) /* find parent pointer, and row */ - std::cerr << "LocalDirectoryStorage::extractData(): indx=" << indx << " Returning this:" << std::endl; - std::cerr << d << std::endl; - return true; } + +bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + if(dir == NULL) + { + std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl; + return false; + } + + unsigned char *section_data = NULL; + uint32_t section_size = 0; + uint32_t section_offset = 0; + + // we need to send: + // - the name of the directory, its TS + // - the index entry for each subdir (the updte TS are exchanged at a higher level) + // - the file info for each subfile + // + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + + // serialise number of subdirs and number of subfiles + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subfiles.size() )) return false ; + + // serialise subdirs entry indexes + + for(uint32_t i=0;isubdirs.size();++i) + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subdirs[i] )) return false ; + + // serialise directory subfiles, with info for each of them + + for(uint32_t i=0;isubfiles.size();++i) + { + unsigned char *file_section_data = NULL ; + uint32_t file_section_offset = 0 ; + uint32_t file_section_size = 0; + + const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ; + + if(file == NULL) + { + std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl; + continue ; + } + + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subfiles[i] )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file->file_modtime)) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ; + + free(file_section_data) ; + } + + std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; + + return true ; +} + +/******************************************************************************************************************/ +/* Remote Directory Storage */ +/******************************************************************************************************************/ + +bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; + + NOT_IMPLEMENTED(); + + return true ; +} + + + + + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 309bff328..dd3353ad1 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -9,7 +9,9 @@ #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } +class RsTlvBinaryData ; class InternalFileHierarchyStorage ; +class RsTlvBinaryData ; class DirectoryStorage { @@ -26,8 +28,8 @@ class DirectoryStorage virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; - bool setUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; + bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -109,6 +111,9 @@ class DirectoryStorage void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + // debug + void locked_check(); + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. std::string mFileName; @@ -125,6 +130,17 @@ class RemoteDirectoryStorage: public DirectoryStorage public: RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} + + /*! + * \brief deserialiseDirEntry + * Loads a serialised directory content coming from a friend. The directory entry needs to exist already, + * as it is created when updating the parent. + * + * \param indx index of the directory to update + * \param bindata binary data to deserialise from + * \return false when the directory cannot be found. + */ + bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; }; class LocalDirectoryStorage: public DirectoryStorage @@ -150,6 +166,16 @@ public: virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; + /*! + * \brief serialiseDirEntry + * Produced a serialised directory content listing suitable for export to friends. + * + * \param indx index of the directory to serialise + * \param bindata binary data created by serialisation + * \return false when the directory cannot be found. + */ + bool serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) ; + private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 9c5818025..a2ee26916 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -20,6 +20,9 @@ static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; class FileListIO { diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 6fa89bd35..768ce370c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -113,7 +113,7 @@ int p3FileDatabase::tick() last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI -// RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } if(mUpdateFlags) @@ -129,7 +129,7 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } - if(mLastRemoteDirSweepTS + 5 < now) + if(mLastRemoteDirSweepTS + 30 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -227,6 +227,9 @@ void p3FileDatabase::cleanup() P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + if(mRemoteDirectories.size() <= friend_index) + mRemoteDirectories.resize(friend_index+1,NULL) ; + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; @@ -324,7 +327,7 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi // [ 10 bits | 22 bits ] // // This means that the whoel software has the following build-in limitation: - // * 1024 friends + // * 1023 friends // * 4M shared files. uint32_t fe = (uint32_t)e ; @@ -346,8 +349,21 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags { RS_STACK_MUTEX(mFLSMtx) ; + d.children.clear(); + // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) - // or at the top of our own list of shred directories, depending on the flags. + // or at the top of our own list of shared directories, depending on the flags. + + // Friend index is used as follows: + // 0 : own id + // 1...n : other friends + // + // entry_index: starts at 0. + // + // The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL. + // So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories. + // + // We should also check the consistency between flags and the content of ref. if (ref == NULL) { @@ -363,7 +379,6 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.age = 0; d.flags.clear() ; d.min_age = 0 ; - d.children.clear(); if(flags & RS_FILE_HINTS_LOCAL) { @@ -379,7 +394,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags else for(uint32_t i=0;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p); DirStub stub; stub.type = DIR_TYPE_PERSON; @@ -390,6 +405,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; + return true ; } @@ -398,9 +416,17 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags convertPointerToEntryIndex(ref,e,fi); + // check consistency + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + { + std::cerr << "remote request on local index or local request on remote index. This should not happen." << std::endl; + return false ; + } + DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + // Case where the index is the top of a single person. Can be us, or a friend. - bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; + bool res = storage->extractData(e,d); // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -410,22 +436,18 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags for(uint32_t i=0;iparentRow(e) ; + d.prow = storage->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - if(flags & RS_FILE_HINTS_LOCAL) - d.id = mOwnId ; - else - d.id = mRemoteDirectories[fi]->peerId(); + d.id = storage->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -670,7 +692,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -686,7 +708,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; - ritem->last_known_recurs_modf_TS = local_update_time ; + ritem->last_known_recurs_modf_TS = local_recurs_max_time ; } } } @@ -725,14 +747,14 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Uptating." << std::endl; + P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; - mRemoteDirectories[fi]->deserialiseUpdateEntry(item->entry_index,item->directory_content_data) ; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data) ; + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); #warning should notify the GUI here // notify the GUI if the hierarchy has changed @@ -750,7 +772,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS_remote_time,local_update_TS; - if(!rds->getUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; @@ -798,6 +820,18 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r locked_recursSweepRemoteDirectory(rds,*it); } +p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) +{ + uint64_t r = e ; + + // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + + for(uint32_t i=0;i::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; - std::cerr << " Children : " ; for(uint32_t i=0;i Date: Sat, 27 Aug 2016 13:22:29 +0200 Subject: [PATCH 33/92] fixed SEGV due to calling layout change in the wrong place --- libretroshare/src/file_sharing/p3filelists.cc | 11 +++++++++-- retroshare-gui/src/gui/RemoteDirModel.cpp | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 768ce370c..a1b98af65 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -443,7 +443,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags } else { - d.prow = storage->parentRow(e) ; + if(d.parent == 0) // child of root node + d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1); + else + d.prow = storage->parentRow(e) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } @@ -480,7 +484,10 @@ uint32_t p3FileDatabase::getType(void *ref) const if(e == 0) return DIR_TYPE_PERSON ; - return mRemoteDirectories[fi]->getEntryType(e) ; + if(fi == 0) + return mLocalSharedDirs->getEntryType(e) ; + else + return mRemoteDirectories[fi-1]->getEntryType(e) ; } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 295ebbf83..9ccbc43ad 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -862,6 +862,7 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const /* Callback from */ void RetroshareDirModel::preMods() { + emit layoutAboutToBeChanged(); #if QT_VERSION < 0x050000 reset(); #else @@ -877,7 +878,7 @@ void RetroshareDirModel::preMods() /* Callback from */ void RetroshareDirModel::postMods() { - emit layoutAboutToBeChanged(); +// emit layoutAboutToBeChanged(); #if QT_VERSION >= 0x040600 beginResetModel(); #endif From 07c7d192a94e812cfc3c478b6cea72998839603a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 14:09:26 +0200 Subject: [PATCH 34/92] added deserialisation and update of remote directories --- .../src/file_sharing/directory_storage.cc | 146 +++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f63f0df6a..228f1b208 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,4 +1,5 @@ #include +#include "serialiser/rstlvbinary.h" #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" @@ -24,6 +25,7 @@ template typename std::set::iterator erase_from_set(typename std::se class InternalFileHierarchyStorage { +public: class FileStorageNode { public: @@ -249,6 +251,65 @@ class InternalFileHierarchyStorage return true; } + + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; + return false; + } + DirEntry& d(*static_cast(mNodes[indx])) ; + + d.most_recent_time = most_recent_time; + d.dir_modtime = dir_modtime; + d.dir_update_time = time(NULL); + + d.subfiles = subfiles_array ; + d.subdirs = subdirs_array ; + + // check that all subdirs already exist. If not, create. + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subdirs_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subdirs_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new DirEntry(""); + + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + node->row = i ; + } + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subfiles_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subfiles_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new FileEntry("",0,0); + + node->row = subdirs_array.size()+i ; + } + + // we should also update the row of each subfile and each subdir + } + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) @@ -906,7 +967,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles @@ -958,11 +1019,90 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) { - RS_STACK_MUTEX(mDirStorageMtx) ; + const unsigned char *section_data = (unsigned char*)bindata.bin_data ; + uint32_t section_size = bindata.bin_len ; + uint32_t section_offset ; std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; - NOT_IMPLEMENTED(); + std::string dir_name ; + time_t most_recent_time ,dir_modtime ; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ; + + // serialise number of subdirs and number of subfiles + + uint32_t n_subdirs,n_subfiles ; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ; + + // serialise subdirs entry indexes + + std::vector subdirs_array ; + uint32_t subdir_index ; + + for(uint32_t i=0;i subfiles_array ; + std::vector subfiles_name ; + std::vector subfiles_size ; + std::vector subfiles_hash ; + std::vector subfiles_modtime ; + + for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) + { + std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; + return false ; + } + + // then update the subfiles + for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) + std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; return true ; } From eca9ca0e4dc0c102fb868a0b112c9dacb2b28ca3 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 16:38:15 +0200 Subject: [PATCH 35/92] fixed a few bugs in serialisation of shared dir content --- .../src/file_sharing/directory_storage.cc | 25 +++--- libretroshare/src/file_sharing/filelist_io.h | 6 +- libretroshare/src/file_sharing/p3filelists.cc | 85 ++++++++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 3 +- .../src/file_sharing/rsfilelistitems.cc | 2 +- .../src/file_sharing/rsfilelistitems.h | 2 +- 6 files changed, 89 insertions(+), 34 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 228f1b208..8c54d9e6f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -391,10 +391,8 @@ public: } uint32_t getType(DirectoryStorage::EntryIndex indx) const { - if(checkIndex(indx,FileStorageNode::TYPE_FILE)) - return FileStorageNode::TYPE_FILE; - else if(checkIndex(indx,FileStorageNode::TYPE_DIR)) - return FileStorageNode::TYPE_DIR; + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx]->type() ; else return FileStorageNode::TYPE_UNKNOWN; } @@ -751,17 +749,14 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.children.clear() ; time_t now = time(NULL) ; - std::cerr << "LocalDirectoryStorage::extractData(). Index=" << indx << std::endl; - - const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + uint32_t type = mFileHierarchy->getType(indx) ; d.ref = (void*)(intptr_t)indx ; - if (dir_entry != NULL) /* has children --- fill */ + if (type == InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR) /* has children --- fill */ { -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif + const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + /* extract all the entries */ for(DirectoryStorage::DirIterator it(this,indx);it;++it) @@ -798,12 +793,10 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.name = mPeerId.toStdString(); } } - else + else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE) { const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif + d.type = DIR_TYPE_FILE; d.count = file_entry->file_size; d.min_age = now - file_entry->file_modtime ; @@ -819,6 +812,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) else d.path = "" ; } + else + return false; d.flags.clear() ; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index a2ee26916..a984f2cd4 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -24,6 +24,8 @@ static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) + class FileListIO { public: @@ -68,9 +70,9 @@ private: static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) { - if(offset + S > buff_size) + if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size) { - buff = (unsigned char *)realloc(buff,offset + S) ; + buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ; buff_size = offset + S ; if(!buff) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index a1b98af65..218c6bed2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -21,6 +21,7 @@ static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. +static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every 55 secs, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") @@ -37,8 +38,18 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + + addSerialType(new RsFileListsSerialiser()) ; } +RsSerialiser *p3FileDatabase::setupSerialiser() +{ + RsSerialiser *rss = new RsSerialiser ; + rss->addSerialType(new RsFileListsSerialiser()) ; + rss->addSerialType(new RsGeneralConfigSerialiser()); + + return rss ; +} void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { RS_STACK_MUTEX(mFLSMtx) ; @@ -97,13 +108,22 @@ int p3FileDatabase::tick() tickRecv() ; tickSend() ; - // cleanup + time_t now = time(NULL) ; + + // cleanup // - remove/delete shared file lists for people who are not friend anymore // - - cleanup(); + + static time_t last_cleanup_time = 0; + +#warning we should use members here, not static + if(last_cleanup_time + 5 < now) + { + cleanup(); + last_cleanup_time = now ; + } static time_t last_print_time = 0; - time_t now = time(NULL) ; if(last_print_time + 20 < now) { @@ -133,8 +153,12 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; + std::set online_peers ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; + for(uint32_t i=0;iroot()) ; + if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; mLastRemoteDirSweepTS = now; } @@ -186,6 +210,8 @@ void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; + std::cerr << "p3FileDatabase::cleanup()" << std::endl; + // look through the list of friend directories. Remove those who are not our friends anymore. // std::set friend_set ; @@ -234,6 +260,29 @@ void p3FileDatabase::cleanup() mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } + + // cancel existing requests for which the peer is offline + + std::set online_peers ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; + + time_t now = time(NULL); + + for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) + if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) + { + std::cerr << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; + + std::map::iterator tmp(it); + ++tmp; + mPendingSyncRequests.erase(it) ; + it = tmp; + } + else + { + std::cerr << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; + ++it ; + } } } @@ -405,8 +454,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; +#ifdef DEBUG_FILE_HIERARCHY + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; +#endif return true ; } @@ -453,8 +504,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.id = storage->peerId(); +#ifdef DEBUG_FILE_HIERARCHY std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; +#endif return true; } @@ -801,13 +854,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r return ; } - P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - - DirSyncRequestData data ; - - data.request_TS = now ; - - mPendingSyncRequests[sync_req_id] = data ; + P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; item->entry_index = e ; @@ -816,7 +863,17 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; item->PeerId(rds->peerId()) ; - sendItem(item) ; + DirSyncRequestData data ; + + data.request_TS = now ; + data.peer_id = item->PeerId(); + data.flags = item->flags; + + std::cerr << "Pushing req in pending list with peer id " << data.peer_id << std::endl; + + mPendingSyncRequests[sync_req_id] = data ; + + sendItem(item) ; // at end! Because item is destroyed by the process. // Dont recurs into sub-directories, since we dont know yet were to go. diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index a15333ace..b06335458 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -129,7 +129,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // virtual bool loadList(std::list& items); virtual bool saveList(bool &cleanup, std::list&); - virtual RsSerialiser *setupSerialiser() { return NULL;} + virtual RsSerialiser *setupSerialiser() ; void cleanup(); void tickRecv(); @@ -177,6 +177,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // struct DirSyncRequestData { + RsPeerId peer_id ; time_t request_TS ; uint32_t flags ; }; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 8441148dd..a4d409d44 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -196,7 +196,7 @@ RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestI } RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size) { - bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM); uint32_t offset = 8; RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem(); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 24f8ab314..1ff00d57a 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -114,7 +114,7 @@ class RsFileListsSerialiser : public RsSerialType { public: - RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} + RsFileListsSerialiser() : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} virtual ~RsFileListsSerialiser() {} From fc5176a652142154f57bab9c26a1ea1849688fdd Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 23:56:23 +0200 Subject: [PATCH 36/92] fixed serialisation/deserialisation of dir content. Fixed exchange of dir data. First working prototype of file list exchange. --- .../src/file_sharing/directory_storage.cc | 34 ++++++++------ libretroshare/src/file_sharing/filelist_io.cc | 21 ++++++++- libretroshare/src/file_sharing/filelist_io.h | 25 +++++------ libretroshare/src/file_sharing/p3filelists.cc | 44 ++++++++++++++++--- 4 files changed, 88 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8c54d9e6f..20a093b2f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -248,6 +248,7 @@ public: fe.file_hash = hash; fe.file_size = size; fe.file_modtime = modf_time; + fe.file_name = fname; return true; } @@ -264,6 +265,7 @@ public: d.most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); + d.dir_name = dir_name; d.subfiles = subfiles_array ; d.subdirs = subdirs_array ; @@ -287,6 +289,7 @@ public: ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; node->row = i ; + node->parent_index = indx ; } for(uint32_t i=0;irow = subdirs_array.size()+i ; + node->parent_index = indx ; } - // we should also update the row of each subfile and each subdir + return true; } bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) @@ -531,7 +535,8 @@ private: } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; } } @@ -961,18 +966,18 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subdirs.size() )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subfiles.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; // serialise subdirs entry indexes for(uint32_t i=0;isubdirs.size();++i) - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subdirs[i] )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subdirs[i] )) return false ; // serialise directory subfiles, with info for each of them @@ -990,11 +995,11 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary continue ; } - if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subfiles[i] )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subfiles[i] )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; - if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file->file_modtime)) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) return false ; // now write the whole string into a single section in the file @@ -1005,6 +1010,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; + bindata.bin_data = section_data ; + bindata.bin_len = section_offset ; + return true ; } @@ -1016,12 +1024,12 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs { const unsigned char *section_data = (unsigned char*)bindata.bin_data ; uint32_t section_size = bindata.bin_len ; - uint32_t section_offset ; + uint32_t section_offset=0 ; std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; std::string dir_name ; - time_t most_recent_time ,dir_modtime ; + uint32_t most_recent_time ,dir_modtime ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; @@ -1068,7 +1076,7 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs std::string entry_name ; uint64_t entry_size ; RsFileHash entry_hash ; - time_t entry_modtime ; + uint32_t entry_modtime ; if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,entry_index )) return false ; if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,entry_name )) return false ; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 3cd153ae8..770478c37 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,11 +1,28 @@ +#include "retroshare/rsids.h" +#include "serialiser/rsbaseserial.h" #include "filelist_io.h" +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const size_t & val) { return setRawUInt64(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; } + +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,size_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast(buff),size,&offset,val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast(buff),size,offset) ; } + +template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; } +template<> uint32_t FileListIO::serial_size(const size_t & ) { return 8 ; } +template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; } +template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); } + bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) { if(!checkSectionSize(buff,buff_size,offset,size)) return false; - if(!writeSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + if(!writeSectionHeader(buff,buff_size,offset,section_tag,size)) return false; memcpy(&buff[offset],val,size) ; @@ -16,7 +33,7 @@ bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32 bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) { - if(!readSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + if(!readSectionHeader(buff,buff_size,offset,check_section_tag,size)) return false; val = (unsigned char *)rs_malloc(size) ; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index a984f2cd4..b22a57cc4 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -32,16 +32,15 @@ public: template static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val) { - if(!checkSectionSize(buff,buff_size,offset,sizeof(T))) + uint32_t s = serial_size(val) ; + + if(!checkSectionSize(buff,buff_size,offset,s)) return false; - if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T))) + if(!writeSectionHeader(buff,buff_size,offset,section_tag,s)) return false; - memcpy(&buff[offset],reinterpret_cast(&val),sizeof(T)) ; - offset += sizeof(T) ; - - return true; + return serialise(buff,buff_size,offset,val) ; } template @@ -52,18 +51,16 @@ public: if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size)) return false; - if(section_size != sizeof(T)) - return false ; - - memcpy(reinterpret_cast(&val),&buff[offset],sizeof(T)) ; - offset += sizeof(T) ; - - return true; + return deserialise(buff,buff_size,offset,val); } static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ; + template static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ; + template static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ; + template static uint32_t serial_size(const T& val) ; + private: static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; @@ -73,7 +70,7 @@ private: if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size) { buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ; - buff_size = offset + S ; + buff_size = offset + S + SECTION_HEADER_MAX_SIZE; if(!buff) return false ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 218c6bed2..403233183 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -148,8 +148,8 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } - - if(mLastRemoteDirSweepTS + 30 < now) +#warning we need to make sure that one req per directory will not cause to keep re-asking the top level dirs. + if(mLastRemoteDirSweepTS + 5 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -158,7 +158,12 @@ int p3FileDatabase::tick() for(uint32_t i=0;ipeerId()) != online_peers.end()) + { + std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; + mRemoteDirectories[i]->print(); + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; + } mLastRemoteDirSweepTS = now; } @@ -780,15 +785,33 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { + P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + + // remove the original request from pending list + + { + RS_STACK_MUTEX(mFLSMtx) ; + + std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; + + if(it == mPendingSyncRequests.end()) + { + std::cerr << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; + return ; + } + mPendingSyncRequests.erase(it) ; + } + // find the correct friend entry uint32_t fi = 0 ; - P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; { RS_STACK_MUTEX(mFLSMtx) ; fi = locked_getFriendIndex(item->PeerId()); + std::cerr << " friend index is " << fi ; + // make sure we have a remote directory for that friend. if(mRemoteDirectories.size() <= fi) @@ -813,11 +836,18 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; - mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data) ; - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) + { + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); -#warning should notify the GUI here - // notify the GUI if the hierarchy has changed + // notify the GUI if the hierarchy has changed + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + } + else + std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; + + std::cerr << " new content after update: " << std::endl; + mRemoteDirectories[fi]->print(); } } From 494d2071c95a4290a91ce6fc2efdc933b9593667 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 28 Aug 2016 18:42:52 +0200 Subject: [PATCH 37/92] fixed update of directory TS, also fixing update of empty dirs at friends --- .../src/file_sharing/directory_storage.cc | 48 +++++++++++++------ .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 10 ++-- libretroshare/src/file_sharing/p3filelists.cc | 9 +--- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 20a093b2f..e801b4ba8 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -47,10 +47,10 @@ public: virtual ~FileEntry() {} // local stuff + time_t file_modtime; std::string file_name ; uint64_t file_size ; RsFileHash file_hash ; - time_t file_modtime ; }; class DirEntry: public FileStorageNode @@ -68,18 +68,21 @@ public: std::vector subdirs ; std::vector subfiles ; - time_t dir_modtime ; // this accounts for deleted files, etc. + time_t dir_modtime; time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. - time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; // class stuff InternalFileHierarchyStorage() : mRoot(0) { - mNodes.push_back(new DirEntry("")) ; - mNodes.back()->row=0; - mNodes.back()->parent_index=0; + DirEntry *de = new DirEntry("") ; + + de->row=0; + de->parent_index=0; + de->dir_modtime=0; + + mNodes.push_back(de) ; } int parentRow(DirectoryStorage::EntryIndex e) @@ -97,14 +100,14 @@ public: return e < mNodes.size() && mNodes[e] != NULL ; } - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; DirEntry& d(*static_cast(mNodes[indx])) ; - std::set should_create(subdirs); + std::map should_create(subdirs); for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) @@ -122,14 +125,18 @@ public: ++i; } - for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) + for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) { - std::cerr << "[directory storage] adding new subdirectory " << *it << " at index " << mNodes.size() << std::endl; + std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; + + DirEntry *de = new DirEntry(it->first) ; + + de->row = mNodes.size(); + de->parent_index = indx; + de->dir_modtime = it->second; d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(new DirEntry(*it)); - mNodes.back()->row = mNodes.size()-1; - mNodes.back()->parent_index = indx; + mNodes.push_back(de) ; } return true; @@ -664,7 +671,7 @@ bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_T return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; } -bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; @@ -1035,6 +1042,10 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ; + std::cerr << " dir name : \"" << dir_name << "\"" << std::endl; + std::cerr << " most recent time : " << most_recent_time << std::endl; + std::cerr << " modification time : " << dir_modtime << std::endl; + // serialise number of subdirs and number of subfiles uint32_t n_subdirs,n_subfiles ; @@ -1042,6 +1053,9 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ; + std::cerr << " number of subdirs : " << n_subdirs << std::endl; + std::cerr << " number of files : " << n_subfiles << std::endl; + // serialise subdirs entry indexes std::vector subdirs_array ; @@ -1095,6 +1109,8 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs RS_STACK_MUTEX(mDirStorageMtx) ; + std::cerr << " updating dir entry..." << std::endl; + // first create the entries for each subdir and each subfile. if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) { @@ -1104,8 +1120,12 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs // then update the subfiles for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; + } return true ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index dd3353ad1..c86477d19 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -94,7 +94,7 @@ class DirectoryStorage const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; - bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index f9fada262..2af4c036f 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -59,10 +59,10 @@ void LocalDirectoryUpdater::sweepSharedDirectories() std::list shared_directory_list ; mSharedDirectories->getSharedDirectoryList(shared_directory_list); - std::set sub_dir_list ; + std::map sub_dir_list ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) - sub_dir_list.insert( (*real_dir_it).filename ) ; + sub_dir_list[(*real_dir_it).filename] = 0 ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. @@ -93,7 +93,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles std::map subfiles ; - std::set subdirs ; + std::map subdirs ; for(;dirIt.isValid();dirIt.next()) { @@ -104,7 +104,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; default: @@ -134,7 +134,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; - for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::map::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) { std::cerr << " recursing into " << stored_dir_it.name() << std::endl; recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 403233183..ea6fce421 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -834,15 +834,10 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; + P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) - { - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); - - // notify the GUI if the hierarchy has changed - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - } + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; From 7c2ed3fca04f75ca092b2a3f4057a505cbfe2117 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 28 Aug 2016 23:14:39 +0200 Subject: [PATCH 38/92] save/restore expanded state in file list tree view so that view gets updated seamlessly --- libretroshare/src/file_sharing/p3filelists.cc | 6 ++ retroshare-gui/src/gui/RemoteDirModel.cpp | 73 ++++++++--------- retroshare-gui/src/gui/RemoteDirModel.h | 3 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 78 +++++++++++++++++-- retroshare-gui/src/gui/SharedFilesDialog.h | 7 +- 5 files changed, 118 insertions(+), 49 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ea6fce421..a3ec08e4d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -129,7 +129,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); +#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI @@ -160,7 +162,9 @@ int p3FileDatabase::tick() if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) { std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; +#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[i]->print(); +#endif locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; } @@ -842,7 +846,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; std::cerr << " new content after update: " << std::endl; +#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[fi]->print(); +#endif } } diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 9ccbc43ad..1c18c5062 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -744,6 +744,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent /* we can just grab the reference now */ +#ifdef RDM_DEBUG + std::cerr << "Creating index 1 row=" << row << ", column=" << column << ", ref=" << (void*)details.children[row].ref << std::endl; +#endif return createIndex(row, column, details.children[row].ref); } QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const @@ -761,7 +764,10 @@ QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent { void *ref = _ref_entries[row].first ; - return createIndex(row, column, ref); +#ifdef RDM_DEBUG + std::cerr << "Creating index 2 row=" << row << ", column=" << column << ", ref=" << (void*)ref << std::endl; +#endif + return createIndex(row, column, ref); } else return QModelIndex(); @@ -810,6 +816,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const std::cerr << "success index(" << details->prow << ",0," << details->parent << ")"; std::cerr << std::endl; + std::cerr << "Creating index 3 row=" << details.prow << ", column=" << 0 << ", ref=" << (void*)details.parent << std::endl; #endif return createIndex(details.prow, 0, details.parent); } @@ -1155,11 +1162,29 @@ void RetroshareDirModel::openSelected(const QModelIndexList &qmil) #endif } +void RetroshareDirModel::getFilePath(const QModelIndex& index, std::string& fullpath) +{ + void *ref = index.sibling(index.row(),1).internalPointer(); + + DirDetails details ; + + if (!requestDirDetails(ref, false,details) ) + { +#ifdef RDM_DEBUG + std::cerr << "getFilePaths() Bad Request" << std::endl; +#endif + return; + } + + fullpath = details.path + "/" + details.name; +} + void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list &fullpaths) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::getFilePaths()" << std::endl; #endif +#warning make sure we atually output something here if (RemoteMode) { #ifdef RDM_DEBUG @@ -1168,48 +1193,18 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list internalPointer(); + std::string path ; - DirDetails details ; - - if (!requestDirDetails(ref, false,details) ) - { + getFilePath(*it,path) ; #ifdef RDM_DEBUG - std::cerr << "getFilePaths() Bad Request" << std::endl; + std::cerr << "Constructed FilePath: " << path << std::endl; #endif - continue; - } - - if (details.type != DIR_TYPE_FILE) - { -#ifdef RDM_DEBUG - std::cerr << "getFilePaths() Not File" << std::endl; -#endif - continue; /* not file! */ - } - -#ifdef RDM_DEBUG - std::cerr << "::::::::::::File Details:::: " << std::endl; - std::cerr << "Name: " << details.name << std::endl; - std::cerr << "Hash: " << details.hash << std::endl; - std::cerr << "Size: " << details.count << std::endl; - std::cerr << "Path: " << details.path << std::endl; -#endif - - std::string filepath = details.path + "/"; - filepath += details.name; - -#ifdef RDM_DEBUG - std::cerr << "Constructed FilePath: " << filepath << std::endl; -#endif - if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), filepath)) - { - fullpaths.push_back(filepath); - } - } +#warning TERRIBLE COST here. Use a std::set! + if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), path)) + fullpaths.push_back(path); + } #ifdef RDM_DEBUG std::cerr << "::::::::::::Done getFilePaths" << std::endl; #endif diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index dd81e2c84..6028b5785 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -72,7 +72,8 @@ class RetroshareDirModel : public QAbstractItemModel void getFileInfoFromIndexList(const QModelIndexList& list, std::list& files_info) ; void openSelected(const QModelIndexList &list); void getFilePaths(const QModelIndexList &list, std::list &fullpaths); - void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } + void getFilePath(const QModelIndex& index, std::string& fullpath); + void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 67a5e3d0e..a74cfe690 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -19,6 +19,8 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ +#include + #include #include #include @@ -327,8 +329,8 @@ void RemoteSharedFilesDialog::processSettings(bool bLoad) void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) { - //disconnect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), NULL, NULL ); - //disconnect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), NULL, NULL ); +// disconnect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), NULL, NULL ); +// disconnect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), NULL, NULL ); if(model!=NULL) model->setVisible(false) ; @@ -352,8 +354,8 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) model->update() ; } - //connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), model, SLOT( collapsed(const QModelIndex & ) ) ); - //connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), model, SLOT( expanded(const QModelIndex & ) ) ); +// connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), this, SLOT( collapsed(const QModelIndex & ) ) ); +// connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); @@ -363,6 +365,7 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) ui.dirTreeView->header()->headerDataChanged(Qt::Horizontal, COLUMN_NAME, COLUMN_DIR) ; +// recursRestoreExpandedItems(ui.dirTreeView->rootIndex(),expanded_indexes); FilterItems(); } @@ -823,20 +826,79 @@ void SharedFilesDialog::preModDirectories(bool local) flat_model->preMods(); } +void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexes) +{ + for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) + { + std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); + recursSaveExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + } +} + +void SharedFilesDialog::restoreExpandedPaths(const std::set& expanded_indexes) +{ + std::cerr << "Restoring expanded items. " << std::endl; + for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) + { + std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); + recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + } +} + +void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) +{ + std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); + std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; + + if(ui.dirTreeView->isExpanded(index)) + { + std::cerr << "Index " << local_path << " is expanded." << std::endl; + if(index.isValid()) + exp.insert(local_path) ; + + for(int row=0;rowmodel()->rowCount(index);++row) + recursSaveExpandedItems(index.child(row,0),local_path,exp) ; + } + else + std::cerr << "Index is not expanded." << std::endl; +} + +void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp) +{ + std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); + std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; + + if(exp.find(local_path) != exp.end()) + { + std::cerr << "re expanding index " << local_path << std::endl; + ui.dirTreeView->setExpanded(index,true) ; + + for(int row=0;rowmodel()->rowCount(index);++row) + recursRestoreExpandedItems(index.child(row,0),local_path,exp) ; + } +} + + void SharedFilesDialog::postModDirectories(bool local) { if (isRemote() == local) { - return; - } + return; + } + std::set expanded_indexes; + saveExpandedPaths(expanded_indexes) ; + std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; - /* Notify both models, only one is visible */ + /* Notify both models, only one is visible */ tree_model->postMods(); flat_model->postMods(); ui.dirTreeView->update() ; - if (ui.filterPatternLineEdit->text().isEmpty() == false) + restoreExpandedPaths(expanded_indexes) ; + + if (ui.filterPatternLineEdit->text().isEmpty() == false) FilterItems(); + std::cerr << "****** updated directories! ******" << std::endl; QCoreApplication::flush(); } diff --git a/retroshare-gui/src/gui/SharedFilesDialog.h b/retroshare-gui/src/gui/SharedFilesDialog.h index c0258c190..9b7387224 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.h +++ b/retroshare-gui/src/gui/SharedFilesDialog.h @@ -22,6 +22,7 @@ #ifndef _SHAREDFILESDIALOG_H #define _SHAREDFILESDIALOG_H +#include #include "RsAutoUpdatePage.h" #include "ui_SharedFilesDialog.h" @@ -51,7 +52,6 @@ protected slots: virtual void spawnCustomPopupMenu(QPoint point) = 0; private slots: - /* For handling the model updates */ void preModDirectories(bool local) ; void postModDirectories(bool local) ; @@ -94,6 +94,11 @@ protected: Ui::SharedFilesDialog ui; virtual void processSettings(bool bLoad) = 0; + void recursRestoreExpandedItems(const QModelIndex& index,const std::string& path,const std::set& exp); + void recursSaveExpandedItems(const QModelIndex& index, const std::string &path, std::set &exp); + void saveExpandedPaths(std::set& paths) ; + void restoreExpandedPaths(const std::set& paths) ; + protected: //now context menu are created again every time theu are called ( in some //slots.. Maybe it's not good... From 1290aa84031ec1e8b2a55815adaaa05e746f14ef Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 29 Aug 2016 21:30:56 +0200 Subject: [PATCH 39/92] fixed proper display of file sharing flags --- .../src/file_sharing/directory_storage.cc | 37 +++++++++++++++++-- libretroshare/src/file_sharing/hash_cache.cc | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 10 +++-- retroshare-gui/src/gui/SharedFilesDialog.cpp | 30 ++++++++++++--- 4 files changed, 65 insertions(+), 14 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e801b4ba8..f5b90f8d6 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -207,8 +207,7 @@ public: f.file_modtime = it->second.modtime; f.file_size = it->second.size; } - else - new_files.erase(f.file_name) ; + new_files.erase(f.file_name) ; ++i; } @@ -881,7 +880,7 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; - std::map::iterator it = mLocalDirs.find(info.virtualname) ; + std::map::iterator it = mLocalDirs.find(info.filename) ; if(it == mLocalDirs.end()) { @@ -944,8 +943,38 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) // here we should update the file sharing flags d.flags.clear() ; + d.parent_groups.clear(); - /* find parent pointer, and row */ + std::string base_dir; + + for(DirectoryStorage::EntryIndex i=((d.type==DIR_TYPE_FILE)?((intptr_t)d.parent):((intptr_t)d.ref));;) + { + const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ; + + if(e == NULL) + break ; + + if(e->parent_index == 0) + { + base_dir = e->dir_name ; + break ; + } + i = e->parent_index ; + } + + if(!base_dir.empty()) + { + std::map::const_iterator it = mLocalDirs.find(base_dir) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; + return false ; + } +#warning we should use a NodeGroupId here + d.flags = it->second.shareflags; + d.parent_groups = it->second.parent_groups; + } return true; } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 1e7cf0fe7..d8e65f810 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -121,7 +121,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t return true ; } #ifdef HASHSTORAGE_DEBUG - std::cerr << "Not in cache. Sceduling for re-hash." << std::endl ; + std::cerr << "Not in cache. Scheduling for re-hash." << std::endl ; #endif // we need to schedule a re-hashing diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index a3ec08e4d..e164a87d0 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -64,8 +64,12 @@ void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) } void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) { - RS_STACK_MUTEX(mFLSMtx) ; - mLocalSharedDirs->updateShareFlags(info) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->updateShareFlags(info) ; + } + + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } p3FileDatabase::~p3FileDatabase() @@ -135,7 +139,7 @@ int p3FileDatabase::tick() last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } if(mUpdateFlags) diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index a74cfe690..cc578cb8d 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -250,9 +250,14 @@ void SharedFilesDialog::showEvent(QShowEvent *) { if(model!=NULL) { - model->setVisible(true) ; + std::set expanded_indexes ; + saveExpandedPaths(expanded_indexes); + + model->setVisible(true) ; model->update() ; - } + + restoreExpandedPaths(expanded_indexes); + } } RemoteSharedFilesDialog::~RemoteSharedFilesDialog() { @@ -348,9 +353,13 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) showProperColumns() ; - if(isVisible()) + std::set expanded_indexes ; + saveExpandedPaths(expanded_indexes); + + if(isVisible()) { - model->setVisible(true) ; + model->setVisible(true) ; + model->update() ; } @@ -360,7 +369,9 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); - QHeaderView * header = ui.dirTreeView->header () ; + restoreExpandedPaths(expanded_indexes); + + QHeaderView * header = ui.dirTreeView->header () ; QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Interactive); ui.dirTreeView->header()->headerDataChanged(Qt::Horizontal, COLUMN_NAME, COLUMN_DIR) ; @@ -828,6 +839,10 @@ void SharedFilesDialog::preModDirectories(bool local) void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexes) { + if(ui.dirTreeView->model() == NULL) + return ; + + std::cerr << "Saving expanded items. " << std::endl; for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -837,6 +852,9 @@ void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexe void SharedFilesDialog::restoreExpandedPaths(const std::set& expanded_indexes) { + if(ui.dirTreeView->model() == NULL) + return ; + std::cerr << "Restoring expanded items. " << std::endl; for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { @@ -886,7 +904,7 @@ void SharedFilesDialog::postModDirectories(bool local) } std::set expanded_indexes; saveExpandedPaths(expanded_indexes) ; - std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; + std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; /* Notify both models, only one is visible */ tree_model->postMods(); From 4b00d47d3a5e31cf2e7b41b0ee67e751c08d8f3d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 30 Aug 2016 21:28:32 +0200 Subject: [PATCH 40/92] added permission check for friends to see shared files or not --- .../src/file_sharing/directory_storage.cc | 32 +++++++++++++++---- .../src/file_sharing/directory_storage.h | 12 +++++++ libretroshare/src/file_sharing/p3filelists.cc | 11 +++++-- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f5b90f8d6..60662e701 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -385,6 +385,14 @@ public: // Low level stuff. Should normally not be used externally. + const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const + { + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx] ; + else + return NULL ; + } + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) @@ -450,7 +458,7 @@ public: hits[0] = 1 ; // because 0 is never the child of anyone for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR) { // stamp the kids const DirEntry& de = *static_cast(mNodes[i]) ; @@ -942,12 +950,24 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) // here we should update the file sharing flags - d.flags.clear() ; - d.parent_groups.clear(); + return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; +} + +bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + flags.clear() ; + parent_groups.clear(); std::string base_dir; - for(DirectoryStorage::EntryIndex i=((d.type==DIR_TYPE_FILE)?((intptr_t)d.parent):((intptr_t)d.ref));;) + const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ; + + if(n == NULL) + return false ; + + for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;) { const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ; @@ -972,8 +992,8 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return false ; } #warning we should use a NodeGroupId here - d.flags = it->second.shareflags; - d.parent_groups = it->second.parent_groups; + flags = it->second.shareflags; + parent_groups = it->second.parent_groups; } return true; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index c86477d19..4f5a48f38 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -164,6 +164,18 @@ public: */ bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; + /*! + * \brief getFileSharingPermissions + * Computes the flags and parent groups for any index. + * \param indx index of the entry to compute the flags for + * \param flags computed flags + * \param parent_groups computed parent groups + * \return + * false if the index is not valid + * false otherwise + */ + bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; /*! diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index e164a87d0..0d634854d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -756,10 +756,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->request_id = item->request_id; ritem->entry_index = item->entry_index ; + std::list node_groups; + FileStorageFlags node_flags; + if(entry_type != DIR_TYPE_DIR) { - P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory. Answering with proper flags." << std::endl; - + P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; + } + else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + { + std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else From 6a30e44756933031b7b39f8d410c234d00c8492a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 30 Aug 2016 22:30:58 +0200 Subject: [PATCH 41/92] fixed bug in removal of directory --- .../src/file_sharing/directory_storage.cc | 26 ++++++++++++++++--- libretroshare/src/file_sharing/p3filelists.cc | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 60662e701..518546c9d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -141,7 +141,7 @@ public: return true; } - bool removeDirectory(const DirectoryStorage::EntryIndex& indx) + bool removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! { // check that it's a directory @@ -151,6 +151,10 @@ public: if(indx == 0) return nodeAccessError("checkIndex(): Cannot remove top level directory") ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "(--) Removing directory " << indx << std::endl; + print(); +#endif // remove from parent DirEntry& d(*static_cast(mNodes[indx])) ; @@ -158,8 +162,19 @@ public: for(uint32_t i=0;ientry_index << " because it does not exist." << std::endl; mRemoteDirectories[fi]->removeDirectory(item->entry_index); + + mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { From 1f3ea6fa0fec6d5078525e52e3a23d1d37d5dddf Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 31 Aug 2016 22:47:05 +0200 Subject: [PATCH 42/92] added selection of subdirs to be seen for friends depending on permissions. Enabled dynamic update at friends --- .../src/file_sharing/directory_storage.cc | 58 +++++++++++++++++-- .../src/file_sharing/directory_storage.h | 10 ++-- libretroshare/src/file_sharing/p3filelists.cc | 3 +- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 518546c9d..a3e0f15d7 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,6 @@ #include #include "serialiser/rstlvbinary.h" +#include "retroshare/rspeers.h" #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" @@ -99,6 +100,14 @@ public: { return e < mNodes.size() && mNodes[e] != NULL ; } + bool stampDirectory(const DirectoryStorage::EntryIndex& indx) + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + static_cast(mNodes[indx])->dir_modtime = time(NULL) ; + return true; + } bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { @@ -904,6 +913,18 @@ void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst lst.push_back(it->second) ; } +static bool sameLists(const std::list& l1,const std::list& l2) +{ + std::list::const_iterator it1(l1.begin()) ; + std::list::const_iterator it2(l2.begin()) ; + + for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2) + if(*it1 != *it2) + return false ; + + return it1 == l1.end() && it2 == l2.end() ; +} + void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -915,7 +936,17 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; return ; } - it->second = info; + + // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just + // want to make sure that if they are different, flags get updated. + + if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) + { + it->second = info; + mFileHierarchy->stampDirectory(0) ; + + std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; + } } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) @@ -976,7 +1007,11 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; + return locked_getFileSharingPermissions(indx,flags,parent_groups) ; +} +bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +{ flags.clear() ; parent_groups.clear(); @@ -1019,7 +1054,7 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil return true; } -bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) +bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -1031,6 +1066,17 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary return false; } + // compute list of allowed subdirs + std::vector allowed_subdirs ; + FileStorageFlags node_flags ; + std::list node_groups ; + + // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter + + for(uint32_t i=0;isubdirs.size();++i) + if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + allowed_subdirs.push_back(dir->subdirs[i]) ; + unsigned char *section_data = NULL; uint32_t section_size = 0; uint32_t section_offset = 0; @@ -1047,13 +1093,13 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // serialise number of subdirs and number of subfiles - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subdirs.size() )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; // serialise subdirs entry indexes - for(uint32_t i=0;isubdirs.size();++i) - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subdirs[i] )) return false ; + for(uint32_t i=0;i& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 09049b5d0..8c948fb94 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -781,7 +781,8 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; ritem->last_known_recurs_modf_TS = local_recurs_max_time; - mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data) ; + // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer. + mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ; } else { From f191fe675d756e8bee50a22e3d4c4ce2e048bed5 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 1 Sep 2016 20:02:47 +0200 Subject: [PATCH 43/92] fixed compilation with new RsNodeGroupId --- .../src/file_sharing/directory_storage.cc | 14 +++++++------- libretroshare/src/file_sharing/directory_storage.h | 4 ++-- libretroshare/src/file_sharing/p3filelists.cc | 2 +- libretroshare/src/retroshare/rstypes.h | 4 ++-- libretroshare/src/rsserver/rstypes.cc | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index a3e0f15d7..b490038f8 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -913,10 +913,10 @@ void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst lst.push_back(it->second) ; } -static bool sameLists(const std::list& l1,const std::list& l2) +static bool sameLists(const std::list& l1,const std::list& l2) { - std::list::const_iterator it1(l1.begin()) ; - std::list::const_iterator it2(l2.begin()) ; + std::list::const_iterator it1(l1.begin()) ; + std::list::const_iterator it2(l2.begin()) ; for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2) if(*it1 != *it2) @@ -1004,13 +1004,13 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; } -bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; return locked_getFileSharingPermissions(indx,flags,parent_groups) ; } -bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list &parent_groups) { flags.clear() ; parent_groups.clear(); @@ -1046,7 +1046,7 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; return false ; } -#warning we should use a NodeGroupId here + flags = it->second.shareflags; parent_groups = it->second.parent_groups; } @@ -1069,7 +1069,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // compute list of allowed subdirs std::vector allowed_subdirs ; FileStorageFlags node_flags ; - std::list node_groups ; + std::list node_groups ; // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index b82696b2c..0f5311944 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -174,7 +174,7 @@ public: * false if the index is not valid * false otherwise */ - bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list &parent_groups); virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; @@ -190,7 +190,7 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: - bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 8c948fb94..21beec912 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -756,7 +756,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->request_id = item->request_id; ritem->entry_index = item->entry_index ; - std::list node_groups; + std::list node_groups; FileStorageFlags node_flags; if(entry_type != DIR_TYPE_DIR) diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 56a9af280..8ca770a76 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -236,13 +236,13 @@ public: RsPeerId id; std::string name; RsFileHash hash; - std::string path; + std::string path; uint64_t count; uint32_t age; FileStorageFlags flags; uint32_t min_age ; // minimum age of files in this subtree - std::list children; + std::vector children; std::list parent_groups; // parent groups for the shared directory }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index f9b7a8471..4254ee258 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -59,7 +59,7 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) std::cerr << " Age : " << d.age << std::endl; std::cerr << " Min age : " << d.min_age << std::endl; std::cerr << " Flags : " << d.flags << std::endl; - std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; + std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; std::cerr << " Children : " ; for(uint32_t i=0;i Date: Thu, 1 Sep 2016 21:04:48 +0200 Subject: [PATCH 44/92] added load/save of configuration and shared dirs list --- .../src/file_sharing/directory_updater.cc | 7 +- .../src/file_sharing/directory_updater.h | 5 + .../src/file_sharing/file_sharing_defaults.h | 9 + libretroshare/src/file_sharing/hash_cache.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 160 ++++++++++++++++-- libretroshare/src/libretroshare.pro | 3 +- 6 files changed, 163 insertions(+), 23 deletions(-) create mode 100644 libretroshare/src/file_sharing/file_sharing_defaults.h diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 2af4c036f..adc3e8fff 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -3,6 +3,7 @@ #include "directory_storage.h" #include "directory_updater.h" +#include "file_sharing_defaults.h" #define DEBUG_LOCAL_DIR_UPDATER 1 @@ -10,15 +11,13 @@ // Local Directory Updater // //=============================================================================================================// -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! - LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { mLastSweepTime = 0; mLastTSUpdateTime = 0; + + mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES; } void LocalDirectoryUpdater::data_tick() diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 94b9a2080..63d6ec260 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -23,6 +23,9 @@ public: virtual void forceUpdate(); + void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } + uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } + protected: virtual void data_tick() ; @@ -36,5 +39,7 @@ private: time_t mLastSweepTime; time_t mLastTSUpdateTime; + + uint32_t mDelayBetweenDirectoryUpdates; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h new file mode 100644 index 000000000..a36e311b0 --- /dev/null +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -0,0 +1,9 @@ +#pragma once + +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! + +static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time +static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6b2dbf8f3..39d9767fe 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -44,7 +44,7 @@ public: // interaction with GUI, called from p3FileLists void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } - void clear() { mFiles.clear(); } + void clear() { mFiles.clear(); mChanged=true; } bool empty() const { return mFiles.empty() ; } // Functions called by the thread diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 21beec912..597762958 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -4,6 +4,7 @@ #include "file_sharing/directory_storage.h" #include "file_sharing/directory_updater.h" #include "file_sharing/rsfilelistitems.h" +#include "file_sharing/file_sharing_defaults.h" #include "retroshare/rsids.h" #include "retroshare/rspeers.h" @@ -20,7 +21,6 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every 55 secs, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) @@ -39,14 +39,19 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + // This is for the transmission of data + addSerialType(new RsFileListsSerialiser()) ; } RsSerialiser *p3FileDatabase::setupSerialiser() { + // This one is for saveList/loadList + RsSerialiser *rss = new RsSerialiser ; rss->addSerialType(new RsFileListsSerialiser()) ; rss->addSerialType(new RsGeneralConfigSerialiser()); + rss->addSerialType(new RsFileConfigSerialiser()); return rss ; } @@ -201,21 +206,137 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } -bool p3FileDatabase::loadList(std::list& items) + +bool p3FileDatabase::saveList(bool &cleanup, std::list& sList) { - // This loads - // - // - list of locally shared directories, and the permissions that go with them +cleanup = true; - NOT_IMPLEMENTED(); +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "Save list" << std::endl; +#endif - return true ; + /* get list of directories */ + std::list dirList; + { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->getSharedDirectoryList(dirList); + } + + for(std::list::iterator it = dirList.begin(); it != dirList.end(); ++it) + { + RsFileConfigItem *fi = new RsFileConfigItem(); + + fi->file.path = (*it).filename ; + fi->file.name = (*it).virtualname ; + fi->flags = (*it).shareflags.toUInt32() ; + + for(std::list::const_iterator it2( (*it).parent_groups.begin());it2!=(*it).parent_groups.end();++it2) + fi->parent_groups.ids.insert(*it2) ; + + sList.push_back(fi); + } + + RsConfigKeyValueSet *rskv = new RsConfigKeyValueSet(); + + /* basic control parameters */ + { + RS_STACK_MUTEX(mFLSMtx) ; + std::string s ; + rs_sprintf(s, "%lu", mHashCache->rememberHashFilesDuration()) ; + + RsTlvKeyValue kv; + + kv.key = HASH_CACHE_DURATION_SS; + kv.value = s ; + + rskv->tlvkvs.pairs.push_back(kv); + } + + { + std::string s ; + rs_sprintf(s, "%d", watchPeriod()) ; + + RsTlvKeyValue kv; + + kv.key = WATCH_FILE_DURATION_SS; + kv.value = s ; + + rskv->tlvkvs.pairs.push_back(kv); + } + + /* Add KeyValue to saveList */ + sList.push_back(rskv); + + return true; } -bool p3FileDatabase::saveList(bool &cleanup, std::list&) +bool p3FileDatabase::loadList(std::list& load) { - NOT_IMPLEMENTED(); - return true ; + /* for each item, check it exists .... + * - remove any that are dead (or flag?) + */ + static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_BROWSABLE_OTHERS | DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_GROUPS | DIR_FLAGS_NETWORK_WIDE_GROUPS ; + +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "Load list" << std::endl; +#endif + + std::list dirList; + + for(std::list::iterator it = load.begin(); it != load.end(); ++it) + { + RsConfigKeyValueSet *rskv ; + + if (NULL != (rskv = dynamic_cast(*it))) + { + /* make into map */ + std::map configMap; + std::map::const_iterator mit ; + + for(std::list::const_iterator kit = rskv->tlvkvs.pairs.begin(); kit != rskv->tlvkvs.pairs.end(); ++kit) + if (kit->key == HASH_CACHE_DURATION_SS) + { + uint32_t t=0 ; + if(sscanf(kit->value.c_str(),"%d",&t) == 1) + mHashCache->setRememberHashFilesDuration(t); + } + else if(kit->key == WATCH_FILE_DURATION_SS) + { + int t=0 ; + if(sscanf(kit->value.c_str(),"%d",&t) == 1) + setWatchPeriod(t); + } + delete *it ; + continue ; + } + + RsFileConfigItem *fi = dynamic_cast(*it); + + if (fi) + { + /* ensure that it exists? */ + + SharedDirInfo info ; + info.filename = RsDirUtil::convertPathToUnix(fi->file.path); + info.virtualname = fi->file.name; + info.shareflags = FileStorageFlags(fi->flags) ; + info.shareflags &= PERMISSION_MASK ; + info.shareflags &= ~DIR_FLAGS_NETWORK_WIDE_GROUPS ; // disabling this flag for know, for consistency reasons + + for(std::set::const_iterator itt(fi->parent_groups.ids.begin());itt!=fi->parent_groups.ids.end();++itt) + info.parent_groups.push_back(*itt) ; + + dirList.push_back(info) ; + } + + delete *it ; + } + + /* set directories */ + mLocalSharedDirs->setSharedDirectoryList(dirList); + load.clear() ; + + return true; } void p3FileDatabase::cleanup() @@ -567,25 +688,30 @@ bool p3FileDatabase::inDirectoryCheck() } void p3FileDatabase::setWatchPeriod(uint32_t seconds) { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalDirWatcher->setFileWatchPeriod(seconds); + IndicateConfigChanged(); } uint32_t p3FileDatabase::watchPeriod() { - NOT_IMPLEMENTED(); - return 0; + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->fileWatchPeriod(); } void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + mHashCache->setRememberHashFilesDuration(days) ; } uint32_t p3FileDatabase::rememberHashCacheDuration() { - NOT_IMPLEMENTED(); - return 0; + RS_STACK_MUTEX(mFLSMtx) ; + return mHashCache->rememberHashFilesDuration() ; } void p3FileDatabase::clearHashCache() { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + mHashCache->clear() ; } bool p3FileDatabase::rememberHashCache() { diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index e6d08a3c6..582f5c508 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -45,7 +45,8 @@ file_lists { file_sharing/filelist_io.h \ file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ - file_sharing/rsfilelistitems.h + file_sharing/rsfilelistitems.h \ + file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ From 52ef7adfae6cc7291040ed1103994792557f85bf Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 1 Sep 2016 21:41:12 +0200 Subject: [PATCH 45/92] fixed saving of config when diretory list is changed --- libretroshare/src/file_sharing/p3filelists.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 597762958..489af067b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -57,10 +57,14 @@ RsSerialiser *p3FileDatabase::setupSerialiser() } void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { - RS_STACK_MUTEX(mFLSMtx) ; + { + RS_STACK_MUTEX(mFLSMtx) ; - mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; - mLocalDirWatcher->forceUpdate(); + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; + mLocalDirWatcher->forceUpdate(); + } + + IndicateConfigChanged(); } void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { @@ -75,6 +79,7 @@ void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) } RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + IndicateConfigChanged(); } p3FileDatabase::~p3FileDatabase() From 7b566f2c55a4f8f7fe2334254fa82604f9116773 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 2 Sep 2016 21:49:43 +0200 Subject: [PATCH 46/92] added load/save for hash cache in the proper place --- .../src/file_sharing/directory_updater.h | 11 +----- .../src/file_sharing/file_sharing_defaults.h | 4 +++ libretroshare/src/file_sharing/hash_cache.cc | 34 +++++++++++++++---- libretroshare/src/file_sharing/hash_cache.h | 5 +-- libretroshare/src/file_sharing/p3filelists.cc | 17 ++++++++-- libretroshare/src/file_sharing/p3filelists.h | 1 + 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 63d6ec260..3fb52c81e 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -6,16 +6,7 @@ #include "file_sharing/hash_cache.h" #include "file_sharing/directory_storage.h" -class DirectoryUpdater -{ - public: - DirectoryUpdater() {} - virtual ~DirectoryUpdater(){} -}; - -#warning: simplify, if we don't keep the remote directory updater - -class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread +class LocalDirectoryUpdater: public HashStorageClient, public RsTickingThread { public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index a36e311b0..b40672478 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -7,3 +7,7 @@ static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 sec static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. +static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. + +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 60 secs. diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index d8e65f810..39bbdc6f3 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -3,6 +3,7 @@ #include "pqi/authssl.h" #include "hash_cache.h" #include "filelist_io.h" +#include "file_sharing_defaults.h" #define HASHSTORAGE_DEBUG 1 @@ -14,8 +15,12 @@ HashStorage::HashStorage(const std::string& save_file_name) { mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; + mLastSaveTime = 0 ; - load() ; + { + RS_STACK_MUTEX(mHashMtx) ; + locked_load() ; + } } void HashStorage::data_tick() @@ -48,10 +53,13 @@ void HashStorage::data_tick() mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; + if(!mChanged) // otherwise it might prevent from saving the hash cache + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + } RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; } @@ -92,11 +100,23 @@ void HashStorage::data_tick() info.modf_stamp = job.ts ; info.time_stamp = time(NULL); info.hash = hash; + + mChanged = true ; } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); + + { + RS_STACK_MUTEX(mHashMtx) ; + if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) + { + locked_save(); + mLastSaveTime = time(NULL) ; + mChanged = false ; + } + } } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -180,7 +200,7 @@ void HashStorage::clean() #endif } -void HashStorage::load() +void HashStorage::locked_load() { uint64_t file_size ; @@ -249,7 +269,7 @@ void HashStorage::load() #endif } -void HashStorage::save() +void HashStorage::locked_save() { #ifdef HASHSTORAGE_DEBUG std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 39d9767fe..6413ce68c 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -55,8 +55,8 @@ public: private: void clean() ; - void save() ; - void load() ; + void locked_save() ; + void locked_load() ; bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; @@ -85,5 +85,6 @@ private: RsMutex mHashMtx ; bool mRunning; uint32_t mInactivitySleepTime ; + time_t mLastSaveTime ; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 489af067b..f92e7be10 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -8,6 +8,7 @@ #include "retroshare/rsids.h" #include "retroshare/rspeers.h" +#include "rsserver/rsaccounts.h" #include "rsserver/p3face.h" @@ -26,13 +27,25 @@ static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { - // loads existing indexes for friends. Some might be already present here. + // make sure the base directory exists + + std::string base_dir = rsAccounts->PathAccountDirectory(); + + if(base_dir.empty()) + throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ; + + mFileSharingDir = base_dir + "/" + FILE_SHARING_DIR_NAME ; + + if(!RsDirUtil::checkCreateDirectory(mFileSharingDir)) + throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ; + + // loads existing indexes for friends. Some might be already present here. // mRemoteDirectories.clear() ; // we should load them! mOwnId = mpeers->getOwnId() ; mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); - mHashCache = new HashStorage("hash_cache.bin") ; + mHashCache = new HashStorage(mFileSharingDir + "/" + HASH_CACHE_FILE_NAME) ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index b06335458..d4504bb77 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -195,5 +195,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; + std::string mFileSharingDir ; }; From 153783acb5d16b45a94728fa10238558bbbd8ff0 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 2 Sep 2016 22:08:27 +0200 Subject: [PATCH 47/92] fixed bug in saving hash cache --- libretroshare/src/file_sharing/hash_cache.cc | 39 ++++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 39bbdc6f3..a0ea48c95 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -33,6 +33,16 @@ void HashStorage::data_tick() bool empty ; uint32_t st ; + { + RS_STACK_MUTEX(mHashMtx) ; + if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) + { + locked_save(); + mLastSaveTime = time(NULL) ; + mChanged = false ; + } + } + { RS_STACK_MUTEX(mHashMtx) ; @@ -93,30 +103,24 @@ void HashStorage::data_tick() // store the result - HashStorageInfo& info(mFiles[job.full_path]); + { + RS_STACK_MUTEX(mHashMtx) ; + HashStorageInfo& info(mFiles[job.full_path]); - info.filename = job.full_path ; - info.size = size ; - info.modf_stamp = job.ts ; - info.time_stamp = time(NULL); - info.hash = hash; + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; - mChanged = true ; + mChanged = true ; + } } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); - { - RS_STACK_MUTEX(mHashMtx) ; - if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) - { - locked_save(); - mLastSaveTime = time(NULL) ; - mChanged = false ; - } - } } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -170,6 +174,8 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t void HashStorage::clean() { + RS_STACK_MUTEX(mHashMtx) ; + #ifdef HASHSTORAGE_DEBUG std::cerr << "Cleaning HashStorage..." << std::endl ; #endif @@ -311,6 +317,7 @@ void HashStorage::locked_save() #ifdef FIM_DEBUG std::cerr << "done." << std::endl ; #endif + std::cerr << mFiles.size() << " Entries saved." << std::endl; save_free: free(encryptedData); From 371171fcbb66adecaadd7ffd5e33410595714b35 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 3 Sep 2016 13:29:23 +0200 Subject: [PATCH 48/92] moved InternalFileHierarchyStorage to separate file. Fixed loading/saving infrastructure (not working yet) --- .../src/file_sharing/dir_hierarchy.cc | 556 +++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 119 ++++ .../src/file_sharing/directory_storage.cc | 651 +----------------- .../src/file_sharing/directory_storage.h | 27 +- .../src/file_sharing/file_sharing_defaults.h | 3 +- libretroshare/src/file_sharing/p3filelists.cc | 17 +- libretroshare/src/file_sharing/p3filelists.h | 1 + libretroshare/src/libretroshare.pro | 2 + 8 files changed, 743 insertions(+), 633 deletions(-) create mode 100644 libretroshare/src/file_sharing/dir_hierarchy.cc create mode 100644 libretroshare/src/file_sharing/dir_hierarchy.h diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc new file mode 100644 index 000000000..74db29e1b --- /dev/null +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -0,0 +1,556 @@ +#include "dir_hierarchy.h" + +#define DEBUG_DIRECTORY_STORAGE 1 + +/******************************************************************************************************************/ +/* Internal File Hierarchy Storage */ +/******************************************************************************************************************/ + +template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) +{ + typename std::set::iterator tmp(it); + ++tmp; + s.erase(it) ; + return tmp; +} + +// This class handles the file hierarchy +// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, +// so that the hierarchy stays completely coherent between calls. + +InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) +{ + DirEntry *de = new DirEntry("") ; + + de->row=0; + de->parent_index=0; + de->dir_modtime=0; + + mNodes.push_back(de) ; +} + +int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) +{ + if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) + return -1 ; + + return mNodes[mNodes[e]->parent_index]->row; +} + +// high level modification routines + +bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) const +{ + return e < mNodes.size() && mNodes[e] != NULL ; +} +bool InternalFileHierarchyStorage::stampDirectory(const DirectoryStorage::EntryIndex& indx) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + static_cast(mNodes[indx])->dir_modtime = time(NULL) ; + return true; +} + +bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + DirEntry& d(*static_cast(mNodes[indx])) ; + + std::map should_create(subdirs); + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) + { + std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + + if( !removeDirectory(d.subdirs[i])) + i++ ; + } + else + { + std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; + ++i; + } + + for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) + { + std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; + + DirEntry *de = new DirEntry(it->first) ; + + de->row = mNodes.size(); + de->parent_index = indx; + de->dir_modtime = it->second; + + d.subdirs.push_back(mNodes.size()) ; + mNodes.push_back(de) ; + } + + return true; +} +bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! +{ + // check that it's a directory + + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + if(indx == 0) + return nodeAccessError("checkIndex(): Cannot remove top level directory") ; + +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "(--) Removing directory " << indx << std::endl; + print(); +#endif + // remove from parent + + DirEntry& d(*static_cast(mNodes[indx])) ; + DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); + + for(uint32_t i=0;i= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("checkIndex(): Node does not exist") ; + + if(! (mNodes[indx]->type() & type)) + return nodeAccessError("checkIndex(): Node is of wrong type") ; + + return true; +} + +bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + DirEntry& d(*static_cast(mNodes[indx])) ; + new_files = subfiles ; + + // remove from new_files the ones that already exist and have a modf time that is not older. + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])) ; + std::map::const_iterator it = subfiles.find(f.file_name) ; + + if(it == subfiles.end()) // file does not exist anymore => delete + { + std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; + + delete mNodes[d.subfiles[i]] ; + mNodes[d.subfiles[i]] = NULL ; + + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; + d.subfiles.pop_back(); + continue; + } + + if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + { + f.file_hash.clear(); // hash needs recomputing + f.file_modtime = it->second.modtime; + f.file_size = it->second.size; + } + new_files.erase(f.file_name) ; + + ++i; + } + + for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) + { + std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; + + d.subfiles.push_back(mNodes.size()) ; + mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; + } + return true; +} +bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) +{ + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; + return false; + } + + std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; + + RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + + old_hash = hash ; + + return true; +} +bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) +{ + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; + return false; + } + + FileEntry& fe(*static_cast(mNodes[file_index])) ; + + std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; + + fe.file_hash = hash; + fe.file_size = size; + fe.file_modtime = modf_time; + fe.file_name = fname; + + return true; +} + +bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; + return false; + } + DirEntry& d(*static_cast(mNodes[indx])) ; + + d.most_recent_time = most_recent_time; + d.dir_modtime = dir_modtime; + d.dir_update_time = time(NULL); + d.dir_name = dir_name; + + d.subfiles = subfiles_array ; + d.subdirs = subdirs_array ; + + // check that all subdirs already exist. If not, create. + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subdirs_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subdirs_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new DirEntry(""); + + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + node->row = i ; + node->parent_index = indx ; + } + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subfiles_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subfiles_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new FileEntry("",0,0); + + node->row = subdirs_array.size()+i ; + node->parent_index = indx ; + } + + return true; +} + +bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + recurs_max_modf_TS = d.most_recent_time ; + local_update_TS = d.dir_update_time ; + + return true; +} +bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + d.most_recent_time = recurs_max_modf_TS ; + d.dir_update_time = local_update_TS ; + + return true; +} + +// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + +time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) +{ + DirEntry& d(*static_cast(mNodes[dir_index])) ; + + time_t largest_modf_time = d.dir_modtime ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + + for(uint32_t i=0;i(mNodes[indx]) ; +} +const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const +{ + if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) + return NULL ; + + return static_cast(mNodes[indx]) ; +} +uint32_t InternalFileHierarchyStorage::getType(DirectoryStorage::EntryIndex indx) const +{ + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx]->type() ; + else + return FileStorageNode::TYPE_UNKNOWN; +} + +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) +{ + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; +} +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) +{ + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; +} + +bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) +{ + std::map::const_iterator it = mHashes.find(hash); + + if( it != mHashes.end() ) + { + results.clear(); + results.push_back(it->second) ; + return true ; + } + else + return false; +} + +bool InternalFileHierarchyStorage::check(std::string& error_string) const // checks consistency of storage. +{ + // recurs go through all entries, check that all + + std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. + hits[0] = 1 ; // because 0 is never the child of anyone + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + // stamp the kids + const DirEntry& de = *static_cast(mNodes[i]) ; + + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subdirs[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subdirs[j]] = 1; + } + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subfiles[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subfiles[j]] = 1; + } + + } + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++ndirs; + } + else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++nfiles; + } + else + { + ++nunknown; + std::cerr << "(EE) Error: unknown type node found!" << std::endl; + } + + std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; + + recursPrint(0,DirectoryStorage::EntryIndex(0)); +} +void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::EntryIndex node) const +{ + std::string indent(2*depth,' '); + + if(mNodes[node] == NULL) + { + std::cerr << "EMPTY NODE !!" << std::endl; + return ; + } + DirEntry& d(*static_cast(mNodes[node])); + + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; + } +} + +bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s) +{ + std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; + return false ; +} + +// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning +// phase. That allows to keep file information when moving them around. + +bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::EntryIndex dir) +{ + DirEntry& d(*static_cast(mNodes[dir])) ; + + for(uint32_t i=0;i +#include +#include + +#include "directory_storage.h" + +class InternalFileHierarchyStorage +{ +public: + class FileStorageNode + { + public: + static const uint32_t TYPE_UNKNOWN = 0x0000 ; + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; + + virtual ~FileStorageNode() {} + virtual uint32_t type() const =0; + + DirectoryStorage::EntryIndex parent_index; + uint32_t row ; + }; + class FileEntry: public FileStorageNode + { + public: + FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + virtual ~FileEntry() {} + + // local stuff + std::string file_name ; + uint64_t file_size ; + time_t file_modtime; + RsFileHash file_hash ; + }; + + class DirEntry: public FileStorageNode + { + public: + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} + virtual ~DirEntry() {} + + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } + + // local stuff + std::string dir_name ; + std::string dir_parent_path ; + + std::vector subdirs ; + std::vector subfiles ; + + time_t dir_modtime; + time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. + }; + + // class stuff + InternalFileHierarchyStorage() ; + + bool load(const std::string& fname) ; + bool save(const std::string& fname) ; + + int parentRow(DirectoryStorage::EntryIndex e); + bool isIndexValid(DirectoryStorage::EntryIndex e) const; + bool stampDirectory(const DirectoryStorage::EntryIndex& indx); + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); + bool removeDirectory(DirectoryStorage::EntryIndex indx) ; + bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); + bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array); + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + + // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + + time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index); + + // file/dir access and modification + bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + + uint32_t mRoot ; + std::vector mNodes;// uses pointers to keep information about valid/invalid objects. + + void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to + // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that + // are not referenced. + + friend class DirectoryStorage ; // only class that can use this. + friend class LocalDirectoryStorage ; // only class that can use this. + + // Low level stuff. Should normally not be used externally. + + const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const; + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const; + const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const; + uint32_t getType(DirectoryStorage::EntryIndex indx) const; + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index); + DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index); + bool searchHash(const RsFileHash& hash,std::list& results); + + bool check(std::string& error_string) const ;// checks consistency of storage. + + void print() const; + +private: + void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; + static bool nodeAccessError(const std::string& s); + + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning + // phase. That allows to keep file information when moving them around. + + bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); + + std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? +}; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b490038f8..f7e9b4775 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -3,622 +3,11 @@ #include "retroshare/rspeers.h" #include "util/rsdir.h" #include "util/rsstring.h" +#include "file_sharing_defaults.h" #include "directory_storage.h" +#include "dir_hierarchy.h" #include "filelist_io.h" -#define DEBUG_DIRECTORY_STORAGE 1 - -/******************************************************************************************************************/ -/* Internal File Hierarchy Storage */ -/******************************************************************************************************************/ - -template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) -{ - typename std::set::iterator tmp(it); - ++tmp; - s.erase(it) ; - return tmp; -} - -// This class handles the file hierarchy -// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, -// so that the hierarchy stays completely coherent between calls. - -class InternalFileHierarchyStorage -{ -public: - class FileStorageNode - { - public: - static const uint32_t TYPE_UNKNOWN = 0x0000 ; - static const uint32_t TYPE_FILE = 0x0001 ; - static const uint32_t TYPE_DIR = 0x0002 ; - - virtual ~FileStorageNode() {} - virtual uint32_t type() const =0; - - DirectoryStorage::EntryIndex parent_index; - uint32_t row ; - }; - class FileEntry: public FileStorageNode - { - public: - FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} - virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } - virtual ~FileEntry() {} - - // local stuff - time_t file_modtime; - std::string file_name ; - uint64_t file_size ; - RsFileHash file_hash ; - }; - - class DirEntry: public FileStorageNode - { - public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} - virtual ~DirEntry() {} - - virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - - // local stuff - std::string dir_name ; - std::string dir_parent_path ; - - std::vector subdirs ; - std::vector subfiles ; - - time_t dir_modtime; - time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. - time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. - }; - - // class stuff - InternalFileHierarchyStorage() : mRoot(0) - { - DirEntry *de = new DirEntry("") ; - - de->row=0; - de->parent_index=0; - de->dir_modtime=0; - - mNodes.push_back(de) ; - } - - int parentRow(DirectoryStorage::EntryIndex e) - { - if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) - return -1 ; - - return mNodes[mNodes[e]->parent_index]->row; - } - - // high level modification routines - - bool isIndexValid(DirectoryStorage::EntryIndex e) const - { - return e < mNodes.size() && mNodes[e] != NULL ; - } - bool stampDirectory(const DirectoryStorage::EntryIndex& indx) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - static_cast(mNodes[indx])->dir_modtime = time(NULL) ; - return true; - } - - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - DirEntry& d(*static_cast(mNodes[indx])) ; - - std::map should_create(subdirs); - - for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) - { - std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; - - if( !removeDirectory(d.subdirs[i])) - i++ ; - } - else - { - std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; - - should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; - ++i; - } - - for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) - { - std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; - - DirEntry *de = new DirEntry(it->first) ; - - de->row = mNodes.size(); - de->parent_index = indx; - de->dir_modtime = it->second; - - d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(de) ; - } - - return true; - } - bool removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! - { - // check that it's a directory - - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - if(indx == 0) - return nodeAccessError("checkIndex(): Cannot remove top level directory") ; - -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "(--) Removing directory " << indx << std::endl; - print(); -#endif - // remove from parent - - DirEntry& d(*static_cast(mNodes[indx])) ; - DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); - - for(uint32_t i=0;i= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("checkIndex(): Node does not exist") ; - - if(! (mNodes[indx]->type() & type)) - return nodeAccessError("checkIndex(): Node is of wrong type") ; - - return true; - } - - bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - DirEntry& d(*static_cast(mNodes[indx])) ; - new_files = subfiles ; - - // remove from new_files the ones that already exist and have a modf time that is not older. - - for(uint32_t i=0;i(mNodes[d.subfiles[i]])) ; - std::map::const_iterator it = subfiles.find(f.file_name) ; - - if(it == subfiles.end()) // file does not exist anymore => delete - { - std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; - - delete mNodes[d.subfiles[i]] ; - mNodes[d.subfiles[i]] = NULL ; - - d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; - d.subfiles.pop_back(); - continue; - } - - if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size - { - f.file_hash.clear(); // hash needs recomputing - f.file_modtime = it->second.modtime; - f.file_size = it->second.size; - } - new_files.erase(f.file_name) ; - - ++i; - } - - for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) - { - std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; - - d.subfiles.push_back(mNodes.size()) ; - mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); - mNodes.back()->row = mNodes.size()-1; - mNodes.back()->parent_index = indx; - } - return true; - } - bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) - { - if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) - { - std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; - return false; - } - - std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; - - RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; - - old_hash = hash ; - - return true; - } - bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) - { - if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) - { - std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; - return false; - } - - FileEntry& fe(*static_cast(mNodes[file_index])) ; - - std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; - - fe.file_hash = hash; - fe.file_size = size; - fe.file_modtime = modf_time; - fe.file_name = fname; - - return true; - } - - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; - return false; - } - DirEntry& d(*static_cast(mNodes[indx])) ; - - d.most_recent_time = most_recent_time; - d.dir_modtime = dir_modtime; - d.dir_update_time = time(NULL); - d.dir_name = dir_name; - - d.subfiles = subfiles_array ; - d.subdirs = subdirs_array ; - - // check that all subdirs already exist. If not, create. - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subdirs_array[i]+1,NULL); - - FileStorageNode *& node(mNodes[subdirs_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) - { - delete node ; - node = NULL ; - } - - if(node == NULL) - node = new DirEntry(""); - - ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; - node->row = i ; - node->parent_index = indx ; - } - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subfiles_array[i]+1,NULL); - - FileStorageNode *& node(mNodes[subfiles_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) - { - delete node ; - node = NULL ; - } - - if(node == NULL) - node = new FileEntry("",0,0); - - node->row = subdirs_array.size()+i ; - node->parent_index = indx ; - } - - return true; - } - - bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) - { - if(!checkIndex(index,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; - return false; - } - - DirEntry& d(*static_cast(mNodes[index])) ; - - recurs_max_modf_TS = d.most_recent_time ; - local_update_TS = d.dir_update_time ; - - return true; - } - bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) - { - if(!checkIndex(index,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; - return false; - } - - DirEntry& d(*static_cast(mNodes[index])) ; - - d.most_recent_time = recurs_max_modf_TS ; - d.dir_update_time = local_update_TS ; - - return true; - } - - // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. - - time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) - { - DirEntry& d(*static_cast(mNodes[dir_index])) ; - - time_t largest_modf_time = d.dir_modtime ; - - for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; - - for(uint32_t i=0;i mNodes;// uses pointers to keep information about valid/invalid objects. - - void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to - // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that - // are not referenced. - - friend class DirectoryStorage ; // only class that can use this. - friend class LocalDirectoryStorage ; // only class that can use this. - - // Low level stuff. Should normally not be used externally. - - const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const - { - if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) - return mNodes[indx] ; - else - return NULL ; - } - - const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return NULL ; - - return static_cast(mNodes[indx]) ; - } - const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const - { - if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) - return NULL ; - - return static_cast(mNodes[indx]) ; - } - uint32_t getType(DirectoryStorage::EntryIndex indx) const - { - if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) - return mNodes[indx]->type() ; - else - return FileStorageNode::TYPE_UNKNOWN; - } - - DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) - { - if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) - return DirectoryStorage::NO_INDEX; - - if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) - return DirectoryStorage::NO_INDEX; - - return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; - } - DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) - { - if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) - return DirectoryStorage::NO_INDEX; - - if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) - return DirectoryStorage::NO_INDEX; - - return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; - } - - bool searchHash(const RsFileHash& hash,std::list& results) - { - std::map::const_iterator it = mHashes.find(hash); - - if( it != mHashes.end() ) - { - results.clear(); - results.push_back(it->second) ; - return true ; - } - else - return false; - } - - bool check(std::string& error_string) const // checks consistency of storage. - { - // recurs go through all entries, check that all - - std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. - hits[0] = 1 ; // because 0 is never the child of anyone - - for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) - { - // stamp the kids - const DirEntry& de = *static_cast(mNodes[i]) ; - - for(uint32_t j=0;j= mNodes.size()) - { - error_string = "Node child out of tab!" ; - return false ; - } - if(hits[de.subdirs[j]] != 0) - { - error_string = "Double hit on a single node" ; - return false; - } - hits[de.subdirs[j]] = 1; - } - for(uint32_t j=0;j= mNodes.size()) - { - error_string = "Node child out of tab!" ; - return false ; - } - if(hits[de.subfiles[j]] != 0) - { - error_string = "Double hit on a single node" ; - return false; - } - hits[de.subfiles[j]] = 1; - } - - } - - for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) - { - std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; - ++ndirs; - } - else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) - { - std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; - ++nfiles; - } - else - { - ++nunknown; - std::cerr << "(EE) Error: unknown type node found!" << std::endl; - } - - std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; - - recursPrint(0,DirectoryStorage::EntryIndex(0)); - } -private: - void recursPrint(int depth,DirectoryStorage::EntryIndex node) const - { - std::string indent(2*depth,' '); - - if(mNodes[node] == NULL) - { - std::cerr << "EMPTY NODE !!" << std::endl; - return ; - } - DirEntry& d(*static_cast(mNodes[node])); - - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; - - for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; - } - } - - static bool nodeAccessError(const std::string& s) - { - std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; - return false ; - } - - // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning - // phase. That allows to keep file information when moving them around. - - bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir) - { - DirEntry& d(*static_cast(mNodes[dir])) ; - - for(uint32_t i=0;i mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? -}; - /******************************************************************************************************************/ /* Iterators */ /******************************************************************************************************************/ @@ -666,8 +55,8 @@ std::string DirectoryStorage::DirIterator::name() const { const InternalFil /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirectoryStorage(const std::string& local_file_name, const RsPeerId &pid) - : mFileName(local_file_name),mPeerId(pid), mDirStorageMtx("Directory storage "+local_file_name) +DirectoryStorage::DirectoryStorage(const RsPeerId &pid) + : mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString()) { RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); @@ -771,10 +160,20 @@ void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_ void DirectoryStorage::load(const std::string& local_file_name) { + std::cerr << "DirectoryStorage::load()" << std::endl; + // first load the header, than all fields. + + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) { + std::cerr << "DirectoryStorage::Save()" << std::endl; + + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->save(local_file_name); + // first write the header, than all fields. } void DirectoryStorage::print() @@ -787,8 +186,8 @@ void DirectoryStorage::print() bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; -#warning todo + NOT_IMPLEMENTED(); return true; } @@ -1142,7 +541,24 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary /* Remote Directory Storage */ /******************************************************************************************************************/ -bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) +RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) + : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname) +{ + load(fname) ; +} + +void RemoteDirectoryStorage::checkSave() +{ + time_t now = time(NULL); + + if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now) + { + save(mFileName); + mLastSavedTime = now ; + } +} + +bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) { const unsigned char *section_data = (unsigned char*)bindata.bin_data ; uint32_t section_size = bindata.bin_len ; @@ -1241,6 +657,7 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!mFileHierarchy->updateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; } + mChanged = true ; return true ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0f5311944..302c24be5 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -16,10 +16,10 @@ class RsTlvBinaryData ; class DirectoryStorage { public: - DirectoryStorage(const std::string& local_file_name,const RsPeerId& pid) ; + DirectoryStorage(const RsPeerId& pid) ; virtual ~DirectoryStorage() {} - typedef int32_t EntryIndex ; + typedef uint32_t EntryIndex ; static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; @@ -104,10 +104,12 @@ class DirectoryStorage void print(); void cleanup(); - private: + protected: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; + private: + void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; @@ -116,7 +118,6 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. - std::string mFileName; RsPeerId mPeerId; protected: @@ -128,7 +129,7 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) ; virtual ~RemoteDirectoryStorage() {} /*! @@ -140,13 +141,24 @@ public: * \param bindata binary data to deserialise from * \return false when the directory cannot be found. */ - bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; + bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; + + /*! + * \brief checkSave + * Checks the time of last saving, last modification time, and saves if needed. + */ + void checkSave() ; + +private: + time_t mLastSavedTime ; + bool mChanged ; + std::string mFileName; }; class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(fname,own_id) {} + LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; @@ -194,6 +206,7 @@ private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename + std::string mFileName; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index b40672478..5bc404c5f 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -10,4 +10,5 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. -static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 60 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index f92e7be10..9e1b323e2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -51,6 +51,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + mLastCleanupTime = 0 ; // This is for the transmission of data @@ -141,13 +142,10 @@ int p3FileDatabase::tick() // - remove/delete shared file lists for people who are not friend anymore // - - static time_t last_cleanup_time = 0; - -#warning we should use members here, not static - if(last_cleanup_time + 5 < now) + if(mLastCleanupTime + 5 < now) { cleanup(); - last_cleanup_time = now ; + mLastCleanupTime = now ; } static time_t last_print_time = 0; @@ -186,6 +184,7 @@ int p3FileDatabase::tick() mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; for(uint32_t i=0;ipeerId()) != online_peers.end()) { std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; @@ -196,6 +195,9 @@ int p3FileDatabase::tick() locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; } + mRemoteDirectories[i]->checkSave() ; + } + mLastRemoteDirSweepTS = now; } @@ -440,8 +442,7 @@ void p3FileDatabase::cleanup() std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const { -#warning we should use the default paths here. Ask p3config - return "dirlist_"+pid.toStdString()+".txt" ; + return mFileSharingDir + "/" + "dirlist_"+pid.toStdString()+".bin" ; } uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) @@ -998,7 +999,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; - if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) + if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index d4504bb77..ec5f1b517 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -196,5 +196,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; std::string mFileSharingDir ; + time_t mLastCleanupTime; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 582f5c508..56d36469c 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -46,6 +46,7 @@ file_lists { file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ file_sharing/rsfilelistitems.h \ + file_sharing/dir_hierarchy.h \ file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ @@ -53,6 +54,7 @@ file_lists { file_sharing/filelist_io.cc \ file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ + file_sharing/dir_hierarchy.cc \ file_sharing/rsfilelistitems.cc } From 5b221c56dbc12f39bce2917974a232a30c7c3559 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 3 Sep 2016 18:46:03 +0200 Subject: [PATCH 49/92] moved encrypted data output/input to FileListIO --- .../src/file_sharing/dir_hierarchy.cc | 3 + libretroshare/src/file_sharing/filelist_io.cc | 88 +++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 3 + libretroshare/src/file_sharing/hash_cache.cc | 88 +++---------------- 4 files changed, 107 insertions(+), 75 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 74db29e1b..d2b3abc05 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -548,9 +548,12 @@ bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::Entry bool InternalFileHierarchyStorage::save(const std::string& fname) { NOT_IMPLEMENTED(); + return true; } bool InternalFileHierarchyStorage::load(const std::string& fname) { NOT_IMPLEMENTED(); + return true; } + diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 770478c37..47d8899e3 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,4 +1,6 @@ #include "retroshare/rsids.h" +#include "pqi/authssl.h" +#include "util/rsdir.h" #include "serialiser/rsbaseserial.h" #include "filelist_io.h" @@ -116,3 +118,89 @@ bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32 return true; } +bool FileListIO::saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size) +{ + void *encryptedData = NULL ; + int encDataLen = 0 ; + + if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,total_size, AuthSSL::getAuthSSL()->OwnId())) + { + std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + return false; + } + + FILE *F = fopen( (fname+".tmp").c_str(),"wb" ) ; + + if(!F) + { + std::cerr << "Cannot open encrypted file cache for writing: " << fname+".tmp" << std::endl; + + free(encryptedData); + return false; + } + if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) + { + std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + fclose(F) ; + + free(encryptedData); + return false; + } + + fclose(F) ; + + RsDirUtil::renameFile(fname+".tmp",fname) ; +#ifdef FIM_DEBUG + std::cerr << "done." << std::endl ; +#endif + + free(encryptedData); + return true; +} + +bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size) +{ + uint64_t file_size ; + + if(!RsDirUtil::checkFile( fname,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file not present." << std::endl; + return false; + } + + // read the binary stream into memory. + // + RsTemporaryMemory buffer(file_size) ; + + if(buffer == NULL) + return false; + + FILE *F = fopen( fname.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << fname << std::endl; + return false; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " + fname << ": something's wrong." << std::endl; + fclose(F) ; + return false; + } + fclose(F) ; + + // now decrypt + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + return false; + } + + data = (unsigned char*)decrypted_data ; + total_size = decrypted_data_size ; + + return true; +} diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index b22a57cc4..bb88fc577 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -61,6 +61,9 @@ public: template static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ; template static uint32_t serial_size(const T& val) ; + static bool saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size); + static bool loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size); + private: static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index a0ea48c95..a92842c54 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -208,59 +208,20 @@ void HashStorage::clean() void HashStorage::locked_load() { - uint64_t file_size ; + unsigned char *data = NULL ; + uint32_t data_size=0; - if(!RsDirUtil::checkFile( mFilePath,file_size,false ) ) + if(!FileListIO::loadEncryptedDataFromFile(mFilePath,data,data_size)) { - std::cerr << "Encrypted hash cache file not present." << std::endl; + std::cerr << "(EE) Cannot read hash cache." << std::endl; return ; } - - // read the binary stream into memory. - // - void *buffer = rs_malloc(file_size) ; - - if(buffer == NULL) - return ; - - FILE *F = fopen( mFilePath.c_str(),"rb") ; - if (!F) - { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl; - free(buffer); - return; - } - if(fread(buffer,1,file_size,F) != file_size) - { - std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl; - free(buffer) ; - fclose(F) ; - return ; - } - fclose(F) ; - - // now decrypt - void *decrypted_data =NULL; - int decrypted_data_size =0; - - if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) - { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; - free(buffer) ; - return ; - } - free(buffer) ; - -#ifdef HASHSTORAGE_DEBUG - std::cerr << "Loading HashCache from file " << mFilePath << std::endl ; - int n=0 ; -#endif - unsigned char *data = (unsigned char*)decrypted_data ; uint32_t offset = 0 ; HashStorageInfo info ; + uint32_t n=0; - while(offset < decrypted_data_size) - if(readHashStorageInfo(data,decrypted_data_size,offset,info)) + while(offset < data_size) + if(readHashStorageInfo(data,data_size,offset,info)) { #ifdef HASHSTORAGE_DEBUG std::cerr << info << std::endl; @@ -269,7 +230,7 @@ void HashStorage::locked_load() mFiles[info.filename] = info ; } - free(decrypted_data) ; + free(data) ; #ifdef HASHSTORAGE_DEBUG std::cerr << n << " entries loaded." << std::endl ; #endif @@ -288,39 +249,16 @@ void HashStorage::locked_save() for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) writeHashStorageInfo(data,total_size,offset,it->second) ; - void *encryptedData = NULL ; - int encDataLen = 0 ; - - if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId())) + if(!FileListIO::saveEncryptedDataToFile(mFilePath,data,offset)) { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; - return; + std::cerr << "(EE) Cannot save hash cache data." << std::endl; + free(data) ; + return ; } - FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ; - - if(!F) - { - std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl; - goto save_free; - } - if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) - { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; - fclose(F) ; - goto save_free; - } - - fclose(F) ; - - RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ; -#ifdef FIM_DEBUG - std::cerr << "done." << std::endl ; -#endif std::cerr << mFiles.size() << " Entries saved." << std::endl; -save_free: - free(encryptedData); + free(data) ; } bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const From 679c53b0b00aedc5b0906c7b7a8774cb97b8b803 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 5 Sep 2016 21:22:11 +0200 Subject: [PATCH 50/92] added load/save of directory hierarchies. Fixed bug in loading of hash cache (caused y non initialisation of dest string in GetRawString()) --- .../src/file_sharing/dir_hierarchy.cc | 205 +++++++++++++++++- .../src/file_sharing/dir_hierarchy.h | 6 +- .../src/file_sharing/directory_storage.cc | 4 +- libretroshare/src/file_sharing/filelist_io.h | 35 +-- libretroshare/src/file_sharing/hash_cache.cc | 3 + libretroshare/src/serialiser/rsbaseserial.cc | 3 + 6 files changed, 229 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d2b3abc05..b6713c9d6 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,4 +1,5 @@ #include "dir_hierarchy.h" +#include "filelist_io.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -235,7 +236,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; - d.most_recent_time = most_recent_time; + d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); d.dir_name = dir_name; @@ -297,7 +298,7 @@ bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryI DirEntry& d(*static_cast(mNodes[index])) ; - recurs_max_modf_TS = d.most_recent_time ; + recurs_max_modf_TS = d.dir_most_recent_time ; local_update_TS = d.dir_update_time ; return true; @@ -312,7 +313,7 @@ bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryI DirEntry& d(*static_cast(mNodes[index])) ; - d.most_recent_time = recurs_max_modf_TS ; + d.dir_most_recent_time = recurs_max_modf_TS ; d.dir_update_time = local_update_TS ; return true; @@ -332,7 +333,7 @@ time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectorySto for(uint32_t i=0;i(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; for(uint32_t i=0;itype() == FileStorageNode::TYPE_FILE) + { + const FileEntry& fe(*static_cast(mNodes[i])) ; + + unsigned char *file_section_data = NULL ; + uint32_t file_section_offset = 0 ; + uint32_t file_section_size = 0; + + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.file_name )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.file_size )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,fe.file_hash )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)fe.file_modtime)) throw std::runtime_error("Write error") ; + + if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,file_section_data,file_section_offset)) throw std::runtime_error("Write error") ; + + free(file_section_data) ; + } + else if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR) + { + const DirEntry& de(*static_cast(mNodes[i])) ; + + unsigned char *dir_section_data = NULL ; + uint32_t dir_section_offset = 0 ; + uint32_t dir_section_size = 0; + + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.dir_name )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,de.dir_parent_path )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)de.dir_modtime )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,(uint32_t)de.dir_update_time )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)de.dir_most_recent_time )) throw std::runtime_error("Write error") ; + + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs.size())) throw std::runtime_error("Write error") ; + + for(uint32_t j=0;j= mNodes.size()) + mNodes.resize(node_index+1,NULL) ; + + mNodes[node_index] = new FileEntry(file_name,file_size,file_modtime,file_hash); + } + else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) + { + uint32_t node_index ; + std::string dir_name ; + std::string dir_parent_path ; + uint32_t dir_modtime ; + uint32_t dir_update_time ; + uint64_t dir_most_recent_time ; + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw std::runtime_error("Read error") ; + + if(node_index >= mNodes.size()) + mNodes.resize(node_index+1,NULL) ; + + DirEntry *de = new DirEntry(dir_name) ; + de->dir_name = dir_name ; + de->dir_parent_path = dir_parent_path ; + de->dir_modtime = dir_modtime ; + de->dir_update_time = dir_update_time ; + de->dir_most_recent_time = dir_most_recent_time ; + + uint32_t n_subdirs = 0 ; + uint32_t n_subfiles = 0 ; + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw std::runtime_error("Read error") ; + + for(uint32_t j=0;jsubdirs.push_back(di) ; + } + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw std::runtime_error("Read error") ; + + for(uint32_t j=0;jsubfiles.push_back(fi) ; + } + } + else + throw std::runtime_error("Unknown node section.") ; + + free(node_section_data) ; + } + free(buffer) ; + return true ; + } + catch(std::exception& e) + { + std::cerr << "Error while writing: " << e.what() << std::endl; + + if(buffer != NULL) + free(buffer) ; + return false; + } } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index abf10a3ba..f0b109c7d 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -26,6 +26,8 @@ public: { public: FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} + FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {} + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } virtual ~FileEntry() {} @@ -39,7 +41,7 @@ public: class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -52,7 +54,7 @@ public: std::vector subfiles ; time_t dir_modtime; - time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + time_t dir_most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f7e9b4775..ba10db1a2 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -229,7 +229,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.type = DIR_TYPE_DIR; d.hash.clear() ; d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); - d.min_age = now - dir_entry->most_recent_time ; + d.min_age = now - dir_entry->dir_most_recent_time ; d.name = dir_entry->dir_name; d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; d.parent = (void*)(intptr_t)dir_entry->parent_index ; @@ -487,7 +487,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index bb88fc577..408ba26c8 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -9,20 +9,25 @@ // This file implements load/save of various fields used for file lists and directory content. // WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. -static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; +static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; -static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; -static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; -static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; -static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; -static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; -static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; -static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; -static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; -static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) @@ -94,11 +99,13 @@ private: if(offset + 1 > buff_size) return false ; - uint8_t section_tag = buff[offset++] ; + uint8_t section_tag = buff[offset] ; // we do the offset++ after, only if the header can be read. Doing so, we can make multiple read attempts. if(section_tag != check_section_tag) return false; + offset++ ; + return read125Size(buff,buff_size,offset,S) ; } }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index a92842c54..c2ed3ed4e 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,4 +1,5 @@ #include "util/rsdir.h" +#include "util/rsprint.h" #include "rsserver/p3face.h" #include "pqi/authssl.h" #include "hash_cache.h" @@ -299,6 +300,8 @@ bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_si if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; + std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl; + std::cerr << "Info.filename = " << info.filename << std::endl; free(section_data) ; return true; diff --git a/libretroshare/src/serialiser/rsbaseserial.cc b/libretroshare/src/serialiser/rsbaseserial.cc index e38dcb8e0..283c4f50e 100644 --- a/libretroshare/src/serialiser/rsbaseserial.cc +++ b/libretroshare/src/serialiser/rsbaseserial.cc @@ -231,6 +231,9 @@ uint32_t getRawStringSize(const std::string &outStr) bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr) { +#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect. + outStr.clear(); + uint32_t len = 0; if (!getRawUInt32(data, size, offset, &len)) { From ad1ba7a77dddf94cc89a0505b9f2d509f1d26372 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 6 Sep 2016 23:05:46 +0200 Subject: [PATCH 51/92] fixed loading/saving of remote directories --- .../src/file_sharing/dir_hierarchy.cc | 39 ++++++++++++++----- .../src/file_sharing/directory_storage.cc | 5 +++ libretroshare/src/file_sharing/filelist_io.h | 2 + libretroshare/src/file_sharing/p3filelists.cc | 4 ++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index b6713c9d6..d0d40d7bf 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -570,6 +570,8 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) uint32_t file_section_offset = 0 ; uint32_t file_section_size = 0; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,(uint32_t)fe.parent_index)) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ROW ,(uint32_t)fe.row )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.file_name )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.file_size )) throw std::runtime_error("Write error") ; @@ -588,6 +590,8 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) uint32_t dir_section_offset = 0 ; uint32_t dir_section_size = 0; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,(uint32_t)de.parent_index )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ROW ,(uint32_t)de.row )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.dir_name )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,de.dir_parent_path )) throw std::runtime_error("Write error") ; @@ -633,7 +637,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) try { - if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_offset) ) + if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_size) ) throw std::runtime_error("Read error") ; // Read some header @@ -667,17 +671,26 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint64_t file_size ; RsFileHash file_hash ; uint32_t file_modtime ; + uint32_t row ; + uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Write error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Read error") ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; - mNodes[node_index] = new FileEntry(file_name,file_size,file_modtime,file_hash); + FileEntry *fe = new FileEntry(file_name,file_size,file_modtime,file_hash); + + fe->parent_index = parent_index ; + fe->row = row ; + + mNodes[node_index] = fe ; } else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) { @@ -686,8 +699,12 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) std::string dir_parent_path ; uint32_t dir_modtime ; uint32_t dir_update_time ; - uint64_t dir_most_recent_time ; + uint32_t dir_most_recent_time ; + uint32_t row ; + uint32_t parent_index ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; @@ -705,6 +722,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) de->dir_update_time = dir_update_time ; de->dir_most_recent_time = dir_most_recent_time ; + de->parent_index = parent_index ; + de->row = row ; + uint32_t n_subdirs = 0 ; uint32_t n_subfiles = 0 ; @@ -725,6 +745,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,fi)) throw std::runtime_error("Read error") ; de->subfiles.push_back(fi) ; } + mNodes[node_index] = de ; } else throw std::runtime_error("Unknown node section.") ; @@ -736,7 +757,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) } catch(std::exception& e) { - std::cerr << "Error while writing: " << e.what() << std::endl; + std::cerr << "Error while reading: " << e.what() << std::endl; if(buffer != NULL) free(buffer) ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index ba10db1a2..94e5f74ed 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -545,6 +545,11 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname) { load(fname) ; + + std::cerr << "***************************************" << std::endl; + std::cerr << "Loaded following directory for peer " << pid << std::endl; + mFileHierarchy->print(); + std::cerr << "***************************************" << std::endl; } void RemoteDirectoryStorage::checkSave() diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 408ba26c8..db4df001a 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -28,6 +28,8 @@ static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; +static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x10 ; +static const uint8_t FILE_LIST_IO_TAG_ROW = 0x11 ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9e1b323e2..600e9edec 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -199,6 +199,9 @@ int p3FileDatabase::tick() } mLastRemoteDirSweepTS = now; + +#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up. + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } return 0; @@ -412,6 +415,7 @@ void p3FileDatabase::cleanup() mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } From eaa8ad883a65ecc4e41da856583d7c95c433a293 Mon Sep 17 00:00:00 2001 From: MrAlice Date: Wed, 7 Sep 2016 22:31:12 +0200 Subject: [PATCH 52/92] half implemented the hash-based communication for dirs between friends --- .../src/file_sharing/dir_hierarchy.cc | 4 +- .../src/file_sharing/dir_hierarchy.h | 16 ++++- .../src/file_sharing/directory_storage.h | 5 +- libretroshare/src/file_sharing/p3filelists.cc | 66 ++++++++++++++----- .../src/file_sharing/rsfilelistitems.cc | 16 ++--- .../src/file_sharing/rsfilelistitems.h | 16 ++--- 6 files changed, 87 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d0d40d7bf..f7e8fe61a 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -393,9 +393,9 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(Direct bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) { - std::map::const_iterator it = mHashes.find(hash); + std::map::const_iterator it = mFileHashes.find(hash); - if( it != mHashes.end() ) + if( it != mFileHashes.end() ) { results.clear(); results.push_back(it->second) ; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f0b109c7d..3115543d5 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -117,5 +117,19 @@ private: bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? + // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // is used for fast search access for FT. + // Note: We should try something faster than std::map. hash_map?? + + std::map mFileHashes ; + + // The directory hashes are the sha1sum of the + // full public path to the directory. + // The later is used by synchronisation items in order + // to avoid sending explicit EntryIndex values. + // This is kept separate from mFileHashes because the two are used + // in very different ways. + // + std::map mDirHashes ; }; + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 302c24be5..7d98730ac 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -94,13 +94,16 @@ class DirectoryStorage const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; - bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; } + bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; } + void print(); void cleanup(); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 600e9edec..d53f3c578 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -200,7 +200,7 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; -#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up. +#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up. RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } @@ -864,6 +864,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - local node sends the last known modf time to friends, // - friends respond with either a full directory content, or an acknowledge that the time is right // +// Directories are designated by their hash, instead of their index. This allows to hide the non shared directories +// behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between +// friends is concerned (We obviously could not do that for GUI display, which has a small and constant cost). void p3FileDatabase::tickRecv() { @@ -898,12 +901,20 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { RS_STACK_MUTEX(mFLSMtx) ; - P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ; + EntryIndex entry_index = DirectoryStorage::NO_INDEX; + + if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + + uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ; ritem->PeerId(item->PeerId()) ; ritem->request_id = item->request_id; - ritem->entry_index = item->entry_index ; + ritem->entry_hash = item->entry_hash ; std::list node_groups; FileStorageFlags node_flags; @@ -913,15 +924,15 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } - else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl; + std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -931,7 +942,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->last_known_recurs_modf_TS = local_recurs_max_time; // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer. - mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ; + mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ; } else { @@ -950,13 +961,21 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { - P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; + + EntryIndex entry_index = DirectoryStorage::NO_INDEX; // remove the original request from pending list { RS_STACK_MUTEX(mFLSMtx) ; + if(entry_index == DirectoryStorage::NO_INDEX) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; if(it == mPendingSyncRequests.end()) @@ -984,12 +1003,19 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(mRemoteDirectories[fi] == NULL) mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); + + if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; + return ; + } + std::cerr << " entry index is " << entry_index ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { - P3FILELISTS_DEBUG() << " removing directory with index " << item->entry_index << " because it does not exist." << std::endl; - mRemoteDirectories[fi]->removeDirectory(item->entry_index); + P3FILELISTS_DEBUG() << " removing directory with index " << entry_index << " because it does not exist." << std::endl; + mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } @@ -997,13 +1023,13 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; - if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data)) + if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; @@ -1051,7 +1077,13 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - item->entry_index = e ; + + if(!rds->getHashFromIndex(e,item->entry_hash) ) + { + std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + return; + } + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; item->request_id = sync_req_id ; item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; @@ -1080,14 +1112,16 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { + static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; - // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way + // that cannot be brute-forced or reverse-engineered, which explains the random bias. for(uint32_t i=0;ientry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -206,7 +206,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash - ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -274,7 +274,7 @@ uint32_t RsFileListsSyncRequestItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -287,7 +287,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -308,7 +308,7 @@ std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t inde printRsItemBase(out, "RsFileListsSyncReqItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; @@ -323,7 +323,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind printRsItemBase(out, "RsFileListsSyncDirItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 1ff00d57a..5644d4648 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -84,10 +84,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // index of the directory to sync - uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash }; class RsFileListsSyncResponseItem : public RsFileListsItem @@ -102,10 +102,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. }; From fafe684cc40917f72b5305f805b50f2ea9f06d3e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 8 Sep 2016 21:34:53 +0200 Subject: [PATCH 53/92] finished implementing hash-based transactions. To be tested. --- .../src/file_sharing/dir_hierarchy.cc | 104 ++++++++++++++---- .../src/file_sharing/dir_hierarchy.h | 2 +- .../src/file_sharing/directory_storage.cc | 46 +++++--- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index f7e8fe61a..94fa8eaff 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -227,7 +227,7 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex return true; } -bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) +bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_hash,const std::vector& subfiles_hash) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) { @@ -241,47 +241,103 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.dir_update_time = time(NULL); d.dir_name = dir_name; - d.subfiles = subfiles_array ; - d.subdirs = subdirs_array ; + d.subdirs.clear(); + d.subfiles.clear(); // check that all subdirs already exist. If not, create. - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subdirs_array[i]+1,NULL); + DirectoryStorage::EntryIndex dir_index = 0; + std::map::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; - FileStorageNode *& node(mNodes[subdirs_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + if(it == mDirHashes.end() || it->second >= mNodes.size()) { - delete node ; - node = NULL ; - } + // find an epty slot + int found = -1 ; - if(node == NULL) + for(uint32_t j=0;jsecond; + + if(mNodes[dir_index] != NULL && mNodes[dir_index]->type() != FileStorageNode::TYPE_DIR) + { + delete mNodes[dir_index] ; + mNodes[dir_index] = NULL ; + } + } + FileStorageNode *& node(mNodes[dir_index]); + if(!node) node = new DirEntry(""); + d.subdirs.push_back(dir_index) ; + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; node->row = i ; node->parent_index = indx ; } - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subfiles_array[i]+1,NULL); + DirectoryStorage::EntryIndex file_index = 0; + std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; - FileStorageNode *& node(mNodes[subfiles_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + if(it == mFileHashes.end()) { - delete node ; - node = NULL ; + // find an epty slot + int found = -1; + + for(uint32_t j=0;jsecond ; - if(node == NULL) - node = new FileEntry("",0,0); + if(mNodes[file_index] != NULL && mNodes[file_index]->type() != FileStorageNode::TYPE_FILE) + { + delete mNodes[file_index] ; + mNodes[file_index] = NULL ; + } - node->row = subdirs_array.size()+i ; + file_index = it->second; + } + FileStorageNode *& node(mNodes[file_index]); + if(!node) + node = new FileEntry("",0,0,subfiles_hash[i]); + + d.subfiles.push_back(file_index) ; + + node->row = subdirs_hash.size()+i ; node->parent_index = indx ; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 3115543d5..13a3f5c3e 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -73,7 +73,7 @@ public: bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_hash); bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 94e5f74ed..bdcd24c84 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -466,7 +466,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary } // compute list of allowed subdirs - std::vector allowed_subdirs ; + std::vector allowed_subdirs ; FileStorageFlags node_flags ; std::list node_groups ; @@ -474,7 +474,16 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) - allowed_subdirs.push_back(dir->subdirs[i]) ; + { + RsFileHash hash ; + + if(!getHashFromIndex(dir->subdirs[i],hash)) + { + std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; + continue ; + } + allowed_subdirs.push_back(hash) ; + } unsigned char *section_data = NULL; uint32_t section_size = 0; @@ -498,7 +507,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // serialise subdirs entry indexes for(uint32_t i=0;isubfiles[i] )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; @@ -594,14 +602,14 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co // serialise subdirs entry indexes - std::vector subdirs_array ; - uint32_t subdir_index ; + std::vector subdirs_hashes ; + RsFileHash subdir_hash ; for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) + // First create the entries for each subdir and each subfile, if needed. + if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_hash)) { std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; return false ; } - // then update the subfiles - for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) - std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; + if(!mFileHierarchy->updateFile(file_index,subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) + std::cerr << "(EE) Cannot update file with index " << file_index <<" and hash " << subfiles_hash[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; } mChanged = true ; From 0ea695beb3dd1b1f71828ff0e8fcdcb38ed60f07 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 8 Sep 2016 23:43:14 +0200 Subject: [PATCH 54/92] fixed a few bugs in hash-based sync. Not there yet. --- .../src/file_sharing/dir_hierarchy.cc | 32 +++++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 7 ++++ .../src/file_sharing/directory_storage.cc | 18 ++++++++-- .../src/file_sharing/directory_storage.h | 4 +-- libretroshare/src/file_sharing/p3filelists.cc | 34 ++++++++----------- retroshare-gui/src/gui/SharedFilesDialog.cpp | 20 +++++++++++ 6 files changed, 91 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 94fa8eaff..8cf824110 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,3 +1,5 @@ +#include "util/rsdir.h" + #include "dir_hierarchy.h" #include "filelist_io.h" @@ -28,6 +30,36 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) de->dir_modtime=0; mNodes.push_back(de) ; +#warning not very elegant. We should remove the leading / + mDirHashes[computeDirHash("/")] = 0 ; +} + +RsFileHash InternalFileHierarchyStorage::computeDirHash(const std::string& dir_path) +{ + return RsDirUtil::sha1sum((unsigned char*)dir_path.c_str(),dir_path.length()) ; +} +bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + return false ; + + DirEntry& d = *static_cast(mNodes[index]) ; + + hash = computeDirHash( d.dir_parent_path + "/" + d.dir_name ) ; + + std::cerr << "Computing dir hash from index " << index << ". Dir=\"" << d.dir_parent_path + "/" + d.dir_name << "\" hash=" << hash << std::endl; + + return true; +} +bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +{ + std::map::const_iterator it = mDirHashes.find(hash) ; + + if(it == mDirHashes.end()) + return false; + + index = it->second; + return true; } int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 13a3f5c3e..5d7fb033a 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -81,6 +81,13 @@ public: time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index); + // hash stuff + + static RsFileHash computeDirHash(const std::string& dir_path); + + bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; + bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; + // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index bdcd24c84..c8a0b0cc3 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -193,6 +193,8 @@ bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { + RS_STACK_MUTEX(mDirStorageMtx) ; + d.children.clear() ; time_t now = time(NULL) ; @@ -266,6 +268,18 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return true; } + +bool DirectoryStorage::getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->getDirHashFromIndex(index,hash) ; +} +bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->getIndexFromDirHash(hash,index) ; +} + /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -477,7 +491,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary { RsFileHash hash ; - if(!getHashFromIndex(dir->subdirs[i],hash)) + if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) { std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; continue ; @@ -664,7 +678,7 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co { DirectoryStorage::EntryIndex file_index ; - if(!getIndexFromHash(subfiles_hash[i],file_index)) + if(!getIndexFromDirHash(subfiles_hash[i],file_index)) { std::cerr << "(EE) Cannot optain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; continue; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7d98730ac..fc9aadf7f 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -101,8 +101,8 @@ class DirectoryStorage bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); - bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; } - bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; } + bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; + bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; void print(); void cleanup(); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index d53f3c578..60312124e 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -12,7 +12,7 @@ #include "rsserver/p3face.h" -#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " +#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << ": p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; @@ -187,7 +187,7 @@ int p3FileDatabase::tick() { if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) { - std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; + P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; #ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[i]->print(); #endif @@ -210,20 +210,20 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "Starting directory watcher thread..." ; + P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; mLocalDirWatcher->start(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; } void p3FileDatabase::stopThreads() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; + P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ; mHashCache->fullstop(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; - std::cerr << "Stopping directory watcher thread..." ; std::cerr.flush() ; + P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ; mLocalDirWatcher->fullstop(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; } void p3FileDatabase::tickWatchers() @@ -367,7 +367,7 @@ void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "p3FileDatabase::cleanup()" << std::endl; + P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl; // look through the list of friend directories. Remove those who are not our friends anymore. // @@ -429,7 +429,7 @@ void p3FileDatabase::cleanup() for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) { - std::cerr << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; + P3FILELISTS_DEBUG() << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; std::map::iterator tmp(it); ++tmp; @@ -438,7 +438,7 @@ void p3FileDatabase::cleanup() } else { - std::cerr << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; + P3FILELISTS_DEBUG() << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; ++it ; } } @@ -905,7 +905,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) EntryIndex entry_index = DirectoryStorage::NO_INDEX; - if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index)) + if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index)) { std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; return; @@ -970,12 +970,6 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { RS_STACK_MUTEX(mFLSMtx) ; - if(entry_index == DirectoryStorage::NO_INDEX) - { - std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; - return; - } - std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; if(it == mPendingSyncRequests.end()) @@ -1004,7 +998,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(mRemoteDirectories[fi] == NULL) mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); - if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index)) + if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index)) { std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; @@ -1078,7 +1072,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - if(!rds->getHashFromIndex(e,item->entry_hash) ) + if(!rds->getDirHashFromIndex(e,item->entry_hash) ) { std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; return; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 209878f68..a0864839a 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -85,6 +85,8 @@ // #define DONT_USE_SEARCH_IN_TREE_VIEW 1 +//#define DEBUG_SHARED_FILES_DIALOG 1 + const QString Image_AddNewAssotiationForFile = ":/images/kcmsystem24.png"; class SFDSortFilterProxyModel : public QSortFilterProxyModel @@ -842,7 +844,9 @@ void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexe if(ui.dirTreeView->model() == NULL) return ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Saving expanded items. " << std::endl; +#endif for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -855,7 +859,9 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand if(ui.dirTreeView->model() == NULL) return ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Restoring expanded items. " << std::endl; +#endif for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -866,29 +872,39 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; +#endif if(ui.dirTreeView->isExpanded(index)) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Index " << local_path << " is expanded." << std::endl; +#endif if(index.isValid()) exp.insert(local_path) ; for(int row=0;rowmodel()->rowCount(index);++row) recursSaveExpandedItems(index.child(row,0),local_path,exp) ; } +#ifdef DEBUG_SHARED_FILES_DIALOG else std::cerr << "Index is not expanded." << std::endl; +#endif } void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; +#endif if(exp.find(local_path) != exp.end()) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "re expanding index " << local_path << std::endl; +#endif ui.dirTreeView->setExpanded(index,true) ; for(int row=0;rowmodel()->rowCount(index);++row) @@ -904,7 +920,9 @@ void SharedFilesDialog::postModDirectories(bool local) } std::set expanded_indexes; saveExpandedPaths(expanded_indexes) ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; +#endif /* Notify both models, only one is visible */ tree_model->postMods(); @@ -916,7 +934,9 @@ void SharedFilesDialog::postModDirectories(bool local) if (ui.filterPatternLineEdit->text().isEmpty() == false) FilterItems(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "****** updated directories! ******" << std::endl; +#endif QCoreApplication::flush(); } From 384d001f3bd9a5ba22ad16a0d8c6879216cf96bb Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 10 Sep 2016 13:57:05 +0200 Subject: [PATCH 55/92] progress in hash-based sync. Not there yet. --- .../src/file_sharing/dir_hierarchy.cc | 70 ++++++++++++++++--- .../src/file_sharing/dir_hierarchy.h | 4 +- .../src/file_sharing/directory_storage.cc | 65 ++++++++++++++--- .../src/file_sharing/directory_storage.h | 4 ++ libretroshare/src/file_sharing/filelist_io.h | 42 ++++++----- libretroshare/src/file_sharing/p3filelists.cc | 4 +- 6 files changed, 147 insertions(+), 42 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 8cf824110..f62000343 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -28,26 +28,20 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) de->row=0; de->parent_index=0; de->dir_modtime=0; + de->dir_hash=RsFileHash() ; // null hash is root by convention. mNodes.push_back(de) ; + #warning not very elegant. We should remove the leading / - mDirHashes[computeDirHash("/")] = 0 ; + mDirHashes[de->dir_hash] = 0 ; } -RsFileHash InternalFileHierarchyStorage::computeDirHash(const std::string& dir_path) -{ - return RsDirUtil::sha1sum((unsigned char*)dir_path.c_str(),dir_path.length()) ; -} bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) return false ; - DirEntry& d = *static_cast(mNodes[index]) ; - - hash = computeDirHash( d.dir_parent_path + "/" + d.dir_name ) ; - - std::cerr << "Computing dir hash from index " << index << ". Dir=\"" << d.dir_parent_path + "/" + d.dir_name << "\" hash=" << hash << std::endl; + hash = static_cast(mNodes[index])->dir_hash ; return true; } @@ -119,6 +113,10 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->row = mNodes.size(); de->parent_index = indx; de->dir_modtime = it->second; + de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; + de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ; + + mDirHashes[de->dir_hash] = mNodes.size() ; d.subdirs.push_back(mNodes.size()) ; mNodes.push_back(de) ; @@ -126,6 +124,24 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage return true; } + +RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name,const std::string& dir_parent_path) +{ + // What we need here: a unique identifier + // - that cannot be bruteforced to find the real directory name and path + // - that is used by friends to refer to a specific directory. + + // Option 1: compute H(some_secret_salt + dir_name + dir_parent_path) + // and keep the same salt so that we can re-create the hash + // + // Option 2: compute H(virtual_path). That only works at the level of LocalDirectoryStorage + // + // Option 3: just compute something random, but then we need to store it so as to not + // confuse friends when restarting. + + return RsFileHash::random(); +} + bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! { // check that it's a directory @@ -268,6 +284,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; + std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl; + d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); @@ -279,6 +297,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; @@ -303,6 +323,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI dir_index = found; mDirHashes[subdirs_hash[i]] = dir_index ; + + std::cerr << " created, at new index " << dir_index << std::endl; } else { @@ -313,6 +335,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI delete mNodes[dir_index] ; mNodes[dir_index] = NULL ; } + std::cerr << " already exists, index=" << dir_index << "." << std::endl; } FileStorageNode *& node(mNodes[dir_index]); if(!node) @@ -329,6 +352,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI DirectoryStorage::EntryIndex file_index = 0; std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; + std::cerr << " subfile hash " << i << ": " << subfiles_hash[i] ; + if(it == mFileHashes.end()) { // find an epty slot @@ -350,6 +375,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI file_index = found; mFileHashes[subfiles_hash[i]] = file_index ; + + std::cerr << " created, at new index " << file_index << std::endl; } else { @@ -362,8 +389,11 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } file_index = it->second; + + std::cerr << " already exists, index=" << file_index << "." << std::endl; } FileStorageNode *& node(mNodes[file_index]); + if(!node) node = new FileEntry("",0,0,subfiles_hash[i]); @@ -579,6 +609,14 @@ void InternalFileHierarchyStorage::print() const std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; recursPrint(0,DirectoryStorage::EntryIndex(0)); + + std::cerr << "Known dir hashes: " << std::endl; + for(std::map::const_iterator it(mDirHashes.begin());it!=mDirHashes.end();++it) + std::cerr << " " << it->first << " at index " << it->second << std::endl; + + std::cerr << "Known file hashes: " << std::endl; + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + std::cerr << " " << it->first << " at index " << it->second << std::endl; } void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::EntryIndex node) const { @@ -591,7 +629,7 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + std::cerr << indent << "dir hash=" << d.dir_hash << ". name:" << d.dir_name << ", parent_path:" << d.dir_parent_path << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; for(uint32_t i=0;i(mNodes[dir])) ; + RsFileHash hash = d.dir_hash ; + for(uint32_t i=0;irow = row ; mNodes[node_index] = fe ; + mFileHashes[fe->file_hash] = node_index ; } else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) { uint32_t node_index ; std::string dir_name ; std::string dir_parent_path ; + RsFileHash dir_hash ; uint32_t dir_modtime ; uint32_t dir_update_time ; uint32_t dir_most_recent_time ; @@ -795,6 +840,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; @@ -806,6 +852,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) DirEntry *de = new DirEntry(dir_name) ; de->dir_name = dir_name ; de->dir_parent_path = dir_parent_path ; + de->dir_hash = dir_hash ; de->dir_modtime = dir_modtime ; de->dir_update_time = dir_update_time ; de->dir_most_recent_time = dir_most_recent_time ; @@ -834,6 +881,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) de->subfiles.push_back(fi) ; } mNodes[node_index] = de ; + mDirHashes[de->dir_hash] = node_index ; } else throw std::runtime_error("Unknown node section.") ; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 5d7fb033a..d84e9e8d4 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -49,6 +49,7 @@ public: // local stuff std::string dir_name ; std::string dir_parent_path ; + RsFileHash dir_hash ; std::vector subdirs ; std::vector subfiles ; @@ -83,8 +84,6 @@ public: // hash stuff - static RsFileHash computeDirHash(const std::string& dir_path); - bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; @@ -118,6 +117,7 @@ public: private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; static bool nodeAccessError(const std::string& s); + static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning // phase. That allows to keep file information when moving them around. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index c8a0b0cc3..f27bfb16b 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -467,6 +467,58 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i return true; } +std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const +{ + if(indx == 0) + return std::string() ; + + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + if(dir->parent_index != 0) + return dir->dir_name ; + + std::map::const_iterator it = mLocalDirs.find(dir->dir_name) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl; + return std::string() ; + } + + return it->second.virtualname ; +} +std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const +{ + if(indx == 0) + return std::string() ; + + std::string res ; + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + while(dir->parent_index != 0) + { + dir = mFileHierarchy->getDirEntry(dir->parent_index) ; + res += dir->dir_name + "/"+ res ; + } + + std::map::const_iterator it = mLocalDirs.find(dir->dir_name) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl; + return std::string() ; + } + return it->second.virtualname + "/" + res; +} +RsFileHash LocalDirectoryStorage::locked_getDirHashFromIndex(EntryIndex indx) const +{ + // hash the full virtual path + + std::string virtual_path = locked_getVirtualPath(indx) ; + + return RsDirUtil::sha1sum((unsigned char*)virtual_path.c_str(),virtual_path.length()) ; +} + bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -489,13 +541,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - RsFileHash hash ; - - if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) - { - std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; - continue ; - } + RsFileHash hash = locked_getDirHashFromIndex(dir->subdirs[i]) ; allowed_subdirs.push_back(hash) ; } @@ -508,10 +554,11 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // - the index entry for each subdir (the updte TS are exchanged at a higher level) // - the file info for each subfile // + std::string virtual_dir_name = locked_getVirtualDirName(indx) ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fc9aadf7f..7a69eb7f6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -205,6 +205,10 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: + RsFileHash locked_getDirHashFromIndex(EntryIndex indx) const ; + std::string locked_getVirtualPath(EntryIndex indx) const ; + std::string locked_getVirtualDirName(EntryIndex indx) const ; + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index db4df001a..06eee527c 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -11,25 +11,31 @@ static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ; -static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; +static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; -static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; -static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; -static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; -static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; -static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; -static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; -static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; -static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; -static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; -static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x10 ; -static const uint8_t FILE_LIST_IO_TAG_ROW = 0x11 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x10 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x11 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x12 ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x13 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x20 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x21 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x22 ; + +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x30 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x31 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x32 ; + +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x40 ; +static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x41 ; + +static const uint8_t FILE_LIST_IO_TAG_DIR_HASH = 0x50 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x51 ; + +static const uint8_t FILE_LIST_IO_TAG_ROW = 0x60 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x61 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 60312124e..c0c603ecc 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -154,9 +154,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; -#ifdef DEBUG_FILE_HIERARCHY +//#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); -#endif +//#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI From 88f85383df2e93283b543daf5d0debefabd99b65 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 10 Sep 2016 17:25:29 +0200 Subject: [PATCH 56/92] fixed a few bugs in hash-based syncing --- .../src/file_sharing/dir_hierarchy.cc | 11 +++++++++++ .../src/file_sharing/dir_hierarchy.h | 1 + .../src/file_sharing/directory_storage.cc | 19 ++++++++----------- .../src/file_sharing/directory_storage.h | 1 - libretroshare/src/file_sharing/p3filelists.cc | 17 ++++++++++++----- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index f62000343..5765dd394 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -55,6 +55,16 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di index = it->second; return true; } +bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +{ + std::map::const_iterator it = mFileHashes.find(hash) ; + + if(it == mFileHashes.end()) + return false; + + index = it->second; + return true; +} int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) { @@ -344,6 +354,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.subdirs.push_back(dir_index) ; ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + ((DirEntry*&)node)->dir_hash = subdirs_hash[i]; node->row = i ; node->parent_index = indx ; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index d84e9e8d4..7d7eb35af 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -86,6 +86,7 @@ public: bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; + bool getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f27bfb16b..fae385d24 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -510,14 +510,6 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const } return it->second.virtualname + "/" + res; } -RsFileHash LocalDirectoryStorage::locked_getDirHashFromIndex(EntryIndex indx) const -{ - // hash the full virtual path - - std::string virtual_path = locked_getVirtualPath(indx) ; - - return RsDirUtil::sha1sum((unsigned char*)virtual_path.c_str(),virtual_path.length()) ; -} bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { @@ -541,7 +533,12 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - RsFileHash hash = locked_getDirHashFromIndex(dir->subdirs[i]) ; + RsFileHash hash ; + if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) + { + std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ; + return false; + } allowed_subdirs.push_back(hash) ; } @@ -725,9 +722,9 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co { DirectoryStorage::EntryIndex file_index ; - if(!getIndexFromDirHash(subfiles_hash[i],file_index)) + if(!mFileHierarchy->getIndexFromFileHash(subfiles_hash[i],file_index)) { - std::cerr << "(EE) Cannot optain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; + std::cerr << "(EE) Cannot obtain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; continue; } std::cerr << " updating file entry " << subfiles_hash[i] << std::endl; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7a69eb7f6..78fcb46ba 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -205,7 +205,6 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: - RsFileHash locked_getDirHashFromIndex(EntryIndex indx) const ; std::string locked_getVirtualPath(EntryIndex indx) const ; std::string locked_getVirtualDirName(EntryIndex indx) const ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c0c603ecc..07f3cd30b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1003,7 +1003,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } - std::cerr << " entry index is " << entry_index ; + std::cerr << " entry index is " << entry_index << " " ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) @@ -1029,9 +1029,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; std::cerr << " new content after update: " << std::endl; -#ifdef DEBUG_FILE_HIERARCHY +//#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[fi]->print(); -#endif +//#endif } } @@ -1097,7 +1097,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // Dont recurs into sub-directories, since we dont know yet were to go. - return ; + //return ; } for(DirectoryStorage::DirIterator it(rds,e);it;++it) @@ -1113,8 +1113,15 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId // that cannot be brute-forced or reverse-engineered, which explains the random bias. for(uint32_t i=0;i Date: Sat, 10 Sep 2016 20:47:51 +0200 Subject: [PATCH 57/92] fixed last bugs in hash-based dir list exchange. It works! --- .../src/file_sharing/dir_hierarchy.cc | 192 ++++++++++-------- .../src/file_sharing/dir_hierarchy.h | 10 +- .../src/file_sharing/directory_storage.cc | 45 ++-- libretroshare/src/file_sharing/p3filelists.cc | 4 +- 4 files changed, 133 insertions(+), 118 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 5765dd394..0ad931077 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -285,7 +285,26 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex return true; } -bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_hash,const std::vector& subfiles_hash) +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex() +{ + int found = -1; + for(uint32_t j=0;j& subdirs_hash,const std::vector& subfiles_array) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) { @@ -301,118 +320,131 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.dir_update_time = time(NULL); d.dir_name = dir_name; + std::map existing_subdirs ; + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_hash] = d.subdirs[i] ; + d.subdirs.clear(); - d.subfiles.clear(); // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::iterator it = existing_subdirs.find(subdirs_hash[i]) ; DirectoryStorage::EntryIndex dir_index = 0; - std::map::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; - if(it == mDirHashes.end() || it->second >= mNodes.size()) + if(it != existing_subdirs.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_DIR) { - // find an epty slot - int found = -1 ; + dir_index = it->second ; - for(uint32_t j=0;jdir_parent_path = d.dir_parent_path + "/" + dir_name ; + de->dir_hash = subdirs_hash[i]; mDirHashes[subdirs_hash[i]] = dir_index ; std::cerr << " created, at new index " << dir_index << std::endl; } - else - { - dir_index = it->second; - - if(mNodes[dir_index] != NULL && mNodes[dir_index]->type() != FileStorageNode::TYPE_DIR) - { - delete mNodes[dir_index] ; - mNodes[dir_index] = NULL ; - } - std::cerr << " already exists, index=" << dir_index << "." << std::endl; - } - FileStorageNode *& node(mNodes[dir_index]); - if(!node) - node = new DirEntry(""); d.subdirs.push_back(dir_index) ; - - ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; - ((DirEntry*&)node)->dir_hash = subdirs_hash[i]; - node->row = i ; - node->parent_index = indx ; + mDirHashes[subdirs_hash[i]] = dir_index ; } - for(uint32_t i=0;i::const_iterator it = existing_subdirs.begin();it!=existing_subdirs.end();++it) { - DirectoryStorage::EntryIndex file_index = 0; - std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; - std::cerr << " subfile hash " << i << ": " << subfiles_hash[i] ; - - if(it == mFileHashes.end()) + if(!checkIndex(it->second,FileStorageNode::TYPE_DIR)) { - // find an epty slot - int found = -1; - - for(uint32_t j=0;jsecond << " because it is not a file. Inconsistency error!" << std::endl; + continue ; } - else + recursRemoveDirectory(it->second) ; + } + + // now update subfiles. This is more stricky because we need to not suppress hash duplicates + + std::map existing_subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name] = d.subfiles[i] ; + + d.subfiles.clear(); + + for(uint32_t i=0;i::iterator it = existing_subfiles.find(subfiles_array[i].file_name) ; + const FileEntry& f(subfiles_array[i]) ; + DirectoryStorage::EntryIndex file_index ; + + std::cerr << " subfile name = " << subfiles_array[i].file_name << ": " ; + + if(it != existing_subfiles.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_FILE) { file_index = it->second ; - if(mNodes[file_index] != NULL && mNodes[file_index]->type() != FileStorageNode::TYPE_FILE) - { - delete mNodes[file_index] ; - mNodes[file_index] = NULL ; - } + std::cerr << " already exists, at index " << file_index << std::endl; - file_index = it->second; + if(!updateFile(file_index,f.file_hash,f.file_name,f.file_size,f.file_modtime)) + std::cerr << "(EE) Cannot update file with index " << it->second <<" and hash " << f.file_hash << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - std::cerr << " already exists, index=" << file_index << "." << std::endl; + existing_subfiles.erase(it) ; } - FileStorageNode *& node(mNodes[file_index]); + else + { + file_index = allocateNewIndex() ; - if(!node) - node = new FileEntry("",0,0,subfiles_hash[i]); + mNodes[file_index] = new FileEntry(f.file_name,f.file_size,f.file_modtime,f.file_hash) ; + mFileHashes[f.file_hash] = file_index ; + + std::cerr << " created, at new index " << file_index << std::endl; + } d.subfiles.push_back(file_index) ; - - node->row = subdirs_hash.size()+i ; - node->parent_index = indx ; } + // remove subfiles that do not exist anymore + + for(std::map::const_iterator it = existing_subfiles.begin();it!=existing_subfiles.end();++it) + { + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; + + if(!checkIndex(it->second,FileStorageNode::TYPE_FILE)) + { + std::cerr << "(EE) Cannot delete node of index " << it->second << " because it is not a file. Inconsistency error!" << std::endl; + continue ; + } + delete mNodes[it->second] ; + mNodes[it->second] = NULL ; + } + + // now update row and parent index for all subnodes + + uint32_t n=0; + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subdirs[i]]->row = n++ ; + } + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subfiles[i]]->row = n++ ; + } + return true; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 7d7eb35af..33c5a562e 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -25,6 +25,7 @@ public: class FileEntry: public FileStorageNode { public: + FileEntry() : file_size(0), file_modtime(0) {} FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {} @@ -74,7 +75,7 @@ public: bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_hash); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_array); bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); @@ -120,14 +121,19 @@ private: static bool nodeAccessError(const std::string& s); static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; + // Allocates a new entry in mNodes, possible re-using an empty slot and returns its index. + + DirectoryStorage::EntryIndex allocateNewIndex(); + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning // phase. That allows to keep file information when moving them around. bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // Map of the hash of all files. The file hashes are the sha1sum of the file data. // is used for fast search access for FT. // Note: We should try something faster than std::map. hash_map?? + // Unlike directories, multiple files may have the same hash. So this cannot be used for anything else than FT. std::map mFileHashes ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index fae385d24..933660986 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -671,11 +671,8 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co } // deserialise directory subfiles, with info for each of them - std::vector subfiles_array ; - std::vector subfiles_name ; - std::vector subfiles_size ; - std::vector subfiles_hash ; - std::vector subfiles_modtime ; + + std::vector subfiles_array ; for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_hash)) + if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array)) { std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; return false ; } - // Then update the subfiles - for(uint32_t i=0;igetIndexFromFileHash(subfiles_hash[i],file_index)) - { - std::cerr << "(EE) Cannot obtain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; - continue; - } - std::cerr << " updating file entry " << subfiles_hash[i] << std::endl; - - if(!mFileHierarchy->updateFile(file_index,subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) - std::cerr << "(EE) Cannot update file with index " << file_index <<" and hash " << subfiles_hash[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - } mChanged = true ; return true ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 07f3cd30b..c7b659a85 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1106,6 +1106,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { +#warning needs to be improved. It's quite likely that random_bias and then e can be bruteforced from the result of this function static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; @@ -1117,11 +1118,8 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId r ^= (0x011933ff92892a94 * e + peer_id.toByteArray()[i] * 0x1001fff92ee640f9) ; r <<= 8 ; r ^= 0xf392843890321808; - - std::cerr << std::hex << "r=" << r << std::endl; } - std::cerr << "making request ID: peer_id=" << peer_id << ", e=" << e << ", random_bias=" << std::hex<< random_bias << " returning " << (r^random_bias) << std::dec << std::endl; return r ^ random_bias; } From c1e538ff131b472e8a4f86aa71488f892f3c83ff Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 11 Sep 2016 17:52:12 +0200 Subject: [PATCH 58/92] added interactive update of dir entries when browsing files from an online friend --- libretroshare/src/file_sharing/p3filelists.cc | 146 ++++++++++++------ libretroshare/src/file_sharing/p3filelists.h | 13 +- libretroshare/src/ft/ftserver.cc | 7 +- libretroshare/src/ft/ftserver.h | 1 + libretroshare/src/retroshare/rsfiles.h | 2 + retroshare-gui/src/gui/RemoteDirModel.cpp | 5 + retroshare-gui/src/gui/RemoteDirModel.h | 9 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 9 +- retroshare-gui/src/gui/SharedFilesDialog.h | 1 + 9 files changed, 136 insertions(+), 57 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c7b659a85..9a2663991 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -192,7 +192,7 @@ int p3FileDatabase::tick() mRemoteDirectories[i]->print(); #endif - locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root(),0) ; } mRemoteDirectories[i]->checkSave() ; @@ -549,6 +549,30 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi return true; } +void p3FileDatabase::requestDirUpdate(void *ref) +{ + uint32_t fi; + DirectoryStorage::EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi); + + if(fi == 0) + return ; // not updating current directory (should we?) + + time_t recurs_max_modf_TS_remote_time,local_update_TS; + + std::cerr << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; + + if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + { + std::cerr << " (EE) Cannot get max known recurs modf time!" << std::endl; + return ; + } + + if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e,recurs_max_modf_TS_remote_time)) + P3FILELISTS_DEBUG() << " Succeed." << std::endl; +} + // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const @@ -1035,20 +1059,24 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) } } -void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) +void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e,int depth) { time_t now = time(NULL) ; + std::string indent(2*depth,' ') ; + // if not up to date, request update, and return (content is not certified, so no need to recurs yet). // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. // get the info for this entry + P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; + time_t recurs_max_modf_TS_remote_time,local_update_TS; if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + std::cerr << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; } @@ -1056,73 +1084,89 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. { - // check if a request already exists and is not too old either: no need to re-ask. + if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) + P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ; + // Dont recurs into sub-directories, since we dont know yet were to go. - std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; - - if(it != mPendingSyncRequests.end()) - { - P3FILELISTS_DEBUG() << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; - return ; - } - - P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - - RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - - if(!rds->getDirHashFromIndex(e,item->entry_hash) ) - { - std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; - return; - } - - item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; - item->request_id = sync_req_id ; - item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; - item->PeerId(rds->peerId()) ; - - DirSyncRequestData data ; - - data.request_TS = now ; - data.peer_id = item->PeerId(); - data.flags = item->flags; - - std::cerr << "Pushing req in pending list with peer id " << data.peer_id << std::endl; - - mPendingSyncRequests[sync_req_id] = data ; - - sendItem(item) ; // at end! Because item is destroyed by the process. - - // Dont recurs into sub-directories, since we dont know yet were to go. - - //return ; + //return ; } for(DirectoryStorage::DirIterator it(rds,e);it;++it) - locked_recursSweepRemoteDirectory(rds,*it); + locked_recursSweepRemoteDirectory(rds,*it,depth+1); } -p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) +p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,const RsFileHash& hash) { -#warning needs to be improved. It's quite likely that random_bias and then e can be bruteforced from the result of this function static uint64_t random_bias = RSRandom::random_u64(); - uint64_t r = e ; + uint64_t r = 0 ; - // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way - // that cannot be brute-forced or reverse-engineered, which explains the random bias. + // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,entry index) pair, in a way + // that cannot be brute-forced or reverse-engineered, which explains the random bias and the usage of the hash, that is itself random. for(uint32_t i=0;igetDirHashFromIndex(e,entry_hash) ) + { + std::cerr << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + return false; + } + + // check if a request already exists and is not too old either: no need to re-ask. + + DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),entry_hash) ; + + std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; + + if(it != mPendingSyncRequests.end()) + { + P3FILELISTS_DEBUG() << " Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + return false ; + } + + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + + item->entry_hash = entry_hash ; + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; + item->request_id = sync_req_id ; + item->last_known_recurs_modf_TS = max_known_recurs_modf_time ; + item->PeerId(rds->peerId()) ; + + DirSyncRequestData data ; + + data.request_TS = now ; + data.peer_id = item->PeerId(); + data.flags = item->flags; + + std::cerr << " Pushing req in pending list with peer id " << data.peer_id << std::endl; + + mPendingSyncRequests[sync_req_id] = data ; + + sendItem(item) ; // at end! Because item is destroyed by the process. + + return true; +} + + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index ec5f1b517..8861ed26d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -92,6 +92,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. + void requestDirUpdate(void *ref) ; // triggers an update. Used when browsing. int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; @@ -142,7 +143,15 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub typedef uint64_t DirSyncRequestId ; - static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ; + static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ; + + /*! + * \brief generateAndSendSyncRequest + * \param rds Remote directory storage for the request + * \param e Entry index to update + * \return true if the request is correctly sent. + */ + bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -185,7 +194,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub time_t mLastRemoteDirSweepTS ; // TS for friend list update std::map mPendingSyncRequests ; // pending requests, waiting for an answer - void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); + void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth); // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 1268ea831..4b4d4c1e3 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -345,7 +345,12 @@ bool ftServer::FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& return mFtController->getFileDownloadChunksDetails(hash,info); } - /* Directory Handling */ +void ftServer::requestDirUpdate(void *ref) +{ + mFileDatabase->requestDirUpdate(ref) ; +} + + /* Directory Handling */ void ftServer::setDownloadDirectory(std::string path) { mFtController->setDownloadDirectory(path); diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 4e5ae3a61..da1deb009 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -199,6 +199,7 @@ public: /*** * Directory Handling ***/ + virtual void requestDirUpdate(void *ref) ; // triggers the update of the given reference. Used when browsing. virtual void setDownloadDirectory(std::string path); virtual void setPartialsDirectory(std::string path); virtual std::string getDownloadDirectory(); diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index cf1bfc157..d49324933 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -205,6 +205,8 @@ class RsFiles /*** * Directory Control ***/ + virtual void requestDirUpdate(void *ref) =0 ; // triggers the update of the given reference. Used when browsing. + virtual void setDownloadDirectory(std::string path) = 0; virtual void setPartialsDirectory(std::string path) = 0; virtual std::string getDownloadDirectory() = 0; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index eb5800867..2153f255d 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -80,6 +80,11 @@ void RetroshareDirModel::treeStyle() peerIcon = QIcon(":/images/user/identity16.png"); } +void TreeStyle_RDM::updateRef(const QModelIndex& indx) const +{ + rsFiles->requestDirUpdate(indx.internalPointer()) ; +} + bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const { diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index 062a49a43..54f60ffad 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -78,6 +78,8 @@ class RetroshareDirModel : public QAbstractItemModel bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} + virtual void updateRef(const QModelIndex&) const =0; + public: virtual QMimeData * mimeData ( const QModelIndexList & indexes ) const; virtual QStringList mimeTypes () const; @@ -167,7 +169,9 @@ class TreeStyle_RDM: public RetroshareDirModel virtual ~TreeStyle_RDM() ; protected: - /* These are all overloaded Virtual Functions */ + virtual void updateRef(const QModelIndex&) const ; + + /* These are all overloaded Virtual Functions */ virtual int rowCount(const QModelIndex &parent = QModelIndex()) const; virtual int columnCount(const QModelIndex &parent = QModelIndex()) const; @@ -203,7 +207,8 @@ class FlatStyle_RDM: public RetroshareDirModel void updateRefs() ; protected: - virtual void postMods(); + virtual void updateRef(const QModelIndex&) const {} + virtual void postMods(); virtual int rowCount(const QModelIndex &parent = QModelIndex()) const; virtual int columnCount(const QModelIndex &parent = QModelIndex()) const; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index a0864839a..aaaa79284 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -228,6 +228,7 @@ RemoteSharedFilesDialog::RemoteSharedFilesDialog(QWidget *parent) ui.checkButton->hide() ; connect(ui.downloadButton, SIGNAL(clicked()), this, SLOT(downloadRemoteSelected())); + connect(ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); // load settings processSettings(true); @@ -366,7 +367,6 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) } // connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), this, SLOT( collapsed(const QModelIndex & ) ) ); -// connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); @@ -517,6 +517,13 @@ QModelIndexList SharedFilesDialog::getSelected() return proxyList ; } +void RemoteSharedFilesDialog::expanded(const QModelIndex& indx) +{ + std::cerr << "Expanding at " << indx.row() << " and " << indx.column() << " ref=" << indx.internalPointer() << ", pointer at 1: " << proxyModel->mapToSource(indx).internalPointer() << std::endl; + + model->updateRef(proxyModel->mapToSource(indx)) ; +} + void RemoteSharedFilesDialog::downloadRemoteSelected() { /* call back to the model (which does all the interfacing? */ diff --git a/retroshare-gui/src/gui/SharedFilesDialog.h b/retroshare-gui/src/gui/SharedFilesDialog.h index 9b7387224..594df6fab 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.h +++ b/retroshare-gui/src/gui/SharedFilesDialog.h @@ -196,6 +196,7 @@ class RemoteSharedFilesDialog : public SharedFilesDialog private slots: void downloadRemoteSelected(); + void expanded(const QModelIndex& indx); }; #endif From 274f924ca01674ef9a8c3220cd91025cd4295ba1 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 12 Sep 2016 20:55:20 +0200 Subject: [PATCH 59/92] fixed getFileInfo and transfer from new database --- .../src/file_sharing/dir_hierarchy.cc | 4 +++ .../src/file_sharing/directory_storage.cc | 31 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 0ad931077..d58d91170 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -260,6 +260,7 @@ bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + mFileHashes[hash] = file_index ; old_hash = hash ; @@ -282,6 +283,9 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex fe.file_modtime = modf_time; fe.file_name = fname; + if(!hash.isNull()) + mFileHashes[hash] = file_index ; + return true; } diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 933660986..779a2c04d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -185,9 +185,36 @@ void DirectoryStorage::print() bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { - RS_STACK_MUTEX(mDirStorageMtx) ; + DirDetails d; + extractData(i,d) ; + + info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. + info.parent_groups = d.parent_groups; + info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* + info.path = d.path + "/" + d.name; + info.fname = d.name; + info.hash = d.hash; + info.size = d.count; + + // all this stuff below is not useful in this case. + + info.mId = 0; /* (GUI) Model Id -> unique number */ + info.ext.clear(); + info.avail = 0; /* how much we have */ + info.rank = 0; + info.age = 0; + info.queue_position =0; + info.searchId = 0; /* 0 if none */ + + /* Transfer Stuff */ + info.transfered = 0; + info.tfRate = 0; /* in kbytes */ + info.downloadStatus = FT_STATE_COMPLETE ; + std::list peers; + + info.priority = SPEED_NORMAL; + info.lastTS = 0; - NOT_IMPLEMENTED(); return true; } From 9f66c0050b63bca5e22119cf6b090ea624379186 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 12 Sep 2016 23:37:19 +0200 Subject: [PATCH 60/92] moved rsexpr to file_sharing directory --- .../src/file_sharing/dir_hierarchy.cc | 44 +++++++++ .../src/file_sharing/dir_hierarchy.h | 5 ++ .../src/file_sharing/directory_storage.cc | 90 ++++++++++--------- .../src/file_sharing/directory_storage.h | 4 +- .../src/{dbase => file_sharing}/rsexpr.cc | 0 libretroshare/src/libretroshare.pro | 13 +-- libretroshare/src/retroshare/rstypes.h | 2 +- 7 files changed, 102 insertions(+), 56 deletions(-) rename libretroshare/src/{dbase => file_sharing}/rsexpr.cc (100%) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d58d91170..778d1cffb 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -2,6 +2,7 @@ #include "dir_hierarchy.h" #include "filelist_io.h" +//#include "rsexpr.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -570,6 +571,49 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const +{ +#ifdef TODO + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + { + const std::string &str1 = static_cast(mNodes[fit->second])->file_name; + + { + /*Evaluate the boolean expression and add it to the results if its true*/ + bool ret = exp->eval(fit->second); + if (ret == true){ + results.push_back(fit->second); + } + } +#endif +} + +int InternalFileHierarchyStorage::searchTerms(const std::list& terms, std::list &results) const +{ + // most entries are likely to be files, so we could do a linear search over the entries tab. + // instead we go through the table of hashes. + + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + if(mNodes[it->second] != NULL) + { + const std::string &str1 = static_cast(mNodes[it->second])->file_name; + + for(std::list::const_iterator iter(terms.begin()); iter != terms.end(); ++iter) + { + /* always ignore case */ + const std::string &str2 = (*iter); +#ifdef TODO + if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC() )) + { + results.push_back(fit->second); + break; + } +#endif + } + } + return 0 ; +} + bool InternalFileHierarchyStorage::check(std::string& error_string) const // checks consistency of storage. { // recurs go through all entries, check that all diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 33c5a562e..f2e81844f 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -110,7 +110,12 @@ public: uint32_t getType(DirectoryStorage::EntryIndex indx) const; DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index); DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index); + + // search. SearchHash is logarithmic. The other two are linear. + bool searchHash(const RsFileHash& hash,std::list& results); + int searchBoolExp(Expression * exp, std::list &results) const ; + int searchTerms(const std::list& terms, std::list &results) const ; bool check(std::string& error_string) const ;// checks consistency of storage. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 779a2c04d..4130481df 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -160,62 +160,29 @@ void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_ void DirectoryStorage::load(const std::string& local_file_name) { - std::cerr << "DirectoryStorage::load()" << std::endl; - - // first load the header, than all fields. - RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) { - std::cerr << "DirectoryStorage::Save()" << std::endl; - RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy->save(local_file_name); - - // first write the header, than all fields. } void DirectoryStorage::print() { RS_STACK_MUTEX(mDirStorageMtx) ; - std::cerr << "LocalDirectoryStorage:" << std::endl; mFileHierarchy->print(); } -bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +int DirectoryStorage::searchTerms(const std::list& terms, std::list &results) const { - DirDetails d; - extractData(i,d) ; - - info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. - info.parent_groups = d.parent_groups; - info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* - info.path = d.path + "/" + d.name; - info.fname = d.name; - info.hash = d.hash; - info.size = d.count; - - // all this stuff below is not useful in this case. - - info.mId = 0; /* (GUI) Model Id -> unique number */ - info.ext.clear(); - info.avail = 0; /* how much we have */ - info.rank = 0; - info.age = 0; - info.queue_position =0; - info.searchId = 0; /* 0 if none */ - - /* Transfer Stuff */ - info.transfered = 0; - info.tfRate = 0; /* in kbytes */ - info.downloadStatus = FT_STATE_COMPLETE ; - std::list peers; - - info.priority = SPEED_NORMAL; - info.lastTS = 0; - - return true; + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchTerms(terms,results); +} +int DirectoryStorage::searchBoolExp(Expression * exp, std::list &results) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchBoolExp(exp,results); } bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) @@ -444,6 +411,47 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; } +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + DirDetails d; + extractData(i,d) ; + + if(d.type != DIR_TYPE_FILE) + { + std::cerr << "(EE) LocalDirectoryStorage: asked for file info for index " << i << " which is not a file." << std::endl; + return false; + } + + info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. + info.parent_groups = d.parent_groups; + info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* + info.path = d.path + "/" + d.name; + info.fname = d.name; + info.hash = d.hash; + info.size = d.count; + + // all this stuff below is not useful in this case. + + info.mId = 0; /* (GUI) Model Id -> unique number */ + info.ext.clear(); + info.avail = 0; /* how much we have */ + info.rank = 0; + info.age = 0; + info.queue_position =0; + info.searchId = 0; /* 0 if none */ + + /* Transfer Stuff */ + info.transfered = 0; + info.tfRate = 0; /* in kbytes */ + info.downloadStatus = FT_STATE_COMPLETE ; + std::list peers; + + info.priority = SPEED_NORMAL; + info.lastTS = 0; + + return true; +} + bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 78fcb46ba..7c88ffe21 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -24,9 +24,9 @@ class DirectoryStorage void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} + virtual int searchTerms(const std::list& terms, std::list &results) const ; + virtual int searchBoolExp(Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; diff --git a/libretroshare/src/dbase/rsexpr.cc b/libretroshare/src/file_sharing/rsexpr.cc similarity index 100% rename from libretroshare/src/dbase/rsexpr.cc rename to libretroshare/src/file_sharing/rsexpr.cc diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 56d36469c..f839ab2d5 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -55,6 +55,7 @@ file_lists { file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ file_sharing/dir_hierarchy.cc \ + file_sharing/rsexpr.cc \ file_sharing/rsfilelistitems.cc } @@ -366,11 +367,6 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR} PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2 -HEADERS += dbase/cachestrapper.h \ - dbase/fimonitor.h \ - dbase/findex.h \ - dbase/fistore.h - HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ @@ -532,13 +528,6 @@ HEADERS += util/folderiterator.h \ util/rsscopetimer.h \ util/stacktrace.h -SOURCES += dbase/cachestrapper.cc \ - dbase/fimonitor.cc \ - dbase/findex.cc \ - dbase/fistore.cc \ - dbase/rsexpr.cc - - SOURCES += ft/ftchunkmap.cc \ ft/ftcontroller.cc \ ft/ftdatamultiplex.cc \ diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 8ca770a76..0a8537bb7 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -236,7 +236,7 @@ public: RsPeerId id; std::string name; RsFileHash hash; - std::string path; + std::string path; // full path of the parent directory, when it is a file; full path of the dir otherwise. uint64_t count; uint32_t age; FileStorageFlags flags; From e9418bb5c65aae018e965ad06764c3aecb29528f Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 00:01:26 +0200 Subject: [PATCH 61/92] added file headers with proper licence terms --- .../src/file_sharing/dir_hierarchy.cc | 24 +++++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 26 ++++++++++++++++++ .../src/file_sharing/directory_storage.cc | 24 +++++++++++++++++ .../src/file_sharing/directory_storage.h | 25 +++++++++++++++++ .../src/file_sharing/directory_updater.cc | 24 +++++++++++++++++ .../src/file_sharing/directory_updater.h | 26 ++++++++++++++++++ .../src/file_sharing/file_sharing_defaults.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/hash_cache.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/hash_cache.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/p3filelists.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/p3filelists.h | 27 +++++++++++++++++++ .../src/file_sharing/rsfilelistitems.cc | 24 +++++++++++++++++ .../src/file_sharing/rsfilelistitems.h | 11 ++++---- 15 files changed, 355 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 778d1cffb..aa1e351ae 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Internal directory storage class. + * + * file_sharing/dir_hierarchy.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/rsdir.h" #include "dir_hierarchy.h" diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f2e81844f..f01f2c056 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Internal directory hierarchy class. + * + * file_sharing/dir_hierarchy.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 4130481df..2adc54498 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File list storage system. + * + * file_sharing/directory_storage.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include #include "serialiser/rstlvbinary.h" #include "retroshare/rspeers.h" diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7c88ffe21..cfec2a018 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,3 +1,28 @@ +/* + * RetroShare C++ Directory Storage system. + * + * file_sharing/directory_storage.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + #pragma once #include diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index adc3e8fff..23c1fe7c6 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Directory watching system. + * + * file_sharing/directory_updater.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/folderiterator.h" #include "rsserver/p3face.h" diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 3fb52c81e..25f94d28a 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Directory parsing code. + * + * file_sharing/directory_updater.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + // This class crawls the given directry hierarchy and updates it. It does so by calling the // shared file list source. This source may be of two types: // - local: directories are crawled n disk and files are hashed / requested from a cache diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 5bc404c5f..4c47584e9 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ File sharing default variables + * + * file_sharing/file_sharing_defaults.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 47d8899e3..f4cd1f6be 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists IO methods. + * + * file_sharing/filelist_io.h + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "retroshare/rsids.h" #include "pqi/authssl.h" #include "util/rsdir.h" diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 06eee527c..f02d07ed7 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ File lists IO methods. + * + * file_sharing/filelist_io.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index c2ed3ed4e..f92964ac4 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Hash cache + * + * file_sharing/hash_cache.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/rsdir.h" #include "util/rsprint.h" #include "rsserver/p3face.h" diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6413ce68c..467c18d6a 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Hash cache. + * + * file_sharing/hash_cache.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9a2663991..db73972e2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists service. + * + * file_sharing/p3filelists.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "serialiser/rsserviceids.h" #include "file_sharing/p3filelists.h" diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 8861ed26d..b15a74488 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -1,3 +1,30 @@ +/* + * RetroShare C++ File lists service. + * + * file_sharing/p3filelists.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + +// // This class is responsible for // - maintaining a list of shared file hierarchies for each known friends // - talking to the GUI diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 841e1d1a7..b3fd1db59 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists service items + * + * file_sharing/rsfilelistsitems.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "serialiser/rsbaseserial.h" #include "file_sharing/rsfilelistitems.h" diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 5644d4648..deaa06e60 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -1,11 +1,9 @@ -#pragma once - /* - * libretroshare/src/serialiser: rsnxssitems.h + * RetroShare File lists service items. * - * RetroShare Serialiser. + * file_sharing/rsfilelistitems.h * - * Copyright 2012 Christopher Evi-Parker, Robert Fernie. + * Copyright 2016 Mr.Alice * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -21,9 +19,10 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * USA. * - * Please report all bugs and problems to "retroshare@lunamutt.com". + * Please report all bugs and problems to "retroshare.project@gmail.com". * */ +#pragma once #include #include From a2e34f9cc62e12b8627720927284517ef05f00aa Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 12:05:22 +0200 Subject: [PATCH 62/92] moved regular expression classes into a separate namespace --- libresapi/src/api/FileSearchHandler.cpp | 4 +- .../src/file_sharing/dir_hierarchy.cc | 2 +- .../src/file_sharing/dir_hierarchy.h | 2 +- .../src/file_sharing/directory_storage.cc | 2 +- .../src/file_sharing/directory_storage.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 2 +- libretroshare/src/file_sharing/p3filelists.h | 2 +- libretroshare/src/file_sharing/rsexpr.cc | 331 ++++++++------- libretroshare/src/ft/ftserver.cc | 4 +- libretroshare/src/ft/ftserver.h | 4 +- libretroshare/src/libretroshare.pro | 1 + libretroshare/src/retroshare/rsexpr.h | 382 +++++++++--------- libretroshare/src/retroshare/rsfiles.h | 6 +- libretroshare/src/retroshare/rsturtle.h | 4 +- libretroshare/src/turtle/p3turtle.cc | 4 +- libretroshare/src/turtle/p3turtle.h | 2 +- libretroshare/src/turtle/rsturtleitem.h | 2 +- .../src/gui/FileTransfer/SearchDialog.cpp | 8 +- .../src/gui/FileTransfer/SearchDialog.h | 5 +- .../gui/advsearch/advancedsearchdialog.cpp | 6 +- .../src/gui/advsearch/advancedsearchdialog.h | 4 +- .../src/gui/advsearch/expressionwidget.cpp | 30 +- .../src/gui/advsearch/expressionwidget.h | 4 +- .../src/gui/advsearch/guiexprelement.cpp | 36 +- .../src/gui/advsearch/guiexprelement.h | 16 +- 25 files changed, 439 insertions(+), 426 deletions(-) diff --git a/libresapi/src/api/FileSearchHandler.cpp b/libresapi/src/api/FileSearchHandler.cpp index 386db94c9..3833b4c9d 100644 --- a/libresapi/src/api/FileSearchHandler.cpp +++ b/libresapi/src/api/FileSearchHandler.cpp @@ -173,8 +173,8 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp) return; } - NameExpression exprs(ContainsAllStrings,words,true) ; - LinearizedExpression lin_exp ; + RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ; + RsRegularExpression::LinearizedExpression lin_exp ; exprs.linearize(lin_exp) ; uint32_t search_id = RSRandom::random_u32(); diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index aa1e351ae..1735beb79 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -595,7 +595,7 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const +int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const { #ifdef TODO for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f01f2c056..fb83e5d9d 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -140,7 +140,7 @@ public: // search. SearchHash is logarithmic. The other two are linear. bool searchHash(const RsFileHash& hash,std::list& results); - int searchBoolExp(Expression * exp, std::list &results) const ; + int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; int searchTerms(const std::list& terms, std::list &results) const ; bool check(std::string& error_string) const ;// checks consistency of storage. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2adc54498..2a7cd8f91 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -203,7 +203,7 @@ int DirectoryStorage::searchTerms(const std::list& terms, std::list RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->searchTerms(terms,results); } -int DirectoryStorage::searchBoolExp(Expression * exp, std::list &results) const +int DirectoryStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->searchBoolExp(exp,results); diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index cfec2a018..be420f618 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -50,7 +50,7 @@ class DirectoryStorage void save() const ; virtual int searchTerms(const std::list& terms, std::list &results) const ; - virtual int searchBoolExp(Expression * exp, std::list &results) const ; + virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index db73972e2..ea77de3a6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -814,7 +814,7 @@ int p3FileDatabase::SearchKeywords(const std::list& keywords, std:: return filterResults(firesults,results,flags,client_peer_id) ; } -int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const +int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index b15a74488..f832dfc53 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -103,7 +103,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; - virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; // ftSearch virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; diff --git a/libretroshare/src/file_sharing/rsexpr.cc b/libretroshare/src/file_sharing/rsexpr.cc index c9e1ff798..6b4d7ce20 100644 --- a/libretroshare/src/file_sharing/rsexpr.cc +++ b/libretroshare/src/file_sharing/rsexpr.cc @@ -26,6 +26,7 @@ #include "dbase/findex.h" #include "retroshare/rsexpr.h" +#include "retroshare/rstypes.h" #include #include @@ -34,33 +35,35 @@ eval functions of relational expressions. ******************************************************************************************/ +namespace RsRegularExpression +{ template<> void RelExpression::linearize(LinearizedExpression& e) const { - e._ints.push_back(Op) ; - e._ints.push_back(LowerValue) ; - e._ints.push_back(HigherValue) ; + e._ints.push_back(Op) ; + e._ints.push_back(LowerValue) ; + e._ints.push_back(HigherValue) ; } -bool DateExpression::eval(FileEntry *file) +bool DateExpression::eval(const ExpFileEntry& file) { - return evalRel(file->modtime); + return evalRel(file.file_modtime()); } -bool SizeExpressionMB::eval(FileEntry *file) +bool SizeExpressionMB::eval(const ExpFileEntry& file) { - return evalRel((int)(file->size/(uint64_t)(1024*1024))); + return evalRel((int)(file.file_size()/(uint64_t)(1024*1024))); } -bool SizeExpression::eval(FileEntry *file) +bool SizeExpression::eval(const ExpFileEntry& file) { - return evalRel(file->size); + return evalRel(file.file_size()); } -bool PopExpression::eval(FileEntry *file) +bool PopExpression::eval(const ExpFileEntry& file) { - return evalRel(file->pop); + return evalRel(file.file_popularity()); } /****************************************************************************************** @@ -68,98 +71,93 @@ Code for evaluating string expressions ******************************************************************************************/ -bool NameExpression::eval(FileEntry *file) +bool NameExpression::eval(const ExpFileEntry& file) { - return evalStr(file->name); + return evalStr(file.file_name()); } -bool PathExpression::eval(FileEntry *file){ - std::string path; - /*Construct the path of this file*/ - DirEntry * curr = file->parent; - while ( curr != NULL ){ - path = curr->name+"/"+ path; - curr = curr->parent; - } - return evalStr(path); +bool PathExpression::eval(const ExpFileEntry& file) +{ + return evalStr(file.file_parent_path()); } -bool ExtExpression::eval(FileEntry *file){ - std::string ext; - /*Get the part of the string after the last instance of . in the filename */ - size_t index = file->name.find_last_of('.'); - if (index != std::string::npos) { - ext = file->name.substr(index+1); - if (ext != "" ){ - return evalStr(ext); - } - } - return false; +bool ExtExpression::eval(const ExpFileEntry& file) +{ + std::string ext; + /*Get the part of the string after the last instance of . in the filename */ + size_t index = file.file_name().find_last_of('.'); + if (index != std::string::npos) { + ext = file.file_name().substr(index+1); + if (ext != "" ){ + return evalStr(ext); + } + } + return false; } -bool HashExpression::eval(FileEntry *file){ - return evalStr(file->hash.toStdString()); +bool HashExpression::eval(const ExpFileEntry& file){ + return evalStr(file.file_hash().toStdString()); } /*Check whether two strings are 'equal' to each other*/ static bool StrEquals(const std::string & str1, const std::string & str2, - bool IgnoreCase ){ - if ( str1.size() != str2.size() ){ - return false; - } else if (IgnoreCase) { - std::equal( str1.begin(), str1.end(), - str2.begin(), CompareCharIC() ); - } - return std::equal( str1.begin(), str1.end(), - str2.begin()); + bool IgnoreCase ){ + if ( str1.size() != str2.size() ){ + return false; + } else if (IgnoreCase) { + std::equal( str1.begin(), str1.end(), + str2.begin(), CompareCharIC() ); + } + return std::equal( str1.begin(), str1.end(), + str2.begin()); } /*Check whether one string contains the other*/ static bool StrContains( const std::string & str1, const std::string & str2, - bool IgnoreCase){ + bool IgnoreCase){ - std::string::const_iterator iter ; - if (IgnoreCase) { - iter = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end(), CompareCharIC() ); - } else { - iter = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end()); - } - - return ( iter != str1.end() ); + std::string::const_iterator iter ; + if (IgnoreCase) { + iter = std::search( str1.begin(), str1.end(), + str2.begin(), str2.end(), CompareCharIC() ); + } else { + iter = std::search( str1.begin(), str1.end(), + str2.begin(), str2.end()); + } + + return ( iter != str1.end() ); } bool StringExpression :: evalStr ( const std::string &str ){ - std::list::iterator iter; - switch (Op) { - case ContainsAllStrings: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrContains (str, *iter, IgnoreCase) == false ){ - return false; - } - } - return true; - break; - case ContainsAnyStrings: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrContains (str,*iter, IgnoreCase) == true ) { - return true; - } - } - break; - case EqualsString: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrEquals (str,*iter, IgnoreCase) == true ) { - return true; - } - } - break; - default: - return false; - } - return false; + std::list::iterator iter; + switch (Op) { + case ContainsAllStrings: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrContains (str, *iter, IgnoreCase) == false ){ + return false; + } + } + return true; + break; + case ContainsAnyStrings: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrContains (str,*iter, IgnoreCase) == true ) { + return true; + } + } + break; + case EqualsString: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrEquals (str,*iter, IgnoreCase) == true ) { + return true; + } + } + break; + default: + return false; + } + return false; } /************************************************************************* @@ -168,122 +166,123 @@ bool StringExpression :: evalStr ( const std::string &str ){ void CompoundExpression::linearize(LinearizedExpression& e) const { - e._tokens.push_back(LinearizedExpression::EXPR_COMP) ; - e._ints.push_back(Op) ; + e._tokens.push_back(LinearizedExpression::EXPR_COMP) ; + e._ints.push_back(Op) ; - Lexp->linearize(e) ; - Rexp->linearize(e) ; + Lexp->linearize(e) ; + Rexp->linearize(e) ; } void StringExpression::linearize(LinearizedExpression& e) const { - e._ints.push_back(Op) ; - e._ints.push_back(IgnoreCase) ; - e._ints.push_back(terms.size()) ; + e._ints.push_back(Op) ; + e._ints.push_back(IgnoreCase) ; + e._ints.push_back(terms.size()) ; - for(std::list::const_iterator it(terms.begin());it!=terms.end();++it) - e._strings.push_back(*it) ; + for(std::list::const_iterator it(terms.begin());it!=terms.end();++it) + e._strings.push_back(*it) ; } Expression *LinearizedExpression::toExpr(const LinearizedExpression& e) { - int i=0,j=0,k=0 ; - return toExpr(e,i,j,k) ; + int i=0,j=0,k=0 ; + return toExpr(e,i,j,k) ; } void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) { - op = static_cast(e._ints[n_ints++]) ; - b = e._ints[n_ints++] ; - int n = e._ints[n_ints++] ; + op = static_cast(e._ints[n_ints++]) ; + b = e._ints[n_ints++] ; + int n = e._ints[n_ints++] ; - strings.clear() ; - for(int i=0;i(e._tokens[n_tok++]) ; + LinearizedExpression::token tok = static_cast(e._tokens[n_tok++]) ; - switch(tok) - { - case EXPR_DATE: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + switch(tok) + { + case EXPR_DATE: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new DateExpression(op,lv,hv) ; - } + return new DateExpression(op,lv,hv) ; + } - case EXPR_POP: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + case EXPR_POP: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new PopExpression(op,lv,hv) ; - } - case EXPR_SIZE: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + return new PopExpression(op,lv,hv) ; + } + case EXPR_SIZE: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new SizeExpression(op,lv,hv) ; - } - case EXPR_HASH: { - std::list strings ; - StringOperator op ; - bool b ; + return new SizeExpression(op,lv,hv) ; + } + case EXPR_HASH: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new HashExpression(op,strings) ; - } - case EXPR_NAME: { - std::list strings ; - StringOperator op ; - bool b ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; + return new HashExpression(op,strings) ; + } + case EXPR_NAME: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new NameExpression(op,strings,b) ; - } - case EXPR_PATH: { - std::list strings ; - StringOperator op ; - bool b ; + return new NameExpression(op,strings,b) ; + } + case EXPR_PATH: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new ExtExpression(op,strings,b) ; - } - case EXPR_EXT: { - std::list strings ; - StringOperator op ; - bool b ; + return new ExtExpression(op,strings,b) ; + } + case EXPR_EXT: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new ExtExpression(op,strings,b) ; - } - case EXPR_COMP: { - LogicalOperator op = static_cast(e._ints[n_ints++]) ; + return new ExtExpression(op,strings,b) ; + } + case EXPR_COMP: { + LogicalOperator op = static_cast(e._ints[n_ints++]) ; - Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ; - Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ; + Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ; + Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ; - return new CompoundExpression(op,e1,e2) ; - } - case EXPR_SIZE_MB: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + return new CompoundExpression(op,e1,e2) ; + } + case EXPR_SIZE_MB: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new SizeExpressionMB(op,lv,hv) ; - } - default: - std::cerr << "No expression match the current value " << tok << std::endl ; - return NULL ; - } + return new SizeExpressionMB(op,lv,hv) ; + } + default: + std::cerr << "No expression match the current value " << tok << std::endl ; + return NULL ; + } } +} diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 4b4d4c1e3..1a558f9d6 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -569,11 +569,11 @@ int ftServer::SearchKeywords(std::list keywords, std::listSearchKeywords(keywords, results,flags,peer_id); } -int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) +int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags) { return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId()); } -int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) +int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ; } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index da1deb009..1173d8311 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -184,8 +184,8 @@ public: virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags); + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); /*** * Utility Functions diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index f839ab2d5..e30e3637e 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -59,6 +59,7 @@ file_lists { file_sharing/rsfilelistitems.cc } + dsdv { DEFINES *= SERVICES_DSDV HEADERS += services/p3dsdv.h \ diff --git a/libretroshare/src/retroshare/rsexpr.h b/libretroshare/src/retroshare/rsexpr.h index 87d216387..fc339e87c 100644 --- a/libretroshare/src/retroshare/rsexpr.h +++ b/libretroshare/src/retroshare/rsexpr.h @@ -1,6 +1,3 @@ -#ifndef RS_EXPRESSIONS_H -#define RS_EXPRESSIONS_H - /* * rs-core/src/rsiface: rsexpr.h * @@ -26,38 +23,41 @@ * */ +#pragma once #include #include #include +#include "retroshare/rstypes.h" /****************************************************************************************** Enumerations defining the Operators usable in the Boolean search expressions ******************************************************************************************/ - +namespace RsRegularExpression +{ enum LogicalOperator{ - AndOp=0, /* exp AND exp */ - OrOp=1, /* exp OR exp */ - XorOp=2 /* exp XOR exp */ + AndOp=0, /* exp AND exp */ + OrOp=1, /* exp OR exp */ + XorOp=2 /* exp XOR exp */ }; -/*Operators for String Queries*/ +/* Operators for String Queries */ enum StringOperator{ - ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */ - ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */ - EqualsString = 2 /* exactly equal*/ + ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */ + ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */ + EqualsString = 2 /* exactly equal*/ }; -/*Relational operators ( >, <, >=, <=, == and InRange )*/ +/* Comparison operators ( >, <, >=, <=, == and InRange ) */ enum RelOperator{ - Equals = 0, - GreaterEquals = 1, - Greater = 2, - SmallerEquals = 3, - Smaller = 4, - InRange = 5 /* lower limit <= value <= upper limit*/ + Equals = 0, + GreaterEquals = 1, + Greater = 2, + SmallerEquals = 3, + Smaller = 4, + InRange = 5 /* lower limit <= value <= upper limit*/ }; /******************************************************************************************** @@ -69,26 +69,26 @@ class Expression ; class LinearizedExpression { - public: - std::vector _tokens ; - std::vector _ints ; - std::vector _strings ; +public: + std::vector _tokens ; + std::vector _ints ; + std::vector _strings ; - typedef enum { EXPR_DATE= 0, - EXPR_POP = 1, - EXPR_SIZE= 2, - EXPR_HASH= 3, - EXPR_NAME= 4, - EXPR_PATH= 5, - EXPR_EXT = 6, - EXPR_COMP= 7, - EXPR_SIZE_MB=8 } token ; + typedef enum { EXPR_DATE = 0, + EXPR_POP = 1, + EXPR_SIZE = 2, + EXPR_HASH = 3, + EXPR_NAME = 4, + EXPR_PATH = 5, + EXPR_EXT = 6, + EXPR_COMP = 7, + EXPR_SIZE_MB = 8 } token ; - static Expression *toExpr(const LinearizedExpression& e) ; + static Expression *toExpr(const LinearizedExpression& e) ; - private: - static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ; - static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) ; +private: + static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ; + static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) ; }; @@ -96,110 +96,122 @@ class LinearizedExpression Boolean Search Expression classes: - Expression: The base class of all expression typest - CompoundExpression: The expression which uses a logical operator to combine - the results of two expressions - StringExpression: An expression which uses some sort of string comparison. - RelExpression: A Relational Expression where > < >= <= == make sense. - e.g. size date etc + Expression: The base class of all expression typest + CompoundExpression: The expression which uses a logical operator to combine + the results of two expressions + StringExpression: An expression which uses some sort of string comparison. + RelExpression: A Relational Expression where > < >= <= == make sense. + e.g. size date etc ******************************************************************************************/ -class FileEntry; +/*! + * \brief The ExpFileEntry class + * This is the base class the regular expressions can operate on. Derive it to fit your own needs! + */ +class ExpFileEntry +{ +public: + virtual const std::string& file_name() const =0; + virtual uint64_t file_size() const =0; + virtual time_t file_modtime() const =0; + virtual time_t file_popularity() const =0; + virtual const std::string file_parent_path() const =0; + virtual const RsFileHash& file_hash() const =0; +}; class Expression { - public: - virtual bool eval (FileEntry *file) = 0; - virtual ~Expression() {}; +public: + virtual bool eval (const ExpFileEntry& file) = 0; + virtual ~Expression() {}; - virtual void linearize(LinearizedExpression& e) const = 0 ; + virtual void linearize(LinearizedExpression& e) const = 0 ; }; - class CompoundExpression : public Expression { - public: - CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2) - : Lexp(exp1), Rexp(exp2), Op(op){ } +public: + CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2) + : Lexp(exp1), Rexp(exp2), Op(op){ } - bool eval (FileEntry *file) { - if (Lexp == NULL or Rexp == NULL) { - return false; - } - switch (Op){ - case AndOp: - return Lexp->eval(file) && Rexp->eval(file); - case OrOp: - return Lexp->eval(file) || Rexp->eval(file); - case XorOp: - return Lexp->eval(file) ^ Rexp->eval(file); - default: - return false; - } - } - virtual ~CompoundExpression(){ - delete Lexp; - delete Rexp; - } + bool eval (const ExpFileEntry& file) { + if (Lexp == NULL or Rexp == NULL) { + return false; + } + switch (Op){ + case AndOp: + return Lexp->eval(file) && Rexp->eval(file); + case OrOp: + return Lexp->eval(file) || Rexp->eval(file); + case XorOp: + return Lexp->eval(file) ^ Rexp->eval(file); + default: + return false; + } + } + virtual ~CompoundExpression(){ + delete Lexp; + delete Rexp; + } - virtual void linearize(LinearizedExpression& e) const ; - private: - Expression *Lexp; - Expression *Rexp; - enum LogicalOperator Op; + virtual void linearize(LinearizedExpression& e) const ; +private: + Expression *Lexp; + Expression *Rexp; + enum LogicalOperator Op; }; class StringExpression: public Expression { - public: - StringExpression(enum StringOperator op, std::list &t, bool ic): Op(op),terms(t), IgnoreCase(ic){} +public: + StringExpression(enum StringOperator op, std::list &t, bool ic): Op(op),terms(t), IgnoreCase(ic){} - virtual void linearize(LinearizedExpression& e) const ; - protected: - bool evalStr(const std::string &str); + virtual void linearize(LinearizedExpression& e) const ; +protected: + bool evalStr(const std::string &str); - enum StringOperator Op; - std::list terms; - bool IgnoreCase; + enum StringOperator Op; + std::list terms; + bool IgnoreCase; }; template class RelExpression: public Expression { - public: - RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {} +public: + RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {} - virtual void linearize(LinearizedExpression& e) const ; - protected: - bool evalRel(T val); + virtual void linearize(LinearizedExpression& e) const ; +protected: + bool evalRel(T val); - enum RelOperator Op; - T LowerValue; - T HigherValue; + enum RelOperator Op; + T LowerValue; + T HigherValue; }; template<> void RelExpression::linearize(LinearizedExpression& e) const ; template bool RelExpression::evalRel(T val) { - switch (Op) { - case Equals: - return LowerValue == val; - case GreaterEquals: - return LowerValue >= val; - case Greater: - return LowerValue > val; - case SmallerEquals: - return LowerValue <= val; - case Smaller: - return LowerValue < val; - case InRange: - return (LowerValue <= val) && (val <= HigherValue); - default: - return false; - } + switch (Op) { + case Equals: + return LowerValue == val; + case GreaterEquals: + return LowerValue >= val; + case Greater: + return LowerValue > val; + case SmallerEquals: + return LowerValue <= val; + case Smaller: + return LowerValue < val; + case InRange: + return (LowerValue <= val) && (val <= HigherValue); + default: + return false; + } } @@ -212,12 +224,12 @@ Binary Predicate for Case Insensitive search *Factor locales in the comparison */ struct CompareCharIC : - public std::binary_function< char , char , bool> { + public std::binary_function< char , char , bool> { - bool operator () ( char ch1 , char ch2 ) const { - return tolower( static_cast < unsigned char > (ch1) ) - == tolower( static_cast < unsigned char > (ch2) ); - } + bool operator () ( char ch1 , char ch2 ) const { + return tolower( static_cast < unsigned char > (ch1) ) + == tolower( static_cast < unsigned char > (ch2) ); + } }; @@ -228,55 +240,55 @@ Some implementations of StringExpressions. class NameExpression: public StringExpression { - public: - NameExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); +public: + NameExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_NAME) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_NAME) ; + StringExpression::linearize(e) ; + } }; class PathExpression: public StringExpression { public: - PathExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); + PathExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_PATH) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_PATH) ; + StringExpression::linearize(e) ; + } }; class ExtExpression: public StringExpression { public: - ExtExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); + ExtExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_EXT) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_EXT) ; + StringExpression::linearize(e) ; + } }; class HashExpression: public StringExpression { public: - HashExpression(enum StringOperator op, std::list &t): - StringExpression(op,t, true) {} - bool eval(FileEntry *file); + HashExpression(enum StringOperator op, std::list &t): + StringExpression(op,t, true) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_HASH) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_HASH) ; + StringExpression::linearize(e) ; + } }; /****************************************************************************************** @@ -286,62 +298,62 @@ Some implementations of Relational Expressions. class DateExpression: public RelExpression { - public: - DateExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - DateExpression(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + DateExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + DateExpression(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_DATE) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_DATE) ; + RelExpression::linearize(e) ; + } }; class SizeExpression: public RelExpression { - public: - SizeExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - SizeExpression(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + SizeExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + SizeExpression(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ; + RelExpression::linearize(e) ; + } }; class SizeExpressionMB: public RelExpression { - public: - SizeExpressionMB(enum RelOperator op, int v): RelExpression(op,v,v){} - SizeExpressionMB(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + SizeExpressionMB(enum RelOperator op, int v): RelExpression(op,v,v){} + SizeExpressionMB(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ; + RelExpression::linearize(e) ; + } }; class PopExpression: public RelExpression { - public: - PopExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - PopExpression(enum RelOperator op, int lv, int hv): RelExpression(op,lv,hv) {} - PopExpression(const LinearizedExpression& e) ; - bool eval(FileEntry *file); +public: + PopExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + PopExpression(enum RelOperator op, int lv, int hv): RelExpression(op,lv,hv) {} + PopExpression(const LinearizedExpression& e) ; + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_POP) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_POP) ; + RelExpression::linearize(e) ; + } }; +} -#endif /* RS_EXPRESSIONS_H */ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index d49324933..2236a6a8b 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -36,7 +36,7 @@ class RsFiles; extern RsFiles *rsFiles; -class Expression; +namespace RsRegularExpression { class Expression; } class CacheStrapper ; class CacheTransfer; @@ -190,8 +190,8 @@ class RsFiles virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) = 0; - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags) = 0; + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; /*** * Utility Functions. diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 6fb1b045c..59b787c8e 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -35,7 +35,7 @@ #include "retroshare/rstypes.h" -class LinearizedExpression ; +namespace RsRegularExpression { class LinearizedExpression ; } class RsTurtleClientService ; class RsTurtle; @@ -102,7 +102,7 @@ class RsTurtle // as they come back. // virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ; - virtual TurtleRequestId turtleSearch(const LinearizedExpression& expr) = 0 ; + virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index b3bcb1630..25d1972a5 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -1763,7 +1763,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list initialResults; // to do: split search string into words. - Expression *exp = LinearizedExpression::toExpr(expr) ; + RsRegularExpression::Expression *exp = RsRegularExpression::LinearizedExpression::toExpr(expr) ; if(exp == NULL) return ; @@ -1820,7 +1820,7 @@ TurtleRequestId p3turtle::turtleSearch(const std::string& string_to_match) return id ; } -TurtleRequestId p3turtle::turtleSearch(const LinearizedExpression& expr) +TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpression& expr) { // generate a new search id. diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index e5faca3bc..bfda7663e 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -239,7 +239,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config // remove the specific file search packets from the turtle router. // virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; - virtual TurtleSearchRequestId turtleSearch(const LinearizedExpression& expr) ; + virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 114ee97ef..11e1969e7 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -102,7 +102,7 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {} RsTurtleRegExpSearchRequestItem(void *data,uint32_t size) ; - LinearizedExpression expr ; // Reg Exp in linearised mode + RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } virtual void performLocalSearch(std::list&) const ; diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index 4c6d9481e..fa80c7ec9 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -774,7 +774,7 @@ void SearchDialog::initSearchResult(const QString& txt, qulonglong searchId, int ui.searchSummaryWidget->setCurrentItem(item2); } -void SearchDialog::advancedSearch(Expression* expression) +void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression) { advSearchDialog->hide(); @@ -782,7 +782,7 @@ void SearchDialog::advancedSearch(Expression* expression) std::list results; // send a turtle search request - LinearizedExpression e ; + RsRegularExpression::LinearizedExpression e ; expression->linearize(e) ; TurtleRequestId req_id = rsTurtle->turtleSearch(e) ; @@ -843,8 +843,8 @@ void SearchDialog::searchKeywords(const QString& keywords) if (n < 1) return; - NameExpression exprs(ContainsAllStrings,words,true) ; - LinearizedExpression lin_exp ; + RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ; + RsRegularExpression::LinearizedExpression lin_exp ; exprs.linearize(lin_exp) ; TurtleRequestId req_id ; diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.h b/retroshare-gui/src/gui/FileTransfer/SearchDialog.h index c0682ecb0..bfc5892cc 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.h @@ -27,9 +27,10 @@ #include "mainpage.h" class AdvancedSearchDialog; -class Expression; class RSTreeWidgetItemCompareRole; +namespace RsRegularExpression { class Expression; } + #define FRIEND_SEARCH 1 #define ANONYMOUS_SEARCH 2 class SearchDialog : public MainPage @@ -99,7 +100,7 @@ private slots: void showAdvSearchDialog(bool=true); /** perform the advanced search */ - void advancedSearch(Expression*); + void advancedSearch(RsRegularExpression::Expression*); void selectSearchResults(int index = -1); void hideOrShowSearchResult(QTreeWidgetItem* resultItem, QString currentSearchId = QString(), int fileTypeIndex = -1); diff --git a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp index 3d60260be..d4b629b4b 100644 --- a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp +++ b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp @@ -119,9 +119,9 @@ void AdvancedSearchDialog::prepareSearch() } -Expression * AdvancedSearchDialog::getRsExpr() +RsRegularExpression::Expression * AdvancedSearchDialog::getRsExpr() { - Expression * wholeExpression; + RsRegularExpression::Expression * wholeExpression; // process the special case: first expression wholeExpression = expressions->at(0)->getRsExpression(); @@ -131,7 +131,7 @@ Expression * AdvancedSearchDialog::getRsExpr() for (int i = 1; i < expressions->size(); ++i) { // extract the expression information and compound it with the // first expression - wholeExpression = new CompoundExpression(expressions->at(i)->getOperator(), + wholeExpression = new RsRegularExpression::CompoundExpression(expressions->at(i)->getOperator(), wholeExpression, expressions->at(i)->getRsExpression()); } diff --git a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h index 296829057..02626283a 100644 --- a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h +++ b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h @@ -36,10 +36,10 @@ class AdvancedSearchDialog : public QDialog, public Ui::AdvancedSearchDialog public: AdvancedSearchDialog(QWidget * parent = 0 ); - Expression * getRsExpr(); + RsRegularExpression::Expression * getRsExpr(); QString getSearchAsString(); signals: - void search(Expression*); + void search(RsRegularExpression::Expression*); private slots: void deleteExpression(ExpressionWidget*); diff --git a/retroshare-gui/src/gui/advsearch/expressionwidget.cpp b/retroshare-gui/src/gui/advsearch/expressionwidget.cpp index 013bec182..d5e83c3fc 100644 --- a/retroshare-gui/src/gui/advsearch/expressionwidget.cpp +++ b/retroshare-gui/src/gui/advsearch/expressionwidget.cpp @@ -130,7 +130,7 @@ void ExpressionWidget::deleteExpression() emit signalDelete(this); } -LogicalOperator ExpressionWidget::getOperator() +RsRegularExpression::LogicalOperator ExpressionWidget::getOperator() { return exprOpElem->getLogicalOperator(); } @@ -145,9 +145,9 @@ static int checkedConversion(uint64_t s) return (int)s ; } -Expression* ExpressionWidget::getRsExpression() +RsRegularExpression::Expression* ExpressionWidget::getRsExpression() { - Expression * expr = NULL; + RsRegularExpression::Expression * expr = NULL; std::list wordList; uint64_t lowVal = 0; @@ -174,54 +174,54 @@ Expression* ExpressionWidget::getRsExpression() switch (searchType) { case NameSearch: - expr = new NameExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::NameExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case PathSearch: - expr = new PathExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::PathExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case ExtSearch: - expr = new ExtExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::ExtExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case HashSearch: - expr = new HashExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::HashExpression(exprCondElem->getStringOperator(), wordList); break; case DateSearch: if (inRangedConfig) { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); } else { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); } break; case PopSearch: if (inRangedConfig) { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); } else { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); } break; case SizeSearch: if (inRangedConfig) { if(lowVal >= (uint64_t)(1024*1024*1024) || highVal >= (uint64_t)(1024*1024*1024)) - expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024))); + expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024))); else - expr = new SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal); + expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal); } else { uint64_t s = exprParamElem->getIntValue() ; if(s >= (uint64_t)(1024*1024*1024)) - expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ; + expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ; else - expr = new SizeExpression(exprCondElem->getRelOperator(), (int)s) ; + expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), (int)s) ; } break; }; diff --git a/retroshare-gui/src/gui/advsearch/expressionwidget.h b/retroshare-gui/src/gui/advsearch/expressionwidget.h index 21f964449..2ebc6b851 100644 --- a/retroshare-gui/src/gui/advsearch/expressionwidget.h +++ b/retroshare-gui/src/gui/advsearch/expressionwidget.h @@ -45,11 +45,11 @@ public: /** delivers the expression represented by this widget the operator to join this expression with any previous expressions is provided by the getOperator method */ - Expression* getRsExpression(); + RsRegularExpression::Expression* getRsExpression(); /** supplies the operator to be used when joining this expression to the whole query */ - LogicalOperator getOperator(); + RsRegularExpression::LogicalOperator getOperator(); QString toString(); diff --git a/retroshare-gui/src/gui/advsearch/guiexprelement.cpp b/retroshare-gui/src/gui/advsearch/guiexprelement.cpp index f3ed6debe..7cf7e0732 100644 --- a/retroshare-gui/src/gui/advsearch/guiexprelement.cpp +++ b/retroshare-gui/src/gui/advsearch/guiexprelement.cpp @@ -62,9 +62,9 @@ QStringList GuiExprElement::relOptionsList; QMap GuiExprElement::TermsIndexMap; -QMap GuiExprElement::logicalOpIndexMap; -QMap GuiExprElement::strConditionIndexMap; -QMap GuiExprElement::relConditionIndexMap; +QMap GuiExprElement::logicalOpIndexMap; +QMap GuiExprElement::strConditionIndexMap; +QMap GuiExprElement::relConditionIndexMap; QMap GuiExprElement::logicalOpStrMap; QMap GuiExprElement::termsStrMap; @@ -141,9 +141,9 @@ void GuiExprElement::initialiseOptionsLists() GuiExprElement::relOptionsList.append(RANGE); // now the maps - GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = AndOp; - GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = OrOp; - GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = XorOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = RsRegularExpression::AndOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = RsRegularExpression::OrOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = RsRegularExpression::XorOp; GuiExprElement::TermsIndexMap[GuiExprElement::NAME_INDEX] = NameSearch; GuiExprElement::TermsIndexMap[GuiExprElement::PATH_INDEX] = PathSearch; @@ -153,9 +153,9 @@ void GuiExprElement::initialiseOptionsLists() GuiExprElement::TermsIndexMap[GuiExprElement::SIZE_INDEX] = SizeSearch; GuiExprElement::TermsIndexMap[GuiExprElement::POP_INDEX] = PopSearch; - GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = ContainsAnyStrings; - GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = ContainsAllStrings; - GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = EqualsString; + GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = RsRegularExpression::ContainsAnyStrings; + GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = RsRegularExpression::ContainsAllStrings; + GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = RsRegularExpression::EqualsString; /* W A R N I N G !!!! the cb elements correspond to their inverse rel op counterparts in rsexpr.h due to the nature of @@ -166,12 +166,12 @@ void GuiExprElement::initialiseOptionsLists() files where the condition is greater than the file size i.e. files whose size is less than or equal to the condition Therefore we invert the mapping of rel conditions here to match the behaviour of the impl. */ - GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = GreaterEquals; - GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = Greater; - GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = Equals; - GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = Smaller; - GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = SmallerEquals; - GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = InRange; + GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = RsRegularExpression::GreaterEquals; + GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = RsRegularExpression::Greater; + GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = RsRegularExpression::Equals; + GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = RsRegularExpression::Smaller; + GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = RsRegularExpression::SmallerEquals; + GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = RsRegularExpression::InRange; // the string to index map GuiExprElement::termsStrMap[GuiExprElement::NAME_INDEX] = NAME; @@ -260,7 +260,7 @@ QString ExprOpElement::toString() } -LogicalOperator ExprOpElement::getLogicalOperator() +RsRegularExpression::LogicalOperator ExprOpElement::getLogicalOperator() { return GuiExprElement::logicalOpIndexMap[cb->currentIndex()]; } @@ -313,12 +313,12 @@ QString ExprConditionElement::toString() return GuiExprElement::relConditionStrMap[cb->currentIndex()]; } -RelOperator ExprConditionElement::getRelOperator() +RsRegularExpression::RelOperator ExprConditionElement::getRelOperator() { return GuiExprElement::relConditionIndexMap[cb->currentIndex()]; } -StringOperator ExprConditionElement::getStringOperator() +RsRegularExpression::StringOperator ExprConditionElement::getStringOperator() { return GuiExprElement::strConditionIndexMap[cb->currentIndex()]; } diff --git a/retroshare-gui/src/gui/advsearch/guiexprelement.h b/retroshare-gui/src/gui/advsearch/guiexprelement.h index 075eddf41..0b78f7975 100644 --- a/retroshare-gui/src/gui/advsearch/guiexprelement.h +++ b/retroshare-gui/src/gui/advsearch/guiexprelement.h @@ -114,9 +114,9 @@ protected: static QStringList relOptionsList; // provides a mapping of condition operators to RSExpr reloperators - static QMap logicalOpIndexMap; - static QMap strConditionIndexMap; - static QMap relConditionIndexMap; + static QMap logicalOpIndexMap; + static QMap strConditionIndexMap; + static QMap relConditionIndexMap; // provides a mapping of indexes to translatable strings static QMap logicalOpStrMap; @@ -134,7 +134,7 @@ class ExprOpElement : public GuiExprElement public: ExprOpElement(QWidget * parent = 0); - LogicalOperator getLogicalOperator(); + RsRegularExpression::LogicalOperator getLogicalOperator(); QString toString(); private: QComboBox * cb; @@ -148,8 +148,8 @@ class ExprTermsElement : public GuiExprElement public: ExprTermsElement(QWidget * parent = 0); int getTermsIndex(); - RelOperator getRelOperator(); - StringOperator getStringOperator(); + RsRegularExpression::RelOperator getRelOperator(); + RsRegularExpression::StringOperator getStringOperator(); void set(int i) {cb->setCurrentIndex(i);} QString toString(); @@ -167,8 +167,8 @@ class ExprConditionElement : public GuiExprElement public: ExprConditionElement(ExprSearchType, QWidget * parent = 0); - RelOperator getRelOperator(); - StringOperator getStringOperator(); + RsRegularExpression::RelOperator getRelOperator(); + RsRegularExpression::StringOperator getStringOperator(); void adjustForSearchType(ExprSearchType); void set(int i) {cb->setCurrentIndex(i);} QString toString(); From 6ac9b27d7b743fd2f00d2e3b61ad4500f1d4c47e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 12:14:16 +0200 Subject: [PATCH 63/92] moved rsexpr.cc into util/ --- libretroshare/src/libretroshare.pro | 2 +- libretroshare/src/{file_sharing => util}/rsexpr.cc | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename libretroshare/src/{file_sharing => util}/rsexpr.cc (100%) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index e30e3637e..260626db0 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -55,7 +55,6 @@ file_lists { file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ file_sharing/dir_hierarchy.cc \ - file_sharing/rsexpr.cc \ file_sharing/rsfilelistitems.cc } @@ -653,6 +652,7 @@ SOURCES += turtle/p3turtle.cc \ SOURCES += util/folderiterator.cc \ util/rsdebug.cc \ + util/rsexpr.cc \ util/rscompress.cc \ util/smallobject.cc \ util/rsdir.cc \ diff --git a/libretroshare/src/file_sharing/rsexpr.cc b/libretroshare/src/util/rsexpr.cc similarity index 100% rename from libretroshare/src/file_sharing/rsexpr.cc rename to libretroshare/src/util/rsexpr.cc From 02d2fb42b7b6093084354afc5cbbe10fa59780a8 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 13:53:51 +0200 Subject: [PATCH 64/92] fixed search and filter for regular expressions and terms --- .../src/file_sharing/dir_hierarchy.cc | 35 ++++++++++----- libretroshare/src/file_sharing/p3filelists.cc | 45 ++++++++++++++----- libretroshare/src/retroshare/rsexpr.h | 4 +- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 1735beb79..8b03060d1 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -23,6 +23,7 @@ * */ #include "util/rsdir.h" +#include "retroshare/rsexpr.h" #include "dir_hierarchy.h" #include "filelist_io.h" @@ -595,21 +596,33 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const { -#ifdef TODO for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) - { - const std::string &str1 = static_cast(mNodes[fit->second])->file_name; + if(mNodes[it->second] != NULL && exp->eval( + DirectoryStorageExprFileEntry(*static_cast(mNodes[it->second]), + *static_cast(mNodes[mNodes[it->second]->parent_index]) + ))) + results.push_back(it->second); - { - /*Evaluate the boolean expression and add it to the results if its true*/ - bool ret = exp->eval(fit->second); - if (ret == true){ - results.push_back(fit->second); - } - } -#endif + return 0; } int InternalFileHierarchyStorage::searchTerms(const std::list& terms, std::list &results) const diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ea77de3a6..3eaf21c87 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -807,19 +807,37 @@ bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags, int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { - RS_STACK_MUTEX(mFLSMtx) ; - std::list firesults; - mLocalSharedDirs->searchTerms(keywords,firesults) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchTerms(keywords,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } + } return filterResults(firesults,results,flags,client_peer_id) ; } int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { - RS_STACK_MUTEX(mFLSMtx) ; - std::list firesults; - mLocalSharedDirs->searchBoolExp(exp,firesults) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchBoolExp(exp,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } + } return filterResults(firesults,results,flags,client_peer_id) ; } @@ -854,18 +872,21 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li { results.clear(); -#ifdef P3FILELISTS_DEBUG - if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) - std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; -#endif + flags &= RS_FILE_HINTS_PERMISSION_MASK; // remove other flags. + /* translate/filter results */ for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) { DirDetails cdetails ; - RequestDirDetails ((void*)(intptr_t)*rit,cdetails,FileSearchFlags(0u)); + + if(!RequestDirDetails ((void*)(intptr_t)*rit,cdetails,RS_FILE_HINTS_LOCAL)) + { + std::cerr << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; + continue ; + } #ifdef P3FILELISTS_DEBUG - std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; + std::cerr << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif if(!peer_id.isNull()) diff --git a/libretroshare/src/retroshare/rsexpr.h b/libretroshare/src/retroshare/rsexpr.h index fc339e87c..4cf9e7983 100644 --- a/libretroshare/src/retroshare/rsexpr.h +++ b/libretroshare/src/retroshare/rsexpr.h @@ -115,8 +115,8 @@ public: virtual const std::string& file_name() const =0; virtual uint64_t file_size() const =0; virtual time_t file_modtime() const =0; - virtual time_t file_popularity() const =0; - virtual const std::string file_parent_path() const =0; + virtual uint32_t file_popularity() const =0; + virtual std::string file_parent_path() const =0; virtual const RsFileHash& file_hash() const =0; }; From 53c65fff9fb7cbd85f2dca68c952dada1b4585a4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 18:54:00 +0200 Subject: [PATCH 65/92] removed cache transfers from the GUI --- libretroshare/src/ft/ftserver.cc | 13 ------- libretroshare/src/ft/ftserver.h | 7 ---- libretroshare/src/libretroshare.pro | 2 -- libretroshare/src/plugins/pluginmanager.cc | 12 ++----- libretroshare/src/retroshare/rsfiles.h | 4 --- libretroshare/src/rsserver/rsinit.cc | 5 --- .../src/gui/FileTransfer/TransfersDialog.cpp | 18 ---------- .../src/gui/FileTransfer/TransfersDialog.h | 3 -- .../src/gui/settings/TransferPage.cpp | 10 ------ .../src/gui/settings/TransferPage.h | 1 - .../src/gui/settings/TransferPage.ui | 36 +++++-------------- 11 files changed, 11 insertions(+), 100 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 1a558f9d6..06c538a70 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -44,11 +44,6 @@ const int ftserverzone = 29539; #include "pqi/p3notify.h" #include "rsserver/p3face.h" - -// Includes CacheStrapper / FiMonitor / FiStore for us. - -#include "ft/ftdbase.h" - #include "pqi/pqi.h" #include "pqi/p3linkmgr.h" @@ -299,14 +294,6 @@ bool ftServer::FileClearCompleted() { return mFtController->FileClearCompleted(); } -void ftServer::setMinPrioritizedTransfers(uint32_t s) -{ - mFtController->setMinPrioritizedTransfers(s) ; -} -uint32_t ftServer::getMinPrioritizedTransfers() -{ - return mFtController->getMinPrioritizedTransfers() ; -} void ftServer::setQueueSize(uint32_t s) { mFtController->setQueueSize(s) ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 1173d8311..e61a72e49 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -47,7 +47,6 @@ #include "turtle/turtleclientservice.h" #include "services/p3service.h" #include "retroshare/rsfiles.h" -//#include "dbase/cachestrapper.h" #include "pqi/pqi.h" #include "pqi/p3cfgmgr.h" @@ -55,10 +54,6 @@ class p3ConnectMgr; class p3FileDatabase; -class CacheStrapper; -class CacheTransfer; - -class ftCacheStrapper; class ftFiStore; class ftFiMonitor; @@ -145,8 +140,6 @@ public: /*** * Control of Downloads Priority. ***/ - virtual uint32_t getMinPrioritizedTransfers() ; - virtual void setMinPrioritizedTransfers(uint32_t s) ; virtual uint32_t getQueueSize() ; virtual void setQueueSize(uint32_t s) ; virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 260626db0..971578c0a 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -371,7 +371,6 @@ HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ ft/ftdatamultiplex.h \ - ft/ftdbase.h \ ft/ftextralist.h \ ft/ftfilecreator.h \ ft/ftfileprovider.h \ @@ -531,7 +530,6 @@ HEADERS += util/folderiterator.h \ SOURCES += ft/ftchunkmap.cc \ ft/ftcontroller.cc \ ft/ftdatamultiplex.cc \ - ft/ftdbase.cc \ ft/ftextralist.cc \ ft/ftfilecreator.cc \ ft/ftfileprovider.cc \ diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index af1bfbb65..688cc0bb0 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -410,7 +410,8 @@ void RsPluginManager::slowTickPlugins(time_t seconds) void RsPluginManager::registerCacheServices() { -#warning this code should go + // this is removed since the old cache syste is gone, but we need to make it register new GXS group services instead. +#ifdef REMOVED std::cerr << " Registering cache services." << std::endl; for(uint32_t i=0;i<_plugins.size();++i) @@ -419,6 +420,7 @@ void RsPluginManager::registerCacheServices() //rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ; } +#endif } void RsPluginManager::registerClientServices(p3ServiceServer *pqih) @@ -547,14 +549,6 @@ bool RsPluginManager::saveList(bool& cleanup, std::list& list) return true; } -// RsCacheService::RsCacheService(uint16_t service_type,uint32_t tick_delay, RsPluginHandler* pgHandler) -// : CacheSource(service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getLocalCacheDir()), -// CacheStore (service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getFileServer()->getCacheTransfer(), pgHandler->getRemoteCacheDir()), -// p3Config(), -// _tick_delay_in_seconds(tick_delay) -// { -// } - RsPQIService::RsPQIService(uint16_t /*service_type*/, uint32_t /*tick_delay_in_seconds*/, RsPluginHandler* /*pgHandler*/) : p3Service(),p3Config() { diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 2236a6a8b..247270007 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -37,8 +37,6 @@ class RsFiles; extern RsFiles *rsFiles; namespace RsRegularExpression { class Expression; } -class CacheStrapper ; -class CacheTransfer; /* These are used mainly by ftController at the moment */ const uint32_t RS_FILE_CTRL_PAUSE = 0x00000100; @@ -147,8 +145,6 @@ class RsFiles /*** * Control of Downloads Priority. ***/ - virtual uint32_t getMinPrioritizedTransfers() = 0 ; - virtual void setMinPrioritizedTransfers(uint32_t s) = 0 ; virtual uint32_t getQueueSize() = 0 ; virtual void setQueueSize(uint32_t s) = 0 ; virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove mv) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 066616633..82c5168f9 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1675,11 +1675,6 @@ int RsServer::StartupRetroShare() /**************************************************************************/ std::cerr << "(2) Load configuration files" << std::endl; - /* NOTE: CacheStrapper's load causes Cache Files to be - * loaded into all the CacheStores/Sources. This happens - * after all the other configurations have happened. - */ - mConfigMgr->loadConfiguration(); /**************************************************************************/ diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index c19762027..195959e16 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -349,9 +349,6 @@ TransfersDialog::TransfersDialog(QWidget *parent) // ui.tunnelInfoWidget->setFocusPolicy(Qt::NoFocus); /** Setup the actions for the context menu */ - toggleShowCacheTransfersAct = new QAction(tr( "Show file list transfers" ), this ); - toggleShowCacheTransfersAct->setCheckable(true) ; - connect(toggleShowCacheTransfersAct,SIGNAL(triggered()),this,SLOT(toggleShowCacheTransfers())) ; // Actions. Only need to be defined once. pauseAct = new QAction(QIcon(IMAGE_PAUSE), tr("Pause"), this); @@ -519,12 +516,6 @@ UserNotify *TransfersDialog::getUserNotify(QObject *parent) return new TransferUserNotify(parent); } -void TransfersDialog::toggleShowCacheTransfers() -{ - _show_cache_transfers = !_show_cache_transfers ; - insertTransfers() ; -} - void TransfersDialog::processSettings(bool bLoad) { m_bProcessSettings = true; @@ -537,9 +528,6 @@ void TransfersDialog::processSettings(bool bLoad) if (bLoad) { // load settings - // state of checks - _show_cache_transfers = Settings->value("showCacheTransfers", false).toBool(); - // state of the lists DLHeader->restoreState(Settings->value("downloadList").toByteArray()); ULHeader->restoreState(Settings->value("uploadList").toByteArray()); @@ -565,9 +553,6 @@ void TransfersDialog::processSettings(bool bLoad) } else { // save settings - // state of checks - Settings->setValue("showCacheTransfers", _show_cache_transfers); - // state of the lists Settings->setValue("downloadList", DLHeader->saveState()); Settings->setValue("uploadList", ULHeader->saveState()); @@ -801,9 +786,6 @@ void TransfersDialog::downloadListCustomPopupMenu( QPoint /*point*/ ) contextMnu.addSeparator() ;//----------------------------------------------- - contextMnu.addAction( toggleShowCacheTransfersAct ) ; - toggleShowCacheTransfersAct->setChecked(_show_cache_transfers) ; - collCreateAct->setEnabled(true) ; collModifAct->setEnabled(single && add_CollActions) ; collViewAct->setEnabled(single && add_CollActions) ; diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h index 0e6e9ad4b..8477d0749 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h @@ -199,7 +199,6 @@ private: QAction *chunkProgressiveAct; QAction *chunkStreamingAct; QAction *detailsFileAct; - QAction *toggleShowCacheTransfersAct; QAction *renameFileAct; QAction *specifyDestinationDirectoryAct; QAction *expandAllAct; @@ -249,7 +248,6 @@ private: /** Qt Designer generated object */ Ui::TransfersDialog ui; - bool _show_cache_transfers ; public slots: // these two functions add entries to the transfers dialog, and return the row id of the entry modified/added // @@ -259,7 +257,6 @@ public slots: int addUploadItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, const QString& sources,const QString& source_id, const QString& status, qlonglong completed, qlonglong remaining); void showFileDetails() ; - void toggleShowCacheTransfers() ; double getProgress(int row, QStandardItemModel *model); double getSpeed(int row, QStandardItemModel *model); diff --git a/retroshare-gui/src/gui/settings/TransferPage.cpp b/retroshare-gui/src/gui/settings/TransferPage.cpp index 273f5be5b..bccdcea3c 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.cpp +++ b/retroshare-gui/src/gui/settings/TransferPage.cpp @@ -36,7 +36,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) ui.setupUi(this); ui._queueSize_SB->setValue(rsFiles->getQueueSize()) ; - ui._minPrioritized_SB->setValue(rsFiles->getMinPrioritizedTransfers()) ; switch(rsFiles->defaultChunkStrategy()) { @@ -48,7 +47,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) ui._diskSpaceLimit_SB->setValue(rsFiles->freeDiskSpaceLimit()) ; QObject::connect(ui._queueSize_SB,SIGNAL(valueChanged(int)),this,SLOT(updateQueueSize(int))) ; - QObject::connect(ui._minPrioritized_SB,SIGNAL(valueChanged(int)),this,SLOT(updateMinPrioritized(int))) ; QObject::connect(ui._defaultStrategy_CB,SIGNAL(activated(int)),this,SLOT(updateDefaultStrategy(int))) ; QObject::connect(ui._diskSpaceLimit_SB,SIGNAL(valueChanged(int)),this,SLOT(updateDiskSizeLimit(int))) ; QObject::connect(ui._max_tr_up_per_sec_SB, SIGNAL( valueChanged( int ) ), this, SLOT( updateMaxTRUpRate(int) ) ); @@ -80,15 +78,7 @@ void TransferPage::updateDiskSizeLimit(int s) rsFiles->setFreeDiskSpaceLimit(s) ; } -void TransferPage::updateMinPrioritized(int s) -{ - rsFiles->setMinPrioritizedTransfers(s) ; -} void TransferPage::updateQueueSize(int s) { - if(ui._minPrioritized_SB->value() > s) - { - ui._minPrioritized_SB->setValue(s) ; - } rsFiles->setQueueSize(s) ; } diff --git a/retroshare-gui/src/gui/settings/TransferPage.h b/retroshare-gui/src/gui/settings/TransferPage.h index 3a28f41ac..373d9a17f 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.h +++ b/retroshare-gui/src/gui/settings/TransferPage.h @@ -46,7 +46,6 @@ class TransferPage: public ConfigPage public slots: void updateQueueSize(int) ; - void updateMinPrioritized(int) ; void updateDefaultStrategy(int) ; void updateDiskSizeLimit(int) ; void updateMaxTRUpRate(int); diff --git a/retroshare-gui/src/gui/settings/TransferPage.ui b/retroshare-gui/src/gui/settings/TransferPage.ui index 14023f0ab..ceeae7f47 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.ui +++ b/retroshare-gui/src/gui/settings/TransferPage.ui @@ -6,8 +6,8 @@ 0 0 - 604 - 340 + 700 + 356 @@ -28,13 +28,6 @@ - - - - Slots reserved for non-cache transfers: - - - @@ -76,19 +69,6 @@ - - - - <html><head/><body><p>You can use this to force RetroShare to download your files rather <br/>than cache files for as many slots as requested. Setting that number <br/>to be equal to the queue size above will always prioritize your files<br/>over cache. <br/><br/>It is however recommended to leave at least a few slots for cache files. For now, cache files are only used to transfer friend file lists.</p></body></html> - - - 1 - - - 3 - - - @@ -168,12 +148,12 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> <html><head><meta name="qrichtext" content="1" /><style type="text/css"> p, li { white-space: pre-wrap; } -</style></head><body style=" font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;"> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'Sans'; font-size:8pt; font-weight:600;">RetroShare</span><span style=" font-family:'Sans'; font-size:8pt;"> is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.</span></p> -<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'Sans'; font-size:8pt;"><br /></p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'Sans'; font-size:8pt;">You can separately setup share flags for each shared directory in the shared files dialog to be:</span></p> -<ul style="margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;"><li style=" font-family:'Sans'; font-size:8pt;" style=" margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Browsable by friends</span>: files are seen by your friends.</li> -<li style=" font-family:'Sans'; font-size:8pt;" style=" margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Anonymously shared</span>: files are anonymously reachable through distant F2F tunnels.</li></ul></body></html> +</style></head><body style=" font-family:'Sans'; font-size:9pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:8pt; font-weight:600;">RetroShare</span><span style=" font-size:8pt;"> is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.</span></p> +<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-size:8pt;"><br /></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:8pt;">You can separately setup share flags for each shared directory in the shared files dialog to be:</span></p> +<ul style="margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;"><li style=" font-size:8pt;" style=" margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Browsable by friends</span>: files are seen by your friends.</li> +<li style=" font-size:8pt;" style=" margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Anonymously shared</span>: files are anonymously reachable through distant F2F tunnels.</li></ul></body></html> From 93818f6088db01ee36fd22ca2f7477634c4a44e9 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 21:23:27 +0200 Subject: [PATCH 66/92] simplified a bit the code in Flat_Model, and tried to fix the crash when sorting items --- libretroshare/src/ft/ftcontroller.h | 1 - libretroshare/src/plugins/pluginmanager.cc | 1 - libretroshare/src/rsserver/rsinit.cc | 1 - libretroshare/src/util/rsexpr.cc | 1 - retroshare-gui/src/gui/RemoteDirModel.cpp | 104 +++++++++++-------- retroshare-gui/src/gui/RemoteDirModel.h | 5 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 23 ++-- 7 files changed, 77 insertions(+), 59 deletions(-) diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index 5ee84623b..516b07050 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -48,7 +48,6 @@ class ftDataMultiplex; class p3turtle ; class p3ServiceControl; -#include "dbase/cachestrapper.h" #include "util/rsthreads.h" #include "pqi/pqiservicemonitor.h" #include "pqi/p3cfgmgr.h" diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 688cc0bb0..98335ea5a 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 82c5168f9..6726c1656 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -805,7 +805,6 @@ bool RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port) #include //#include -#include "dbase/cachestrapper.h" #include "ft/ftserver.h" #include "ft/ftcontroller.h" diff --git a/libretroshare/src/util/rsexpr.cc b/libretroshare/src/util/rsexpr.cc index 6b4d7ce20..7e35a1873 100644 --- a/libretroshare/src/util/rsexpr.cc +++ b/libretroshare/src/util/rsexpr.cc @@ -24,7 +24,6 @@ */ -#include "dbase/findex.h" #include "retroshare/rsexpr.h" #include "retroshare/rstypes.h" #include diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 2153f255d..1afdc94ef 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -194,6 +194,7 @@ int FlatStyle_RDM::rowCount(const QModelIndex &parent) const std::cerr << "RetroshareDirModel::rowCount(): " << parent.internalPointer(); std::cerr << ": "; #endif + RS_STACK_MUTEX(_ref_mutex) ; return _ref_entries.size() ; } @@ -501,7 +502,12 @@ QVariant FlatStyle_RDM::sortRole(const QModelIndex& index,const DirDetails& deta case 1: return (qulonglong) details.count; case 2: return details.age; case 3: return QString::fromUtf8(rsPeers->getPeerName(details.id).c_str()); - case 4: return _ref_entries[index.row()].second ; + + case 4: { + RS_STACK_MUTEX(_ref_mutex) ; + + return computeDirectoryPath(details); + } } } return QVariant(); @@ -765,9 +771,11 @@ QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent if(row < 0) return QModelIndex() ; - if(row < (int) _ref_entries.size()) + RS_STACK_MUTEX(_ref_mutex) ; + + if(row < (int) _ref_entries.size()) { - void *ref = _ref_entries[row].first ; + void *ref = _ref_entries[row]; #ifdef RDM_DEBUG std::cerr << "Creating index 2 row=" << row << ", column=" << column << ", ref=" << (void*)ref << std::endl; @@ -903,7 +911,7 @@ void RetroshareDirModel::postMods() // changePersistentIndexList(piList, empty); /* Clear caches */ - mCache.clear(); + //mCache.clear(); #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::postMods()" << std::endl; @@ -1316,17 +1324,18 @@ TreeStyle_RDM::~TreeStyle_RDM() } void FlatStyle_RDM::postMods() { - if(visible()) + if(visible()) { - _ref_entries.clear() ; - _ref_stack.clear() ; + emit layoutAboutToBeChanged(); - _ref_stack.push_back(NULL) ; // init the stack with the topmost parent directory - - std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl; - _needs_update = false ; - updateRefs() ; - } + { + RS_STACK_MUTEX(_ref_mutex) ; + _ref_stack.clear() ; + _ref_stack.push_back(NULL) ; // init the stack with the topmost parent directory + _needs_update = false ; + } + QTimer::singleShot(100,this,SLOT(updateRefs())) ; + } else _needs_update = true ; } @@ -1339,47 +1348,56 @@ void FlatStyle_RDM::updateRefs() return ; } - RetroshareDirModel::preMods() ; + RetroshareDirModel::preMods() ; - static const uint32_t MAX_REFS_PER_SECOND = 2000 ; + + static const uint32_t MAX_REFS_PER_SECOND = 2000 ; uint32_t nb_treated_refs = 0 ; - while(!_ref_stack.empty()) - { - void *ref = _ref_stack.back() ; + { + RS_STACK_MUTEX(_ref_mutex) ; + + _ref_entries.clear() ; + + std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl; + + while(!_ref_stack.empty()) + { + void *ref = _ref_stack.back() ; #ifdef RDM_DEBUG - std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; + std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; #endif - _ref_stack.pop_back() ; + _ref_stack.pop_back() ; - DirDetails details ; + DirDetails details ; - if (requestDirDetails(ref, RemoteMode,details)) - { - if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. - _ref_entries.push_back(std::pair(ref,computeDirectoryPath(details))); + if (requestDirDetails(ref, RemoteMode,details)) + { + if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. + _ref_entries.push_back(ref) ; #ifdef RDM_DEBUG - std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; + std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; #endif - for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to - { // the user and setup a timer to finish the job later. - _needs_update = true ; + for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to + { // the user and setup a timer to finish the job later. + _needs_update = true ; - if(visible()) - QTimer::singleShot(2000,this,SLOT(updateRefs())) ; - else - std::cerr << "Not visible: suspending update"<< std::endl; - break ; - } - } - std::cerr << "reference tab contains " << _ref_entries.size() << " files" << std::endl; + if(visible()) + QTimer::singleShot(2000,this,SLOT(updateRefs())) ; + else + std::cerr << "Not visible: suspending update"<< std::endl; + break ; + } + } + std::cerr << "reference tab contains " << _ref_entries.size() << " files" << std::endl; - if(_ref_stack.empty()) - _needs_update = false ; + if(_ref_stack.empty()) + _needs_update = false ; + } - RetroshareDirModel::postMods() ; + RetroshareDirModel::postMods() ; } diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index 54f60ffad..72a3fad7c 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -194,7 +194,7 @@ class FlatStyle_RDM: public RetroshareDirModel public: FlatStyle_RDM(bool mode) - : RetroshareDirModel(mode) + : RetroshareDirModel(mode), _ref_mutex("Flat file list") { _needs_update = true ; } @@ -223,7 +223,8 @@ class FlatStyle_RDM: public RetroshareDirModel QString computeDirectoryPath(const DirDetails& details) const ; - std::vector > _ref_entries ;// used to store the refs to display + mutable RsMutex _ref_mutex ; + std::vector _ref_entries ;// used to store the refs to display std::vector _ref_stack ; // used to store the refs to update bool _needs_update ; }; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index aaaa79284..80cbfa856 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -144,14 +144,17 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD tree_proxyModel->setSortRole(RetroshareDirModel::SortRole); tree_proxyModel->sort(COLUMN_NAME); - flat_proxyModel = new SFDSortFilterProxyModel(flat_model, this); - flat_proxyModel->setDynamicSortFilter(true); - flat_proxyModel->setSourceModel(flat_model); - flat_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); - flat_proxyModel->setSortRole(RetroshareDirModel::SortRole); - flat_proxyModel->sort(COLUMN_NAME); + flat_proxyModel = new SFDSortFilterProxyModel(flat_model, this); + flat_proxyModel->setSourceModel(flat_model); + flat_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); + flat_proxyModel->setSortRole(RetroshareDirModel::SortRole); + flat_proxyModel->sort(COLUMN_NAME); - connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); + // Mr.Alice: I removed this because it causes a crash for some obscur reason. Apparently when the model is changed, the proxy model cannot + // deal with the change by itself. Should I call something specific? I've no idea. Removing this does not seem to cause any harm either. + //flat_proxyModel->setDynamicSortFilter(true); + + connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); connect(ui.filterStartButton, SIGNAL(clicked()), this, SLOT(startFilter())); connect(ui.filterPatternLineEdit, SIGNAL(returnPressed()), this, SLOT(startFilter())); connect(ui.filterPatternLineEdit, SIGNAL(textChanged(const QString &)), this, SLOT(filterRegExpChanged())); @@ -346,13 +349,13 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) if(viewTypeIndex==VIEW_TYPE_TREE) { model = tree_model ; - proxyModel = tree_proxyModel ; + proxyModel = tree_proxyModel ; } else { model = flat_model ; - proxyModel = flat_proxyModel ; - } + proxyModel = flat_proxyModel ; + } showProperColumns() ; From afbb641b58b63405000307d3987feea755a88c1a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 20:41:22 +0200 Subject: [PATCH 67/92] fixed forceDirectoryCheck and inDirectoryCheck methods --- .../src/file_sharing/directory_updater.cc | 5 ++ .../src/file_sharing/directory_updater.h | 3 +- .../src/file_sharing/file_sharing_defaults.h | 6 +- libretroshare/src/file_sharing/p3filelists.cc | 58 ++++++++++--------- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 23c1fe7c6..06b9af252 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -164,6 +164,11 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p } } +bool LocalDirectoryUpdater::inDirectoryCheck() const +{ + return mHashCache->isRunning(); +} + void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) { if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 25f94d28a..a4a3f4242 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -38,7 +38,8 @@ public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void forceUpdate(); + void forceUpdate(); + bool inDirectoryCheck() const ; void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 4c47584e9..43cf5406f 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -26,9 +26,9 @@ #pragma once -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 300 ; // 5 minutes static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 3eaf21c87..db52d40e3 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -36,7 +36,8 @@ #include "rsserver/p3face.h" -#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << ": p3FileLists: " +#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : " +#define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; @@ -564,7 +565,7 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi if(fi+1 >= (1<= (1<< NB_ENTRY_INDEX_BITS)) { - std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; + P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; return false ; } @@ -585,11 +586,11 @@ void p3FileDatabase::requestDirUpdate(void *ref) time_t recurs_max_modf_TS_remote_time,local_update_TS; - std::cerr << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; + P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << " (EE) Cannot get max known recurs modf time!" << std::endl; + P3FILELISTS_ERROR() << " (EE) Cannot get max known recurs modf time!" << std::endl; return ; } @@ -660,8 +661,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); #ifdef DEBUG_FILE_HIERARCHY - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; + P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; #endif return true ; @@ -675,7 +675,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // check consistency if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) { - std::cerr << "remote request on local index or local request on remote index. This should not happen." << std::endl; + P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; return false ; } DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); @@ -710,8 +710,8 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.id = storage->peerId(); #ifdef DEBUG_FILE_HIERARCHY - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; + P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + P3FILELISTS_DEBUG() << d << std::endl; #endif return true; @@ -751,11 +751,12 @@ uint32_t p3FileDatabase::getType(void *ref) const void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { NOT_IMPLEMENTED(); + mLocalDirWatcher->forceUpdate(); } bool p3FileDatabase::inDirectoryCheck() { NOT_IMPLEMENTED(); - return 0; + return mLocalDirWatcher->inDirectoryCheck(); } void p3FileDatabase::setWatchPeriod(uint32_t seconds) { @@ -882,11 +883,11 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li if(!RequestDirDetails ((void*)(intptr_t)*rit,cdetails,RS_FILE_HINTS_LOCAL)) { - std::cerr << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; + P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; continue ; } #ifdef P3FILELISTS_DEBUG - std::cerr << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; + P3FILELISTS_DEBUG() << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif if(!peer_id.isNull()) @@ -950,7 +951,7 @@ void p3FileDatabase::tickRecv() case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: handleDirSyncResponse( dynamic_cast(item) ) ; break ; default: - std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; + P3FILELISTS_ERROR() << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; } delete item ; @@ -976,7 +977,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index)) { - std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + P3FILELISTS_ERROR() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; return; } @@ -995,7 +996,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; + P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else @@ -1043,7 +1044,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(it == mPendingSyncRequests.end()) { - std::cerr << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; + P3FILELISTS_ERROR() << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; return ; } mPendingSyncRequests.erase(it) ; @@ -1069,38 +1070,39 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index)) { - std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; + std::cerr << std::endl; + P3FILELISTS_ERROR() << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } - std::cerr << " entry index is " << entry_index << " " ; + P3FILELISTS_DEBUG() << " entry index is " << entry_index << " " ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { - P3FILELISTS_DEBUG() << " removing directory with index " << entry_index << " because it does not exist." << std::endl; + std::cerr << " removing directory with index " << entry_index << " because it does not exist." << std::endl; mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { - P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; + std::cerr << " Directory is up to date. Setting local TS." << std::endl; mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; + std::cerr << " Item contains directory data. Deserialising/Updating." << std::endl; if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else - std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; + P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; - std::cerr << " new content after update: " << std::endl; -//#ifdef DEBUG_FILE_HIERARCHY +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << " new content after update: " << std::endl; mRemoteDirectories[fi]->print(); -//#endif +#endif } } @@ -1121,7 +1123,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + P3FILELISTS_ERROR() << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; } @@ -1172,7 +1174,7 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons if(!rds->getDirHashFromIndex(e,entry_hash) ) { - std::cerr << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + P3FILELISTS_ERROR() << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; return false; } @@ -1202,7 +1204,7 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons data.peer_id = item->PeerId(); data.flags = item->flags; - std::cerr << " Pushing req in pending list with peer id " << data.peer_id << std::endl; + P3FILELISTS_DEBUG() << " Pushing req in pending list with peer id " << data.peer_id << std::endl; mPendingSyncRequests[sync_req_id] = data ; From e985a2c9271555f13ec0399d255332934045ee9d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 20:43:52 +0200 Subject: [PATCH 68/92] removed call to setDynamicSortFilter in SharedFilesDialog, as a workaround for the crash when sorting --- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 80cbfa856..8ebb22d39 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -138,7 +138,6 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD flat_model = _flat_model ; tree_proxyModel = new SFDSortFilterProxyModel(tree_model, this); - tree_proxyModel->setDynamicSortFilter(true); tree_proxyModel->setSourceModel(tree_model); tree_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); tree_proxyModel->setSortRole(RetroshareDirModel::SortRole); @@ -152,6 +151,7 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD // Mr.Alice: I removed this because it causes a crash for some obscur reason. Apparently when the model is changed, the proxy model cannot // deal with the change by itself. Should I call something specific? I've no idea. Removing this does not seem to cause any harm either. + //tree_proxyModel->setDynamicSortFilter(true); //flat_proxyModel->setDynamicSortFilter(true); connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); From ddc89a653502c50431ee1801a93c8ec3a698d158 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 21:52:16 +0200 Subject: [PATCH 69/92] fixed core dump in printout of empty slots. Improved dir sync-ing strategy --- .../src/file_sharing/dir_hierarchy.cc | 11 ++++--- libretroshare/src/file_sharing/p3filelists.cc | 30 ++++++++++++------- retroshare-gui/src/gui/SharedFilesDialog.cpp | 13 +++++--- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 8b03060d1..45ac0e8bb 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -466,6 +466,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI uint32_t n=0; for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_update_time = 0 ; // force the update of the subdir. + mNodes[d.subdirs[i]]->parent_index = indx ; mNodes[d.subdirs[i]]->row = n++ ; } @@ -767,10 +769,11 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry recursPrint(depth+1,d.subdirs[i]) ; for(uint32_t i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; - } + if(mNodes[d.subfiles[i]] != NULL) + { + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; + } } bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index db52d40e3..0a51b72dd 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -930,9 +930,23 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - the max time is computed upward until the root of the hierarchy // - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. // -// Remote dirs store the last modif time of the files/dir in the friend's time -// - local node sends the last known modf time to friends, -// - friends respond with either a full directory content, or an acknowledge that the time is right +// Remote dirs store +// 1 - recursive modif time of the files/dir in the friend's time +// 2 - update TS in the local time +// +// The update algorithm is the following: +// +// [Client side] +// - every 30 sec, send a sync packet for the root of the hierarchy containing the remote known max recurs TS. +// - crawl through all directories and if the update TS is 0, ask for sync too. +// - when a sync respnse is received: +// - if response is ACK, set the update time to now. Nothing changed. +// - if response is an update +// * locally update the subfiles +// * for all subdirs, set the local update time to 0 (so as to force a sync) +// +// [Server side] +// - when sync req is received, compare to local recurs TS, and send ACK or full update accordingly // // Directories are designated by their hash, instead of their index. This allows to hide the non shared directories // behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between @@ -1004,7 +1018,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) time_t local_recurs_max_time,local_update_time; mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); - if(item->last_known_recurs_modf_TS < local_recurs_max_time) + if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; @@ -1129,16 +1143,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS - if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. - { + if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - // Dont recurs into sub-directories, since we dont know yet were to go. - - //return ; - } - for(DirectoryStorage::DirIterator it(rds,e);it;++it) locked_recursSweepRemoteDirectory(rds,*it,depth+1); } diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 8ebb22d39..f48394cca 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -163,10 +163,10 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD QHeaderView * header = ui.dirTreeView->header () ; header->resizeSection ( COLUMN_NAME, 490 ); - header->resizeSection ( COLUMN_SIZE, 70 ); - header->resizeSection ( COLUMN_AGE, 100 ); - header->resizeSection ( COLUMN_FRIEND, 100 ); - header->resizeSection ( COLUMN_DIR, 100 ); + header->resizeSection ( COLUMN_SIZE, 70 ); + header->resizeSection ( COLUMN_AGE, 100 ); + header->resizeSection ( COLUMN_FRIEND,100); + header->resizeSection ( COLUMN_DIR, 100 ); header->setStretchLastSection(false); @@ -869,6 +869,10 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand if(ui.dirTreeView->model() == NULL) return ; + // we need to disable this, because the signal will trigger unnecessary update at the friend. + + ui.dirTreeView->blockSignals(true) ; + #ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Restoring expanded items. " << std::endl; #endif @@ -877,6 +881,7 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); } + ui.dirTreeView->blockSignals(false) ; } void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) From f58e9c1a731262df18a216868ad46a3bd7ee6893 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 22:41:02 +0200 Subject: [PATCH 70/92] improved debug output of directory storage, and fixed bug causing loading error (previously crashing files should be removed) --- .../src/file_sharing/dir_hierarchy.cc | 73 ++++++++++++------- libretroshare/src/file_sharing/filelist_io.cc | 6 +- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 45ac0e8bb..fe5f67d5c 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -22,14 +22,16 @@ * Please report all bugs and problems to "retroshare.project@gmail.com". * */ +#include #include "util/rsdir.h" +#include "util/rsprint.h" #include "retroshare/rsexpr.h" #include "dir_hierarchy.h" #include "filelist_io.h" //#include "rsexpr.h" -#define DEBUG_DIRECTORY_STORAGE 1 +//#define DEBUG_DIRECTORY_STORAGE 1 /******************************************************************************************************************/ /* Internal File Hierarchy Storage */ @@ -891,6 +893,22 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) } } +class read_error +{ +public: + read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag) + { + std::ostringstream s ; + s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl; + err_string = s.str(); + } + read_error(const std::string& s) : err_string(s) {} + + const std::string& what() const { return err_string ; } +private: + std::string err_string ; +}; + bool InternalFileHierarchyStorage::load(const std::string& fname) { unsigned char *buffer = NULL ; @@ -900,16 +918,16 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) try { if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_size) ) - throw std::runtime_error("Read error") ; + throw read_error("Cannot decrypt") ; // Read some header uint32_t version, n_nodes ; - if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION) ; if(version != (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001) throw std::runtime_error("Wrong version number") ; - if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_nodes)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_nodes)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; // Write all file/dir entries @@ -925,6 +943,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) unsigned char *node_section_data = NULL ; uint32_t node_section_size = 0 ; uint32_t node_section_offset = 0 ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "reading node " << i << ", offset " << buffer_offset << " : " << RsUtil::BinToHex(&buffer[buffer_offset],std::min((int)buffer_size - (int)buffer_offset,100)) << "..." << std::endl; +#endif if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,node_section_data,node_section_size)) { @@ -936,13 +957,13 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t row ; uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ) ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; @@ -967,15 +988,15 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t row ; uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS) ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; @@ -994,35 +1015,35 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t n_subdirs = 0 ; uint32_t n_subfiles = 0 ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; for(uint32_t j=0;jsubdirs.push_back(di) ; } - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; for(uint32_t j=0;jsubfiles.push_back(fi) ; } mNodes[node_index] = de ; mDirHashes[de->dir_hash] = node_index ; } else - throw std::runtime_error("Unknown node section.") ; + throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY) ; free(node_section_data) ; } free(buffer) ; return true ; } - catch(std::exception& e) + catch(read_error& e) { std::cerr << "Error while reading: " << e.what() << std::endl; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index f4cd1f6be..90297ab0e 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -88,8 +88,10 @@ bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& o if(offset+2 > data_size) return false; - data[offset++] = (uint8_t)((S >> 8) + 192) ; - data[offset++] = (uint8_t)((S & 255) - 192) ; + data[offset+1] = (S - 192) & 0xff ; + data[offset ] = ((S - 192 - data[offset+1]) >> 8) + 192; + + offset += 2 ; return true; } From ca30b0da4476f57aaa62d1bcffa42792767ce20a Mon Sep 17 00:00:00 2001 From: MrAlice Date: Thu, 15 Sep 2016 10:41:40 +0200 Subject: [PATCH 71/92] fixed terrible cost of calling requestDirDetails before createIndex --- .../src/file_sharing/dir_hierarchy.cc | 21 +++++++++ .../src/file_sharing/dir_hierarchy.h | 1 + .../src/file_sharing/directory_storage.cc | 6 +++ .../src/file_sharing/directory_storage.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 45 +++++++++++++++++++ libretroshare/src/file_sharing/p3filelists.h | 1 + libretroshare/src/ft/ftserver.cc | 4 ++ libretroshare/src/ft/ftserver.h | 1 + libretroshare/src/retroshare/rsfiles.h | 3 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 42 ++++------------- 10 files changed, 90 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index fe5f67d5c..cbe6eeab6 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -94,6 +94,27 @@ bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,D return true; } +bool InternalFileHierarchyStorage::getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const +{ + if(!checkIndex(e,FileStorageNode::TYPE_DIR)) + return false ; + + const DirEntry& d = *static_cast(mNodes[e]) ; + + if((uint32_t)row < d.subdirs.size()) + { + c = d.subdirs[row] ; + return true ; + } + + if((uint32_t)row < d.subdirs.size() + d.subfiles.size()) + { + c = d.subfiles[row - (int)d.subdirs.size()] ; + return true ; + } + return false; +} + int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) { if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index fb83e5d9d..905fe0552 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -95,6 +95,7 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; bool stampDirectory(const DirectoryStorage::EntryIndex& indx); + bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2a7cd8f91..faa3ceaee 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -96,6 +96,12 @@ int DirectoryStorage::parentRow(EntryIndex e) const return mFileHierarchy->parentRow(e) ; } +bool DirectoryStorage::getChildIndex(EntryIndex e,int row,EntryIndex& c) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + return mFileHierarchy->getChildIndex(e,row,c) ; +} uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) { diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index be420f618..dd1202726 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -118,6 +118,7 @@ class DirectoryStorage EntryIndex root() const ; // returns the index of the root directory entry. const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; + bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0a51b72dd..5a7621b17 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -598,6 +598,51 @@ void p3FileDatabase::requestDirUpdate(void *ref) P3FILELISTS_DEBUG() << " Succeed." << std::endl; } +bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const +{ + RS_STACK_MUTEX(mFLSMtx) ; + + result = NULL ; + + if (ref == NULL) + if(flags & RS_FILE_HINTS_LOCAL) + { + if(row != 0) + return false ; + + convertEntryIndexToPointer(0,0,result); + + return true ; + } + else for(uint32_t i=0;iroot(),i+1,result); + + return true; + } + + uint32_t fi; + DirectoryStorage::EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi); + + // check consistency + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + { + P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; + return false ; + } + DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + + // Case where the index is the top of a single person. Can be us, or a friend. + + EntryIndex c = 0; + bool res = storage->getChildIndex(e,row,c); + + convertEntryIndexToPointer(c,fi,result) ; + + return res; +} // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f832dfc53..f72628877 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -116,6 +116,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; int RequestDirDetails(const std::string& path, DirDetails &details) const ; + bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 06c538a70..6d6547884 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -535,6 +535,10 @@ int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, Di return mFileDatabase->RequestDirDetails(uid, path, details); } +bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) +{ + return mFileDatabase->findChildPointer(ref,row,result,flags) ; +} int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) { return mFileDatabase->RequestDirDetails(ref,details,flags) ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index e61a72e49..cc4004263 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -173,6 +173,7 @@ public: ***/ virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); + virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ; virtual uint32_t getType(void *ref,FileSearchFlags flags) ; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 247270007..7f6de8be9 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -182,7 +182,8 @@ class RsFiles */ virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) = 0; virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) = 0; - virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0; + virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) =0; + virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 1afdc94ef..a2f3fd09b 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -710,6 +710,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent std::cerr << ": row:" << row << " col:" << column << " "; #endif + // This function is used extensively. There's no way we can use requestDirDetails() in it, which would + // cause far too much overhead. So we use a dedicated function that only grabs the required information. + if(row < 0) return QModelIndex() ; @@ -722,44 +725,15 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } ********/ - DirDetails details ; + void *result ; - if (! requestDirDetails(ref, RemoteMode,details)) - { -#ifdef RDM_DEBUG - std::cerr << "lookup failed -> invalid"; - std::cerr << std::endl; -#endif + if(rsFiles->findChildPointer(ref, row, result, ((RemoteMode) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL))) + return createIndex(row, column, result) ; + else return QModelIndex(); - } - - - /* now iterate through the details to - * get the reference number - */ - - if (row >= (int) details.children.size()) - { -#ifdef RDM_DEBUG - std::cerr << "wrong number of children -> invalid"; - std::cerr << std::endl; -#endif - return QModelIndex(); - } - -#ifdef RDM_DEBUG - std::cerr << "success index(" << row << "," << column << "," << details->childrenVector[row].ref << ")"; - std::cerr << std::endl; -#endif - - /* we can just grab the reference now */ - -#ifdef RDM_DEBUG - std::cerr << "Creating index 1 row=" << row << ", column=" << column << ", ref=" << (void*)details.children[row].ref << std::endl; -#endif - return createIndex(row, column, details.children[row].ref); } + QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const { Q_UNUSED(parent); From d3b46221ff22d3964c0130786b4ee39b31491b71 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 21:45:00 +0200 Subject: [PATCH 72/92] merged modifications from origin --- .../src/file_sharing/dir_hierarchy.cc | 23 +++----- .../src/file_sharing/dir_hierarchy.h | 8 ++- .../src/file_sharing/directory_storage.cc | 57 ++++++++++--------- .../src/file_sharing/directory_storage.h | 9 ++- .../src/file_sharing/directory_updater.cc | 4 ++ libretroshare/src/file_sharing/p3filelists.cc | 31 +++++----- libretroshare/src/file_sharing/p3filelists.h | 2 +- libretroshare/src/util/folderiterator.cc | 19 +++++++ libretroshare/src/util/folderiterator.h | 7 +++ 9 files changed, 96 insertions(+), 64 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index cbe6eeab6..ad1004715 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -129,14 +129,6 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) { return e < mNodes.size() && mNodes[e] != NULL ; } -bool InternalFileHierarchyStorage::stampDirectory(const DirectoryStorage::EntryIndex& indx) -{ - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - static_cast(mNodes[indx])->dir_modtime = time(NULL) ; - return true; -} bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { @@ -504,33 +496,32 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI return true; } -bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +bool InternalFileHierarchyStorage::getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* m) const { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + std::cerr << "[directory storage] (EE) cannot get TS for index " << index << ". Not a valid index or not a directory." << std::endl; return false; } DirEntry& d(*static_cast(mNodes[index])) ; - recurs_max_modf_TS = d.dir_most_recent_time ; - local_update_TS = d.dir_update_time ; + TS = d.*m ; return true; } -bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + +bool InternalFileHierarchyStorage::setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* m) { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + std::cerr << "[directory storage] (EE) cannot get TS for index " << index << ". Not a valid index or not a directory." << std::endl; return false; } DirEntry& d(*static_cast(mNodes[index])) ; - d.dir_most_recent_time = recurs_max_modf_TS ; - d.dir_update_time = local_update_TS ; + d.*m = TS; return true; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 905fe0552..9320f3102 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -94,7 +94,6 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; - bool stampDirectory(const DirectoryStorage::EntryIndex& indx); bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; @@ -103,8 +102,11 @@ public: bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_array); - bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); - bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + + // TS get/set functions. Take one of the class members as argument. + + bool getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) const; + bool setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) ; // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index faa3ceaee..d5292498a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -115,16 +115,14 @@ uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) return DIR_TYPE_UNKNOWN; } } -bool DirectoryStorage::getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) -{ - RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->getDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; -} -bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) -{ - RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; -} + +bool DirectoryStorage::getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); } +bool DirectoryStorage::getDirectoryRecursModTime(EntryIndex index,time_t& rec_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } +bool DirectoryStorage::getDirectoryLocalModTime (EntryIndex index,time_t& loc_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } + +bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); } +bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } +bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) { @@ -364,26 +362,33 @@ static bool sameLists(const std::list& l1,const std::list::iterator it = mLocalDirs.find(info.filename) ; - - if(it == mLocalDirs.end()) { - std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; - return ; + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::map::iterator it = mLocalDirs.find(info.filename) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; + return ; + } + + // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just + // want to make sure that if they are different, flags get updated. + + if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) + { + it->second = info; + + std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; + changed = true ; + } } - // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just - // want to make sure that if they are different, flags get updated. - - if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) - { - it->second = info; - mFileHierarchy->stampDirectory(0) ; - - std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; - } + if(changed) + setDirectoryLocalModTime(0,time(NULL)) ; } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index dd1202726..abd91b6dd 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -53,8 +53,13 @@ class DirectoryStorage virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; - bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; + bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; + bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; + + bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ; + bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ; + bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 06b9af252..1a9bad468 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -134,6 +134,10 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } } + // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files. + + mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ; + // update file and dir lists for current directory. mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 5a7621b17..88d7ea8bd 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -584,17 +584,9 @@ void p3FileDatabase::requestDirUpdate(void *ref) if(fi == 0) return ; // not updating current directory (should we?) - time_t recurs_max_modf_TS_remote_time,local_update_TS; - P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; - if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) - { - P3FILELISTS_ERROR() << " (EE) Cannot get max known recurs modf time!" << std::endl; - return ; - } - - if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e,recurs_max_modf_TS_remote_time)) + if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) P3FILELISTS_DEBUG() << " Succeed." << std::endl; } @@ -1060,8 +1052,8 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else { - time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); + time_t local_recurs_max_time ; + mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ; if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { @@ -1147,7 +1139,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { std::cerr << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirectoryUpdateTime(entry_index,time(NULL)) ; } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { @@ -1178,9 +1170,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; - time_t recurs_max_modf_TS_remote_time,local_update_TS; + time_t local_update_TS; - if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + if(!rds->getDirectoryUpdateTime(e,local_update_TS)) { P3FILELISTS_ERROR() << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; @@ -1189,7 +1181,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. - if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) + if(generateAndSendSyncRequest(rds,e)) P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; for(DirectoryStorage::DirIterator it(rds,e);it;++it) @@ -1220,11 +1212,18 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId return r ^ random_bias; } -bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time) +bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) { RsFileHash entry_hash ; time_t now = time(NULL) ; + time_t max_known_recurs_modf_time ; + + if(!rds->getDirectoryRecursModTime(e,max_known_recurs_modf_time)) + { + P3FILELISTS_ERROR() << " (EE) cannot find recurs mod time for entry index " << e << ". This is very unexpected." << std::endl; + return false; + } if(!rds->getDirHashFromIndex(e,entry_hash) ) { P3FILELISTS_ERROR() << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f72628877..5198a0a4e 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -179,7 +179,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub * \param e Entry index to update * \return true if the request is correctly sent. */ - bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time); + bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 105c7776c..9e8ba66d3 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -13,6 +13,23 @@ namespace librs { namespace util { FolderIterator::FolderIterator(const std::string& folderName) : mFolderName(folderName) { + // Grab the last modification time for the directory + + struct stat64 buf ; + +#ifdef WINDOWS_SYS + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(folderName, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) +#else + if ( 0 == stat64(folderName.c_str(), &buf)) +#endif + { + mFolderModTime = buf.st_mtime ; + } + + // Now open directory content and read the first entry + #ifdef WINDOWS_SYS std::wstring utf16Name; if(! ConvertUtf8ToUtf16(folderName, utf16Name)) { @@ -123,6 +140,8 @@ bool FolderIterator::d_name(std::string& dest) return true; } +time_t FolderIterator::dir_modtime() const { return mFolderModTime ; } + const std::string& FolderIterator::file_fullpath() { return mFullPath ; } const std::string& FolderIterator::file_name() { return mFileName ; } uint64_t FolderIterator::file_size() { return mFileSize ; } diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index dcf8ed3f3..1dcc07bc3 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -30,10 +30,16 @@ public: TYPE_DIR = 0x02 }; + // info about current parent directory + time_t dir_modtime() const ; + + // info about directory content + bool isValid() const { return validity; } bool readdir(); void next(); +#warning this one should go, as it reports the same information than file_name() bool d_name(std::string& dest); bool closedir(); @@ -58,6 +64,7 @@ private: bool mStatInfoOk ; time_t mFileModTime ; + time_t mFolderModTime ; uint64_t mFileSize ; uint8_t mType ; std::string mFileName ; From 2343c910554f6780cf7c166e6fbfc31ebb875b1e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 21:40:53 +0200 Subject: [PATCH 73/92] added display of hash progress --- libretroshare/src/file_sharing/hash_cache.cc | 37 ++++++++++++++++++-- libretroshare/src/file_sharing/hash_cache.h | 5 +++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index f92964ac4..47edf9b5a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -41,12 +41,33 @@ HashStorage::HashStorage(const std::string& save_file_name) mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; mLastSaveTime = 0 ; + mTotalSizeToHash = 0; + mTotalFilesToHash = 0; { RS_STACK_MUTEX(mHashMtx) ; locked_load() ; } } +static std::string friendlyUnit(uint64_t val) +{ + const std::string units[5] = {"B","KB","MB","GB","TB"}; + char buf[50] ; + + double fact = 1.0 ; + + for(unsigned int i=0; i<5; ++i) + if(double(val)/fact < 1024.0) + { + sprintf(buf,"%2.2f",double(val)/fact) ; + return std::string(buf) + " " + units[i]; + } + else + fact *= 1024.0f ; + + sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ; + return std::string(buf) + " TB"; +} void HashStorage::data_tick() { @@ -93,6 +114,8 @@ void HashStorage::data_tick() std::cerr << "Stopping hashing thread." << std::endl; shutdown(); mRunning = false ; + mTotalSizeToHash = 0; + mTotalFilesToHash = 0; std::cerr << "done." << std::endl; } @@ -110,6 +133,7 @@ void HashStorage::data_tick() { RS_STACK_MUTEX(mHashMtx) ; + job = mFilesToHash.begin()->second ; mFilesToHash.erase(mFilesToHash.begin()) ; } @@ -117,9 +141,9 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); std::string tmpout; - //rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", cnt+1, n_files, friendlyUnit(size).c_str(), int(size/double(total_size)*100.0), fe.name.c_str()) ; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, job.full_path) ; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; @@ -139,6 +163,8 @@ void HashStorage::data_tick() info.hash = hash; mChanged = true ; + ++mHashCounter ; + mTotalHashedSize += size ; } } // call the client @@ -181,16 +207,23 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t FileHashJob job ; job.client = c ; + job.size = size ; job.client_param = client_param ; job.full_path = full_path ; job.ts = mod_time ; mFilesToHash[full_path] = job; + mTotalSizeToHash += size ; + ++mTotalFilesToHash; + if(!mRunning) { mRunning = true ; std::cerr << "Starting hashing thread." << std::endl; + mHashCounter = 0; + mTotalHashedSize = 0; + start() ; } diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 467c18d6a..9ab4199a6 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -97,6 +97,7 @@ private: struct FileHashJob { std::string full_path; + uint64_t size ; HashStorageClient *client; uint32_t client_param ; time_t ts; @@ -110,7 +111,11 @@ private: RsMutex mHashMtx ; bool mRunning; + uint32_t mHashCounter; uint32_t mInactivitySleepTime ; + uint64_t mTotalSizeToHash ; + uint64_t mTotalHashedSize ; + uint64_t mTotalFilesToHash ; time_t mLastSaveTime ; }; From 3f9acb5ff81830b1ebae484a7d1b5af1402fb3db Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 23:51:46 +0200 Subject: [PATCH 74/92] added item splitting and checksum checking in p3filelists --- .../src/file_sharing/file_sharing_defaults.h | 2 + libretroshare/src/file_sharing/p3filelists.cc | 120 +++++++++++++++++- libretroshare/src/file_sharing/p3filelists.h | 6 + .../src/file_sharing/rsfilelistitems.cc | 4 + .../src/file_sharing/rsfilelistitems.h | 3 + 5 files changed, 130 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 43cf5406f..486c684ee 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -38,3 +38,5 @@ static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-co static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this + +static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 88d7ea8bd..2d57fb76b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -606,12 +606,13 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe return true ; } - else for(uint32_t i=0;iroot(),i+1,result); - + convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ; @@ -1077,11 +1078,120 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) // sends the response. - sendItem(ritem); + splitAndSendItem(ritem) ; } -void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) +void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) { + ritem->checksum = RsDirUtil::sha1sum((uint8_t*)ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + + while(ritem->directory_content_data.bin_len > MAX_DIR_SYNC_RESPONSE_DATA_SIZE) + { + P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; + + RsFileListsSyncResponseItem *subitem = new RsFileListsSyncResponseItem() ; + + subitem->entry_hash = ritem->entry_hash; + subitem->flags = ritem->flags | RsFileListsItem::FLAGS_SYNC_PARTIAL; + subitem->last_known_recurs_modf_TS = ritem->last_known_recurs_modf_TS; + subitem->request_id = ritem->request_id; + subitem->checksum = ritem->checksum ; + + // copy a subpart of the data + subitem->directory_content_data.tlvtype = ritem->directory_content_data.tlvtype ; + subitem->directory_content_data.setBinData(ritem->directory_content_data.bin_data, MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ; + + // update ritem to chop off the data that was sent. + memmove(ritem->directory_content_data.bin_data, &((unsigned char*)ritem->directory_content_data.bin_data)[MAX_DIR_SYNC_RESPONSE_DATA_SIZE], ritem->directory_content_data.bin_len - MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ; + ritem->directory_content_data.bin_len -= MAX_DIR_SYNC_RESPONSE_DATA_SIZE ; + + // send + subitem->PeerId(ritem->PeerId()) ; + + sendItem(subitem) ; + + // fix up last chunk + if(ritem->directory_content_data.bin_len <= MAX_DIR_SYNC_RESPONSE_DATA_SIZE) + ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL_END ; + } + + sendItem(ritem) ; +} + +// This function should not take memory ownership of ritem, so it makes copies. + +RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncResponseItem *ritem) +{ +#warning make sure about how robust that is to disconnections, etc. + if(!(ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL )) + return ritem ; + + // item is a partial item. Look first for a starting entry + + P3FILELISTS_DEBUG() << "Item from peer " << ritem->PeerId() << " is partial. Size = " << ritem->directory_content_data.bin_len << std::endl; + + RS_STACK_MUTEX(mFLSMtx) ; + + bool is_ending = (ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL_END); + std::map::iterator it = mPartialResponseItems.find(ritem->request_id) ; + + if(it == mPartialResponseItems.end()) + { + if(is_ending) + { + P3FILELISTS_ERROR() << "Impossible situation: partial item ended right away. Dropping..." << std::endl; + return NULL; + } + P3FILELISTS_DEBUG() << "Creating new item buffer" << std::endl; + + mPartialResponseItems[ritem->request_id] = new RsFileListsSyncResponseItem(*ritem) ; + return NULL ; + } + else if(it->second->checksum != ritem->checksum) + { + P3FILELISTS_ERROR() << "Impossible situation: partial items with different checksums. Dropping..." << std::endl; + mPartialResponseItems.erase(it); + return NULL; + } + + // collapse the item at the end of the existing partial item. Dont delete ritem as it will be by the caller. + + it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,it->second->directory_content_data.bin_len + ritem->directory_content_data.bin_len) ; + memcpy(it->second->directory_content_data.bin_data,ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + + // if finished, return the item + + if(is_ending) + { + P3FILELISTS_DEBUG() << "Item is complete. Returning it" << std::endl; + + RsFileListsSyncResponseItem *ret = it->second ; + mPartialResponseItems.erase(it) ; + + ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL_END ; + ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL ; + + return ret ; + } + else + return NULL ; +} + +void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) +{ + RsFileListsSyncResponseItem *item = recvAndRebuildItem(sitem) ; + + if(!item) + return ; + + // check the hash. If anything goes wrong (in the chunking for instance) the hash will not match + + if(RsDirUtil::sha1sum((uint8_t*)item->directory_content_data.bin_data,item->directory_content_data.bin_len) != item->checksum) + { + P3FILELISTS_ERROR() << "Checksum error in response item " << std::hex << item->request_id << std::dec << " . This is unexpected, and might be due to connection problems." << std::endl; + return ; + } + P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; EntryIndex entry_index = DirectoryStorage::NO_INDEX; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 5198a0a4e..52dd00967 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -173,6 +173,11 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ; + // utility functions to send items with some maximum size. + + void splitAndSendItem(RsFileListsSyncResponseItem *ritem); + RsFileListsSyncResponseItem *recvAndRebuildItem(RsFileListsSyncResponseItem *ritem); + /*! * \brief generateAndSendSyncRequest * \param rds Remote directory storage for the request @@ -221,6 +226,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub time_t mLastRemoteDirSweepTS ; // TS for friend list update std::map mPendingSyncRequests ; // pending requests, waiting for an answer + std::map mPartialResponseItems; void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index b3fd1db59..bd07bb5f6 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -152,6 +152,7 @@ bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const /* RsFileListsSyncMsgItem */ ok &= entry_hash.serialise(data, size, offset); + ok &= checksum.serialise(data, size, offset); ok &= setRawUInt32(data, size, &offset, flags ); ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); ok &= setRawUInt64(data, size, &offset, request_id); @@ -231,6 +232,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons uint64_t request_id; // use to determine if changes that have occured since last hash ok &= item->entry_hash.deserialise(data, *size, offset); + ok &= item->checksum.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -312,6 +314,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const uint32_t s = 8; //header size s += RsFileHash::serial_size(); // entry hash + s += RsFileHash::serial_size(); // checksum s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -348,6 +351,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind uint16_t int_Indent = indent + 2; printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; + printIndent(out , int_Indent); out << "Checksum : " << checksum << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index deaa06e60..8c1d466e1 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -66,6 +66,8 @@ public: static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ; static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ; static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ; + static const uint32_t FLAGS_SYNC_PARTIAL = 0x0020 ; + static const uint32_t FLAGS_SYNC_PARTIAL_END = 0x0040 ; }; /*! @@ -102,6 +104,7 @@ public: virtual uint32_t serial_size() const ; RsFileHash entry_hash ; // hash of the directory to sync + RsFileHash checksum ; // checksum of the bindary data, for checking uint32_t flags; // is it a partial/final item (used for large items only) uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash From 04c908e0466579d1912b599c671da6b7d669379f Mon Sep 17 00:00:00 2001 From: MrAlice Date: Fri, 16 Sep 2016 09:35:03 +0200 Subject: [PATCH 75/92] fixed two bugs in dirlist chunking system causing checksum errors --- libretroshare/src/file_sharing/p3filelists.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 2d57fb76b..3b743e3d6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1089,10 +1089,12 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) { P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; + ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL ; + RsFileListsSyncResponseItem *subitem = new RsFileListsSyncResponseItem() ; subitem->entry_hash = ritem->entry_hash; - subitem->flags = ritem->flags | RsFileListsItem::FLAGS_SYNC_PARTIAL; + subitem->flags = ritem->flags ; subitem->last_known_recurs_modf_TS = ritem->last_known_recurs_modf_TS; subitem->request_id = ritem->request_id; subitem->checksum = ritem->checksum ; @@ -1156,8 +1158,14 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR // collapse the item at the end of the existing partial item. Dont delete ritem as it will be by the caller. - it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,it->second->directory_content_data.bin_len + ritem->directory_content_data.bin_len) ; - memcpy(it->second->directory_content_data.bin_data,ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + uint32_t old_len = it->second->directory_content_data.bin_len ; + uint32_t added = ritem->directory_content_data.bin_len; + + it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,old_len + added) ; + memcpy(&((unsigned char*)it->second->directory_content_data.bin_data)[old_len],ritem->directory_content_data.bin_data,added); + it->second->directory_content_data.bin_len = old_len + added ; + + P3FILELISTS_DEBUG() << "Added new chunk of length " << added << ". New size is " << old_len + added << std::endl; // if finished, return the item From de104d3e344fe397ded0a4be3b3f089111a66c3e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 18:34:39 +0200 Subject: [PATCH 76/92] fixed up GUI parameters in Directories. Removed HashCache duration handles since this is now automatic --- .../src/file_sharing/directory_storage.cc | 20 ++++++- .../src/file_sharing/directory_storage.h | 20 +++++-- .../src/file_sharing/directory_updater.cc | 39 ++++++++++++- .../src/file_sharing/directory_updater.h | 10 +++- .../src/file_sharing/file_sharing_defaults.h | 16 ++--- libretroshare/src/file_sharing/hash_cache.cc | 58 +++++++++---------- libretroshare/src/file_sharing/hash_cache.h | 9 ++- libretroshare/src/file_sharing/p3filelists.cc | 51 ++++++++-------- libretroshare/src/file_sharing/p3filelists.h | 7 +-- libretroshare/src/ft/ftserver.cc | 32 ++-------- libretroshare/src/ft/ftserver.h | 7 +-- libretroshare/src/retroshare/rsfiles.h | 12 ++-- libretroshare/src/rsserver/rsinit.cc | 4 +- .../src/gui/settings/DirectoriesPage.cpp | 51 ++-------------- .../src/gui/settings/DirectoriesPage.h | 3 - .../src/gui/settings/DirectoriesPage.ui | 50 +--------------- 16 files changed, 169 insertions(+), 220 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d5292498a..efb88bd08 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -337,6 +337,8 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list::const_iterator it(processed_list.begin());it!=processed_list.end();++it) mLocalDirs[it->filename] = *it; + + mTSChanged = true ; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { @@ -388,7 +390,10 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) } if(changed) + { setDirectoryLocalModTime(0,time(NULL)) ; + mTSChanged = true ; + } } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) @@ -411,13 +416,24 @@ bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std:: return true; } +void LocalDirectoryStorage::notifyTSChanged() +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mTSChanged = true ; +} void LocalDirectoryStorage::updateTimeStamps() { RS_STACK_MUTEX(mDirStorageMtx) ; - time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; + if(mTSChanged) + { + std::cerr << "Updating recursive TS for local shared dirs..." << std::endl; - std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; + time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; + mTSChanged = false ; + + std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; + } } std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index abd91b6dd..b76d73ebd 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -201,7 +201,17 @@ public: void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; + /*! + * \brief updateTimeStamps + * Checks recursive TS and update the if needed. + */ void updateTimeStamps(); + + /*! + * \brief notifyTSChanged + * Use this to force an update of the recursive TS, when calling updateTimeStamps(); + */ + void notifyTSChanged(); /*! * \brief getFileInfo Converts an index info a full file info structure. * \param i index in the directory structure @@ -239,11 +249,13 @@ private: std::string locked_getVirtualPath(EntryIndex indx) const ; std::string locked_getVirtualDirName(EntryIndex indx) const ; - bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); - std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; - std::map mLocalDirs ; // map is better for search. it->first=it->second.filename - std::string mFileName; + std::map mLocalDirs ; // map is better for search. it->first=it->second.filename + std::string mFileName; + + bool mTSChanged ; }; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 1a9bad468..16ca35be8 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -42,16 +42,35 @@ LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStora mLastTSUpdateTime = 0; mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES; + mIsEnabled = false ; +} + +bool LocalDirectoryUpdater::isEnabled() const +{ + return mIsEnabled ; +} +void LocalDirectoryUpdater::setEnabled(bool b) +{ + if(mIsEnabled == b) + return ; + + if(b) + start() ; + else + shutdown(); + + mIsEnabled = b ; } void LocalDirectoryUpdater::data_tick() { time_t now = time(NULL) ; - if(now > DELAY_BETWEEN_DIRECTORY_UPDATES + mLastSweepTime) + if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime) { sweepSharedDirectories() ; mLastSweepTime = now; + mSharedDirectories->notifyTSChanged() ; } if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime) @@ -177,10 +196,24 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri { if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; + + mSharedDirectories->notifyTSChanged() ; +} + +bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param) +{ + return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ; +} + +void LocalDirectoryUpdater::setFileWatchPeriod(int seconds) +{ + mDelayBetweenDirectoryUpdates = seconds ; +} +uint32_t LocalDirectoryUpdater::fileWatchPeriod() const +{ + return mDelayBetweenDirectoryUpdates ; } - - diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index a4a3f4242..d0f920cb0 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -41,13 +41,18 @@ public: void forceUpdate(); bool inDirectoryCheck() const ; - void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } - uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } + void setFileWatchPeriod(int seconds) ; + uint32_t fileWatchPeriod() const ; + + void setEnabled(bool b) ; + bool isEnabled() const ; protected: virtual void data_tick() ; virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); + virtual bool hash_confirm(uint32_t client_param) ; + void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); void sweepSharedDirectories(); @@ -59,5 +64,6 @@ private: time_t mLastTSUpdateTime; uint32_t mDelayBetweenDirectoryUpdates; + bool mIsEnabled ; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 486c684ee..94e2981f7 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -28,15 +28,17 @@ static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 300 ; // 5 minutes +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. Buy we only update for real if something has changed. -static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time -static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time +static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch -static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. -static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. +static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. +static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. -static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. -static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission +static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 47edf9b5a..4748e7125 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -43,6 +43,7 @@ HashStorage::HashStorage(const std::string& save_file_name) mLastSaveTime = 0 ; mTotalSizeToHash = 0; mTotalFilesToHash = 0; + mMaxStorageDurationDays = DEFAULT_HASH_STORAGE_DURATION_DAYS ; { RS_STACK_MUTEX(mHashMtx) ; @@ -138,40 +139,42 @@ void HashStorage::data_tick() mFilesToHash.erase(mFilesToHash.begin()) ; } - std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - - std::string tmpout; - rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; - else - std::cerr << "done."<< std::endl; - - // store the result - + if(job.client->hash_confirm(job.client_param)) { - RS_STACK_MUTEX(mHashMtx) ; - HashStorageInfo& info(mFiles[job.full_path]); + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - info.filename = job.full_path ; - info.size = size ; - info.modf_stamp = job.ts ; - info.time_stamp = time(NULL); - info.hash = hash; + std::string tmpout; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - mChanged = true ; - ++mHashCounter ; - mTotalHashedSize += size ; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + else + std::cerr << "done."<< std::endl; + + // store the result + + { + RS_STACK_MUTEX(mHashMtx) ; + HashStorageInfo& info(mFiles[job.full_path]); + + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; + + mChanged = true ; + ++mHashCounter ; + mTotalHashedSize += size ; + } } } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); - } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -234,14 +237,11 @@ void HashStorage::clean() { RS_STACK_MUTEX(mHashMtx) ; -#ifdef HASHSTORAGE_DEBUG - std::cerr << "Cleaning HashStorage..." << std::endl ; -#endif time_t now = time(NULL) ; time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds #ifdef HASHSTORAGE_DEBUG - std::cerr << "cleaning hash cache." << std::endl ; + std::cerr << "Cleaning hash cache." << std::endl ; #endif for(std::map::iterator it(mFiles.begin());it!=mFiles.end();) diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 9ab4199a6..0f92f09c1 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -36,7 +36,14 @@ public: HashStorageClient() {} virtual ~HashStorageClient() {} + // the result of the hashing info is sent to this method + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0; + + // this method is used to check that the client param is still valid just before hashing. This avoids hashing files + // that are still in queue while removed from shared lists. + + virtual bool hash_confirm(uint32_t client_param)=0 ; }; class HashStorage: public RsTickingThread @@ -90,7 +97,7 @@ private: // Local configuration and storage uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries - std::map mFiles ; // stored as (full_path, hash_info) + std::map mFiles ; // stored as (full_path, hash_info) std::string mFilePath ; bool mChanged ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 3b743e3d6..c67f0b322 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -312,6 +312,15 @@ cleanup = true; rskv->tlvkvs.pairs.push_back(kv); } + { + RsTlvKeyValue kv; + + kv.key = WATCH_FILE_ENABLED_SS; + kv.value = watchEnabled()?"YES":"NO" ; + + rskv->tlvkvs.pairs.push_back(kv); + } + /* Add KeyValue to saveList */ sList.push_back(rskv); @@ -354,6 +363,10 @@ bool p3FileDatabase::loadList(std::list& load) if(sscanf(kit->value.c_str(),"%d",&t) == 1) setWatchPeriod(t); } + else if(kit->key == WATCH_FILE_ENABLED_SS) + { + setWatchEnabled(kit->value == "YES") ; + } delete *it ; continue ; } @@ -788,14 +801,23 @@ uint32_t p3FileDatabase::getType(void *ref) const void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { - NOT_IMPLEMENTED(); mLocalDirWatcher->forceUpdate(); } bool p3FileDatabase::inDirectoryCheck() { - NOT_IMPLEMENTED(); return mLocalDirWatcher->inDirectoryCheck(); } +void p3FileDatabase::setWatchEnabled(bool b) +{ + RS_STACK_MUTEX(mFLSMtx) ; + mLocalDirWatcher->setEnabled(b) ; + IndicateConfigChanged(); +} +bool p3FileDatabase::watchEnabled() +{ + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->isEnabled() ; +} void p3FileDatabase::setWatchPeriod(uint32_t seconds) { RS_STACK_MUTEX(mFLSMtx) ; @@ -808,31 +830,6 @@ uint32_t p3FileDatabase::watchPeriod() RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->fileWatchPeriod(); } -void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) -{ - RS_STACK_MUTEX(mFLSMtx) ; - mHashCache->setRememberHashFilesDuration(days) ; -} -uint32_t p3FileDatabase::rememberHashCacheDuration() -{ - RS_STACK_MUTEX(mFLSMtx) ; - return mHashCache->rememberHashFilesDuration() ; -} -void p3FileDatabase::clearHashCache() -{ - RS_STACK_MUTEX(mFLSMtx) ; - mHashCache->clear() ; -} -bool p3FileDatabase::rememberHashCache() -{ - NOT_IMPLEMENTED(); - return false; -} -void p3FileDatabase::setRememberHashCache(bool) -{ - NOT_IMPLEMENTED(); -} - bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 52dd00967..6b7052677 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -135,11 +135,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void setWatchPeriod(uint32_t seconds); uint32_t watchPeriod() ; - void setRememberHashCacheDuration(uint32_t days) ; - uint32_t rememberHashCacheDuration() ; - void clearHashCache() ; - bool rememberHashCache() ; - void setRememberHashCache(bool) ; + void setWatchEnabled(bool b) ; + bool watchEnabled() ; // interfact for directory parsing diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 6d6547884..832530ade 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -675,35 +675,11 @@ bool ftServer::removeSharedDirectory(std::string dir) return true; } -void ftServer::setWatchPeriod(int minutes) -{ - mFileDatabase->setWatchPeriod(minutes*60) ; -} -int ftServer::watchPeriod() const -{ - return mFileDatabase->watchPeriod()/60 ; -} +bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; } +int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; } -void ftServer::setRememberHashFiles(bool b) -{ - mFileDatabase->setRememberHashCache(b) ; -} -bool ftServer::rememberHashFiles() const -{ - return mFileDatabase->rememberHashCache() ; -} -void ftServer::setRememberHashFilesDuration(uint32_t days) -{ - mFileDatabase->setRememberHashCacheDuration(days) ; -} -uint32_t ftServer::rememberHashFilesDuration() const -{ - return mFileDatabase->rememberHashCacheDuration() ; -} -void ftServer::clearHashCache() -{ - mFileDatabase->clearHashCache() ; -} +void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnabled(b) ; } +void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; } bool ftServer::getShareDownloadDirectory() { diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index cc4004263..c5d1d23eb 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -208,13 +208,10 @@ public: virtual bool getShareDownloadDirectory(); virtual bool shareDownloadDirectory(bool share); - virtual void setRememberHashFilesDuration(uint32_t days) ; - virtual uint32_t rememberHashFilesDuration() const ; - virtual bool rememberHashFiles() const ; - virtual void setRememberHashFiles(bool) ; - virtual void clearHashCache() ; virtual void setWatchPeriod(int minutes) ; virtual int watchPeriod() const ; + virtual void setWatchEnabled(bool b) ; + virtual bool watchEnabled() ; /***************************************************************/ /*************** Data Transfer Interface ***********************/ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 7f6de8be9..cd12396fb 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -213,13 +213,11 @@ class RsFiles virtual bool addSharedDirectory(const SharedDirInfo& dir) = 0; virtual bool updateShareFlags(const SharedDirInfo& dir) = 0; // updates the flags. The directory should already exist ! virtual bool removeSharedDirectory(std::string dir) = 0; - virtual void setRememberHashFilesDuration(uint32_t days) = 0 ; - virtual uint32_t rememberHashFilesDuration() const = 0 ; - virtual void clearHashCache() = 0 ; - virtual bool rememberHashFiles() const =0; - virtual void setRememberHashFiles(bool) =0; - virtual void setWatchPeriod(int minutes) =0; - virtual int watchPeriod() const =0; + + virtual void setWatchPeriod(int minutes) =0; + virtual void setWatchEnabled(bool b) =0; + virtual int watchPeriod() const =0; + virtual bool watchEnabled() =0; virtual bool getShareDownloadDirectory() = 0; virtual bool shareDownloadDirectory(bool share) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 6726c1656..c34ae7b94 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1239,8 +1239,8 @@ int RsServer::StartupRetroShare() //pqih = new pqipersongrpDummy(none, flags); /****** New Ft Server **** !!! */ - ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl); - ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); + ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl); + ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); ftserver->SetupFtServer() ; diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.cpp b/retroshare-gui/src/gui/settings/DirectoriesPage.cpp index 1bb387a26..b9d2ecc5e 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.cpp +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.cpp @@ -36,18 +36,9 @@ DirectoriesPage::DirectoriesPage(QWidget * parent, Qt::WindowFlags flags) connect(ui.incomingButton, SIGNAL(clicked( bool ) ), this , SLOT( setIncomingDirectory() ) ); connect(ui.partialButton, SIGNAL(clicked( bool ) ), this , SLOT( setPartialsDirectory() ) ); connect(ui.editShareButton, SIGNAL(clicked()), this, SLOT(editDirectories())); - connect(ui.cleanHashCachePB, SIGNAL(clicked()), this, SLOT(clearHashCache())); - connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(clickedRememberHashes(bool))); - connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(toggleRememberHashes())); connect(ui.autoCheckDirectories_CB, SIGNAL(clicked(bool)), this, SLOT(toggleAutoCheckDirectories(bool))); } -void DirectoriesPage::clearHashCache() -{ - if(QMessageBox::question(this, tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::Yes) - rsFiles->clearHashCache() ; -} - void DirectoriesPage::toggleAutoCheckDirectories(bool b) { ui.autoCheckDirectoriesDelay_SB->setEnabled(b); @@ -58,28 +49,9 @@ void DirectoriesPage::editDirectories() ShareManager::showYourself() ; } -void DirectoriesPage::clickedRememberHashes(bool b) -{ - if (!b) { - if (QMessageBox::question(this,tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::No) { - ui.rememberHashesCB->setChecked(true); - } - } -} - -void DirectoriesPage::toggleRememberHashes() -{ - bool b = ui.rememberHashesCB->isChecked(); - ui.rememberHashesSB->setEnabled(b); - ui.cleanHashCachePB->setEnabled(b); -} - /** Saves the changes on this page */ bool DirectoriesPage::save(QString &/*errmsg*/) { - rsFiles->setRememberHashFilesDuration(ui.rememberHashesSB->value()); - rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); - std::string dir = ui.incomingDir->text().toUtf8().constData(); if (!dir.empty()) { @@ -92,18 +64,8 @@ bool DirectoriesPage::save(QString &/*errmsg*/) rsFiles->setPartialsDirectory(dir); } - if (ui.rememberHashesCB->isChecked()) { - rsFiles->setRememberHashFiles(true); - } else { - rsFiles->setRememberHashFiles(false); - rsFiles->clearHashCache() ; - } - - if (ui.autoCheckDirectories_CB->isChecked()) { - rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); - } else { - rsFiles->setWatchPeriod(-ui.autoCheckDirectoriesDelay_SB->value()); - } + rsFiles->setWatchEnabled(ui.autoCheckDirectories_CB->isChecked()) ; + rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); rsFiles->shareDownloadDirectory(ui.shareDownloadDirectoryCB->isChecked()); @@ -115,14 +77,9 @@ void DirectoriesPage::load() { ui.shareDownloadDirectoryCB->setChecked(rsFiles->getShareDownloadDirectory()); - ui.rememberHashesSB->setValue(rsFiles->rememberHashFilesDuration()); - ui.rememberHashesCB->setChecked(rsFiles->rememberHashFiles()); - toggleRememberHashes(); - int u = rsFiles->watchPeriod() ; - ui.autoCheckDirectoriesDelay_SB->setValue(abs(u)) ; - ui.autoCheckDirectories_CB->setChecked(u>0) ; - ui.autoCheckDirectoriesDelay_SB->setEnabled(u>0) ; + ui.autoCheckDirectoriesDelay_SB->setValue(u) ; + ui.autoCheckDirectories_CB->setChecked(rsFiles->watchEnabled()) ; ; ui.incomingDir->setText(QString::fromUtf8(rsFiles->getDownloadDirectory().c_str())); ui.partialsDir->setText(QString::fromUtf8(rsFiles->getPartialsDirectory().c_str())); diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.h b/retroshare-gui/src/gui/settings/DirectoriesPage.h index 97eedad6b..ffeb1bf34 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.h +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.h @@ -45,9 +45,6 @@ private slots: void editDirectories() ; void setIncomingDirectory(); void setPartialsDirectory(); - void clearHashCache(); - void clickedRememberHashes(bool); - void toggleRememberHashes(); void toggleAutoCheckDirectories(bool); private: diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.ui b/retroshare-gui/src/gui/settings/DirectoriesPage.ui index b9f13e9b6..9413ac2f9 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.ui +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.ui @@ -6,7 +6,7 @@ 0 0 - 485 + 895 549 @@ -144,50 +144,7 @@ - - - - - Remember file hashes even if not shared. -This might be useful if you're sharing an -external HD, to avoid re-hashing files when -you plug it in. - - - Remember hashed files for - - - true - - - - - - - days - - - 1 - - - 365 - - - 10 - - - - - - - Forget any hashed file that is not anymore shared. - - - Clean Hash Cache - - - - + @@ -240,9 +197,6 @@ you plug it in. shareDownloadDirectoryCB editShareButton - rememberHashesCB - rememberHashesSB - cleanHashCachePB autoCheckDirectories_CB autoCheckDirectoriesDelay_SB incomingDir From e4568a896f31611c5d4fabaa3a151f1ed7161951 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 21:16:25 +0200 Subject: [PATCH 77/92] removed some unused code --- libretroshare/src/file_sharing/hash_cache.cc | 5 +++-- libretroshare/src/file_sharing/p3filelists.cc | 20 +++++-------------- libretroshare/src/file_sharing/p3filelists.h | 14 +++++-------- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 4748e7125..becbf48f5 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -314,7 +314,7 @@ void HashStorage::locked_save() return ; } - std::cerr << mFiles.size() << " Entries saved." << std::endl; + std::cerr << mFiles.size() << " entries saved in hash cache." << std::endl; free(data) ; } @@ -356,9 +356,10 @@ bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_si // now write the whole string into a single section in the file if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; - +#ifdef HASHSTORAGE_DEBUG std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl; std::cerr << "Info.filename = " << info.filename << std::endl; +#endif free(section_data) ; return true; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c67f0b322..489164c8f 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -773,11 +773,11 @@ int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& pat NOT_IMPLEMENTED(); return 0; } -int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const -{ - NOT_IMPLEMENTED(); - return 0; -} +//int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const +//{ +// NOT_IMPLEMENTED(); +// return 0; +//} uint32_t p3FileDatabase::getType(void *ref) const { RS_STACK_MUTEX(mFLSMtx) ; @@ -830,16 +830,6 @@ uint32_t p3FileDatabase::watchPeriod() RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->fileWatchPeriod(); } -bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const -{ - RS_STACK_MUTEX(mFLSMtx) ; - - std::list firesults; - mLocalSharedDirs->searchHash(hash,firesults) ; - - NOT_IMPLEMENTED(); - return false; -} int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 6b7052677..ec4e583ba 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -98,15 +98,10 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // virtual int tick() ; - // access to own/remote shared files - // - virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; - - virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; - virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - // ftSearch virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; + virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; + virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; // Interface for browsing dir hierarchy // @@ -114,8 +109,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void stopThreads() ; void startThreads() ; - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. @@ -124,6 +117,9 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; + // proxy method used by the web UI. + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; + // set/update shared directories void setSharedDirectories(const std::list& dirs); From ff4c43819ea4b7f15deadc5cfd2b57cb957de52e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 21:41:18 +0200 Subject: [PATCH 78/92] fixed reloading of ShareManager due to postModDirectories and made it reload after addign new share --- libretroshare/src/file_sharing/p3filelists.cc | 1 - retroshare-gui/src/gui/MainWindow.cpp | 5 ++++- retroshare-gui/src/gui/ShareManager.cpp | 8 ++++++-- retroshare-gui/src/gui/common/GroupFlagsWidget.cpp | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 489164c8f..9608ef8a6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1111,7 +1111,6 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncResponseItem *ritem) { -#warning make sure about how robust that is to disconnections, etc. if(!(ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL )) return ritem ; diff --git a/retroshare-gui/src/gui/MainWindow.cpp b/retroshare-gui/src/gui/MainWindow.cpp index 4f8cfb37e..b6f8913df 100644 --- a/retroshare-gui/src/gui/MainWindow.cpp +++ b/retroshare-gui/src/gui/MainWindow.cpp @@ -756,7 +756,10 @@ void MainWindow::updateFriends() void MainWindow::postModDirectories(bool update_local) { RSettingsWin::postModDirectories(update_local); - ShareManager::postModDirectories(update_local); + + // Why would we need that?? The effect is to reset the flags while we're changing them, so it's really not + // a good idea. + //ShareManager::postModDirectories(update_local); QCoreApplication::flush(); } diff --git a/retroshare-gui/src/gui/ShareManager.cpp b/retroshare-gui/src/gui/ShareManager.cpp index 3e9477fdd..61aadefb4 100644 --- a/retroshare-gui/src/gui/ShareManager.cpp +++ b/retroshare-gui/src/gui/ShareManager.cpp @@ -148,7 +148,7 @@ void ShareManager::load() GroupFlagsWidget *widget = new GroupFlagsWidget(NULL,(*it).shareflags); - listWidget->setRowHeight(row, 32); + listWidget->setRowHeight(row, 32 * QFontMetricsF(font()).height()/14.0); listWidget->setCellWidget(row, COLUMN_SHARE_FLAGS, widget); listWidget->setItem(row, COLUMN_GROUPS, new QTableWidgetItem()) ; @@ -157,7 +157,7 @@ void ShareManager::load() //connect(widget,SIGNAL(flagsChanged(FileStorageFlags)),this,SLOT(updateFlags())) ; } - listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132) ; + listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132 * QFontMetricsF(font()).height()/14.0) ; //ui.incomingDir->setText(QString::fromStdString(rsFiles->getDownloadDirectory())); @@ -282,6 +282,7 @@ void ShareManager::editShareDirectory() ShareDialog sharedlg (it->filename, this); sharedlg.setWindowTitle(tr("Edit Shared Folder")); sharedlg.exec(); + load(); break; } } @@ -318,6 +319,7 @@ void ShareManager::showShareDialog() { ShareDialog sharedlg ("", this); sharedlg.exec(); + load(); } void ShareManager::shareddirListCurrentCellChanged(int currentRow, int currentColumn, int previousRow, int previousColumn) @@ -387,4 +389,6 @@ void ShareManager::dropEvent(QDropEvent *event) event->setDropAction(Qt::CopyAction); event->accept(); + + load(); } diff --git a/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp b/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp index be72f7eb1..a86703fc7 100644 --- a/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp @@ -35,8 +35,8 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags) { _layout = new QHBoxLayout(this) ; - setMinimumSize(128,32) ; - setMaximumSize(128,32) ; + setMinimumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ; + setMaximumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ; setSizePolicy(QSizePolicy::Fixed,QSizePolicy::Fixed); _icons[INDEX_GROUP_BROWSABLE] = new QIcon(FLAGS_GROUP_BROWSABLE_ICON) ; @@ -58,7 +58,7 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags) _buttons[i] = new QPushButton(this) ; _buttons[i]->setCheckable(true) ; _buttons[i]->setChecked(flags & _flags[i]) ; - _buttons[i]->setIconSize(QSize(32,32)); + _buttons[i]->setIconSize(QSize(32 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0)); update_button_state(_buttons[i]->isChecked(),i) ; _layout->addWidget(_buttons[i]) ; } From 9cf950472cbbb5452f9eca63ad7e0f37725a9535 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:05:27 +0200 Subject: [PATCH 79/92] removed most of debug info in file list sharing code --- .../src/file_sharing/directory_storage.cc | 20 ++++-- .../src/file_sharing/directory_updater.cc | 21 ++++-- libretroshare/src/file_sharing/hash_cache.cc | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 69 +++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 3 - 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index efb88bd08..45cb3ccca 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -32,6 +32,8 @@ #include "dir_hierarchy.h" #include "filelist_io.h" +//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1 + /******************************************************************************************************************/ /* Iterators */ /******************************************************************************************************************/ @@ -384,7 +386,9 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { it->second = info; +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; +#endif changed = true ; } } @@ -427,12 +431,16 @@ void LocalDirectoryStorage::updateTimeStamps() if(mTSChanged) { +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Updating recursive TS for local shared dirs..." << std::endl; +#endif time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; mTSChanged = false ; +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +#endif } } @@ -444,7 +452,7 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const if (cit == mLocalDirs.end()) { - std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: " << virtual_rootdir << std::endl; + std::cerr << "(EE) locked_findRealRootFromVirtualFilename() Invalid RootDir: " << virtual_rootdir << std::endl; return std::string(); } return cit->second.filename; @@ -681,7 +689,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary free(file_section_data) ; } +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; +#endif bindata.bin_data = section_data ; bindata.bin_len = section_offset ; @@ -698,10 +708,10 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st { load(fname) ; - std::cerr << "***************************************" << std::endl; - std::cerr << "Loaded following directory for peer " << pid << std::endl; + std::cerr << "Loaded remote directory for peer " << pid << std::endl; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE mFileHierarchy->print(); - std::cerr << "***************************************" << std::endl; +#endif } void RemoteDirectoryStorage::checkSave() @@ -721,7 +731,9 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co uint32_t section_size = bindata.bin_len ; uint32_t section_offset=0 ; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; +#endif std::string dir_name ; uint32_t most_recent_time ,dir_modtime ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 16ca35be8..c2e20aeca 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -29,7 +29,7 @@ #include "directory_updater.h" #include "file_sharing_defaults.h" -#define DEBUG_LOCAL_DIR_UPDATER 1 +//#define DEBUG_LOCAL_DIR_UPDATER 1 //=============================================================================================================// // Local Directory Updater // @@ -89,8 +89,9 @@ void LocalDirectoryUpdater::forceUpdate() void LocalDirectoryUpdater::sweepSharedDirectories() { RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl; +#endif // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories @@ -114,7 +115,9 @@ void LocalDirectoryUpdater::sweepSharedDirectories() for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it) { - std::cerr << " recursing into " << stored_dir_it.name() << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl; +#endif recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two // are not necessarily in the same order. @@ -124,7 +127,9 @@ void LocalDirectoryUpdater::sweepSharedDirectories() void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) { - std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl; +#endif // make sure list of subdirs is the same // make sure list of subfiles is the same @@ -143,11 +148,15 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p { case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; subfiles[dirIt.file_name()].size = dirIt.file_size(); +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; +#endif break; case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; +#endif break; default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; @@ -182,7 +191,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p for(std::map::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) { +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " recursing into " << stored_dir_it.name() << std::endl; +#endif recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; } } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index becbf48f5..3f96abec1 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -30,7 +30,7 @@ #include "filelist_io.h" #include "file_sharing_defaults.h" -#define HASHSTORAGE_DEBUG 1 +//#define HASHSTORAGE_DEBUG 1 static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000; static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; @@ -100,7 +100,9 @@ void HashStorage::data_tick() // sleep off mutex! if(empty) { +#ifdef HASHSTORAGE_DEBUG std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl; +#endif usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9608ef8a6..0fd6bc5db 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -39,6 +39,8 @@ #define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : " #define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : " +//#define DEBUG_P3FILELISTS 1 + static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; @@ -179,9 +181,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; -//#ifdef DEBUG_FILE_HIERARCHY +#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); -//#endif +#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI @@ -200,7 +202,7 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -#warning we need to make sure that one req per directory will not cause to keep re-asking the top level dirs. + if(mLastRemoteDirSweepTS + 5 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -235,20 +237,29 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; +#endif mLocalDirWatcher->start(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; +#endif } void p3FileDatabase::stopThreads() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ; +#endif mHashCache->fullstop(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; - P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ; +#endif mLocalDirWatcher->fullstop(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; +#endif } void p3FileDatabase::tickWatchers() @@ -404,8 +415,9 @@ void p3FileDatabase::cleanup() { { RS_STACK_MUTEX(mFLSMtx) ; - +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl; +#endif // look through the list of friend directories. Remove those who are not our friends anymore. // @@ -422,7 +434,10 @@ void p3FileDatabase::cleanup() for(uint32_t i=0;ipeerId()) == friend_set.end()) { + +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; +#endif delete mRemoteDirectories[i]; mRemoteDirectories[i] = NULL ; @@ -446,7 +461,9 @@ void p3FileDatabase::cleanup() if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) continue ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; +#endif if(mRemoteDirectories.size() <= friend_index) mRemoteDirectories.resize(friend_index+1,NULL) ; @@ -467,7 +484,9 @@ void p3FileDatabase::cleanup() for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; +#endif std::map::iterator tmp(it); ++tmp; @@ -476,7 +495,9 @@ void p3FileDatabase::cleanup() } else { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; +#endif ++it ; } } @@ -597,10 +618,16 @@ void p3FileDatabase::requestDirUpdate(void *ref) if(fi == 0) return ; // not updating current directory (should we?) +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; +#endif if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) + { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Succeed." << std::endl; +#endif + } } bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const @@ -1010,7 +1037,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; +#endif EntryIndex entry_index = DirectoryStorage::NO_INDEX; @@ -1030,7 +1059,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(entry_type != DIR_TYPE_DIR) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) @@ -1045,7 +1076,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; ritem->last_known_recurs_modf_TS = local_recurs_max_time; @@ -1055,7 +1088,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; ritem->last_known_recurs_modf_TS = local_recurs_max_time ; @@ -1074,7 +1109,9 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) while(ritem->directory_content_data.bin_len > MAX_DIR_SYNC_RESPONSE_DATA_SIZE) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; +#endif ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL ; @@ -1116,7 +1153,9 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR // item is a partial item. Look first for a starting entry +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Item from peer " << ritem->PeerId() << " is partial. Size = " << ritem->directory_content_data.bin_len << std::endl; +#endif RS_STACK_MUTEX(mFLSMtx) ; @@ -1130,7 +1169,9 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR P3FILELISTS_ERROR() << "Impossible situation: partial item ended right away. Dropping..." << std::endl; return NULL; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Creating new item buffer" << std::endl; +#endif mPartialResponseItems[ritem->request_id] = new RsFileListsSyncResponseItem(*ritem) ; return NULL ; @@ -1151,13 +1192,17 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR memcpy(&((unsigned char*)it->second->directory_content_data.bin_data)[old_len],ritem->directory_content_data.bin_data,added); it->second->directory_content_data.bin_len = old_len + added ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Added new chunk of length " << added << ". New size is " << old_len + added << std::endl; +#endif // if finished, return the item if(is_ending) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Item is complete. Returning it" << std::endl; +#endif RsFileListsSyncResponseItem *ret = it->second ; mPartialResponseItems.erase(it) ; @@ -1186,7 +1231,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) return ; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; +#endif EntryIndex entry_index = DirectoryStorage::NO_INDEX; @@ -1229,7 +1276,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) P3FILELISTS_ERROR() << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " entry index is " << entry_index << " " ; +#endif } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) @@ -1272,7 +1321,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // get the info for this entry +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; +#endif time_t local_update_TS; @@ -1286,7 +1337,11 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. if(generateAndSendSyncRequest(rds,e)) + { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; +#endif + } for(DirectoryStorage::DirIterator it(rds,e);it;++it) locked_recursSweepRemoteDirectory(rds,*it,depth+1); @@ -1342,7 +1397,9 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons if(it != mPendingSyncRequests.end()) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; +#endif return false ; } @@ -1360,7 +1417,9 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons data.peer_id = item->PeerId(); data.flags = item->flags; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Pushing req in pending list with peer id " << data.peer_id << std::endl; +#endif mPendingSyncRequests[sync_req_id] = data ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index ec4e583ba..bd35a8015 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -139,9 +139,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed bool inDirectoryCheck(); - // debug - void full_print(); - protected: int filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const; From 0579fb03faa7ff9a094d910351eb9b5a235db24a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:14:25 +0200 Subject: [PATCH 80/92] removed unused tickWatchers() --- libretroshare/src/file_sharing/p3filelists.cc | 8 -------- libretroshare/src/file_sharing/p3filelists.h | 1 - 2 files changed, 9 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0fd6bc5db..0e734e56a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -154,10 +154,6 @@ RsServiceInfo p3FileDatabase::getServiceInfo() } int p3FileDatabase::tick() { - // tick the watchers, possibly create new ones if additional friends do connect. - // - tickWatchers(); - // tick the input/output list of update items and process them // tickRecv() ; @@ -262,10 +258,6 @@ void p3FileDatabase::stopThreads() #endif } -void p3FileDatabase::tickWatchers() -{ -} - bool p3FileDatabase::saveList(bool &cleanup, std::list& sList) { cleanup = true; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index bd35a8015..08afd0e88 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -153,7 +153,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void cleanup(); void tickRecv(); void tickSend(); - void tickWatchers(); private: p3ServiceControl *mServCtrl ; From 7488ad457d024a25bb7071d653c5c0861d879f12 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:32:30 +0200 Subject: [PATCH 81/92] added display of files being hashed with green color and blue icon. This could be improved I suppose. --- retroshare-gui/src/gui/RemoteDirModel.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index a2f3fd09b..bf644713e 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -299,7 +299,10 @@ QVariant RetroshareDirModel::decorationRole(const DirDetails& details,int coln) else if (details.type == DIR_TYPE_FILE) /* File */ { // extensions predefined - return FilesDefs::getIconFromFilename(QString::fromUtf8(details.name.c_str())); + if(details.hash.isNull()) + return QIcon(":/images/reset.png") ; // file is being hashed + else + return FilesDefs::getIconFromFilename(QString::fromUtf8(details.name.c_str())); } else return QVariant(); @@ -541,7 +544,9 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if (role == Qt::TextColorRole) { - if(details.min_age > ageIndicator) + if(details.type == DIR_TYPE_FILE && details.hash.isNull()) + return QVariant(QColor(Qt::green)) ; + else if(details.min_age > ageIndicator) return QVariant(QColor(Qt::gray)) ; else if(RemoteMode) { From 0f5cf56cc6caa3a42e63c4296c2be0b2814bc9ba Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 21 Sep 2016 22:27:53 +0200 Subject: [PATCH 82/92] removed dbase directory --- libretroshare/src/dbase/cachestrapper.cc | 1323 ----------------- libretroshare/src/dbase/cachestrapper.h | 475 ------ libretroshare/src/dbase/cachetest.h | 63 - libretroshare/src/dbase/fimonitor.cc | 1706 ---------------------- libretroshare/src/dbase/fimonitor.h | 220 --- libretroshare/src/dbase/findex.cc | 1516 ------------------- libretroshare/src/dbase/findex.h | 278 ---- libretroshare/src/dbase/fistore.cc | 459 ------ libretroshare/src/dbase/fistore.h | 102 -- 9 files changed, 6142 deletions(-) delete mode 100644 libretroshare/src/dbase/cachestrapper.cc delete mode 100644 libretroshare/src/dbase/cachestrapper.h delete mode 100644 libretroshare/src/dbase/cachetest.h delete mode 100644 libretroshare/src/dbase/fimonitor.cc delete mode 100644 libretroshare/src/dbase/fimonitor.h delete mode 100644 libretroshare/src/dbase/findex.cc delete mode 100644 libretroshare/src/dbase/findex.h delete mode 100644 libretroshare/src/dbase/fistore.cc delete mode 100644 libretroshare/src/dbase/fistore.h diff --git a/libretroshare/src/dbase/cachestrapper.cc b/libretroshare/src/dbase/cachestrapper.cc deleted file mode 100644 index 4ecdedf64..000000000 --- a/libretroshare/src/dbase/cachestrapper.cc +++ /dev/null @@ -1,1323 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "dbase/cachestrapper.h" -#include "serialiser/rsserviceids.h" -#include "serialiser/rsconfigitems.h" -#include "pqi/p3servicecontrol.h" -#include "pqi/p3peermgr.h" -#include "util/rsdir.h" - -#include -#include - -/**** -* #define CS_DEBUG 1 -***/ - -bool operator<(const CacheId &a, const CacheId &b) -{ - if (a.type == b.type) - return (a.subid < b.subid); - return (a.type < b.type); -} - -bool operator<(const CachePair &a, const CachePair &b) -{ - return (a.id < b.id); -} - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d) -{ - out << "[ p: " << d.pid << " id: <" << d.cid.type << "," << d.cid.subid; - out << "> #" << d.hash << " size: " << d.size; - out << " \"" << d.name << "\"@\"" << d.path; - out << "\" ]"; - return out; -} - -/********************************* Cache Store / Source ************************** - * This is a generic interface which interacts with the FileTransfer Unit - * to collect Data to be Cached. - * - ********************************* Cache Store / Source *************************/ - -CacheSource::CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), cacheDir(cachedir), cMutex("CacheSource") - { - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheSource::lockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheSource::unlockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - - /* to be overloaded for inherited Classes */ -bool CacheSource::loadLocalCache(const RsCacheData &data) -{ - return refreshCache(data); -} - - /* control Caches available */ -bool CacheSource::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::set::const_iterator it(destination_peers.begin());it!=destination_peers.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; -} -bool CacheSource::refreshCache(const RsCacheData &data) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - std::list ids; - rsPeers->getOnlineList(ids); - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::list::const_iterator it(ids.begin());it!=ids.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; - -} - -// bool CacheSource::refreshCache(const RsCacheData &data) -// { -// bool ret = false; -// { -// RsStackMutex mtx(cMutex); /* LOCK MUTEX */ -// -// if (data.cid.type == getCacheType()) -// { -// int subid = 0; -// if (isMultiCache()) -// { -// subid = data.cid.subid; -// } -// -// /* Backup the old Caches */ -// CacheSet::const_iterator it; -// if (caches.end() != (it = caches.find(subid))) -// { -// mOldCaches[it->second.hash] = it->second; -// } -// -// /* store new cache */ -// caches[subid] = data; -// ret = true; -// } -// } -// -// if (mStrapper) /* allow testing without full feedback */ -// mStrapper->refreshCache(data); -// -// return ret; -// } -bool CacheSource::clearCache(CacheId id) -{ - lockData(); /* LOCK MUTEX */ - - bool ret = false; - if (id.type == getCacheType()) - { - CacheSet::iterator it; - if (caches.end() != (it = caches.find(id.subid))) - { - /* Backup the old Caches */ - mOldCaches[it->second.hash] = it->second; - caches.erase(it); - ret = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return ret; -} - -//bool CacheSource::cachesAvailable(const RsPeerId& pid, std::map &ids) -//{ -// if(!isPeerAcceptedAsCacheReceiver(pid)) -// return false ; -// -// lockData(); /* LOCK MUTEX */ -// -// /* can overwrite for more control! */ -// CacheSet::iterator it; -// for(it = caches.begin(); it != caches.end(); ++it) -// { -// ids[(it->second).cid] = it->second; -// } -// bool ret = (caches.size() > 0); -// -// unlockData(); /* UNLOCK MUTEX */ -// return ret; -// -//} - - -bool CacheSource::findCache(const RsFileHash &hash, RsCacheData &data) const -{ - lockData(); /* LOCK MUTEX */ - - bool found = false; - CacheSet::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if (hash == (it->second).hash) - { - data = it->second; - found = true; - break; - } - } - - if (!found) - { - std::map::const_iterator oit; - oit = mOldCaches.find(hash); - if (oit != mOldCaches.end()) - { - data = oit->second; - found = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return found; -} - - -void CacheSource::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - CacheSet::iterator it; - out << "CacheSource::listCaches() [" << getCacheType(); - out << "] Total: " << caches.size() << std::endl; - int i; - for(i = 0, it = caches.begin(); it != caches.end(); ++it, ++i) - { - out << "\tC[" << i << "] : " << it->second << std::endl; - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - -CacheStore::CacheStore(uint16_t t, bool m, - CacheStrapper *cs, CacheTransfer *cft, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), - cacheTransfer(cft), cacheDir(cachedir), cMutex("CacheStore") - { - /* not much */ - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheStore::lockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheStore::unlockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - -void CacheStore::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - std::map::iterator pit; - out << "CacheStore::listCaches() [" << getCacheType(); - out << "] Total People: " << caches.size(); - out << std::endl; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator it; - out << "\tTotal for [" << pit->first << "] : " << (pit->second).size(); - out << std::endl; - for(it = (pit->second).begin(); it != (pit->second).end(); ++it) - { - out << "\t\t" << it->second; - out << std::endl; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - - /* look for stored data. using pic/cid in RsCacheData - */ -bool CacheStore::getStoredCache(RsCacheData &data) -{ - lockData(); /* LOCK MUTEX */ - - bool ok = locked_getStoredCache(data); - - unlockData(); /* UNLOCK MUTEX */ - return ok; -} - - -bool CacheStore::locked_getStoredCache(RsCacheData &data) -{ - if (data.cid.type != getCacheType()) - { - return false; - } - - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - return false; - } - - CacheSet::iterator cit; - if (isMultiCache()) - { - /* look for subid */ - if ((pit->second).end() == - (cit = (pit->second).find(data.cid.subid))) - { - return false; - } - } - else - { - if ((pit->second).end() == - (cit = (pit->second).find(0))) - { - return false; - } - } - - /* we found it! (cit) */ - data = cit->second; - - return true; -} - - - -bool CacheStore::getAllStoredCaches(std::list &data) -{ - lockData(); /* LOCK MUTEX */ - - std::map::iterator pit; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator cit; - /* look for subid */ - for(cit = (pit->second).begin(); - cit != (pit->second).end(); ++cit) - { - data.push_back(cit->second); - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return true; -} - - - /* input from CacheStrapper. - * check if we want to download it... - * determine the new name/path - * then request it. - */ -void CacheStore::availableCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::availableCache() :" << data << std::endl; -#endif - - /* basic checks */ - lockData(); /* LOCK MUTEX */ - - bool rightCache = (data.cid.type == getCacheType()); - - unlockData(); /* UNLOCK MUTEX */ - - if (!rightCache) - { - return; /* bad id */ - } - - /* These Functions lock the Mutex themselves - */ - - if (!fetchCache(data)) - { - return; /* ignore it */ - } - - RsCacheData rData = data; - - /* get new name */ - if (!nameCache(rData)) - { - return; /* error naming */ - } - - /* request it */ - if(isPeerAcceptedAsCacheProvider(rData.pid)) // Check for service permission - cacheTransfer -> RequestCache(rData, this); - - /* will get callback when it is complete */ - return; -} - - - - /* called when the download is completed ... updates internal data */ -void CacheStore::downloadedCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::downloadedCache() :" << data << std::endl; -#endif - - /* updates data */ - if (!loadCache(data)) - { - return; - } -} - /* called when the download is completed ... updates internal data */ -void CacheStore::failedCache(const RsCacheData &data) -{ - (void) data; -#ifdef CS_DEBUG - std::cerr << "CacheStore::failedCache() :" << data << std::endl; -#endif - - return; -} - - - /* virtual function overloaded by cache implementor */ -bool CacheStore::fetchCache(const RsCacheData &data) -{ - /* should we fetch it? */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() tofetch?:" << data << std::endl; -#endif - - RsCacheData incache = data; - - lockData(); /* LOCK MUTEX */ - - bool haveCache = ((locked_getStoredCache(incache)) && (data.hash == incache.hash)); - - unlockData(); /* UNLOCK MUTEX */ - - - if (haveCache) - { -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Already have it: false" << std::endl; -#endif - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Missing this cache: true" << std::endl; -#endif - return true; -} - - -int CacheStore::nameCache(RsCacheData &data) -{ - /* name it... */ - lockData(); /* LOCK MUTEX */ - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() for:" << data << std::endl; -#endif - - data.name = data.hash.toStdString(); - data.path = getCacheDir(); - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() done:" << data << std::endl; -#endif - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - - -int CacheStore::loadCache(const RsCacheData &data) -{ - /* attempt to load -> dummy function */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::loadCache() Dummy Load for:" << data << std::endl; -#endif - - lockData(); /* LOCK MUTEX */ - - locked_storeCacheEntry(data); - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - -/* This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - -void CacheStore::locked_storeCacheEntry(const RsCacheData &data) -{ - /* store what we loaded - overwriting if necessary */ - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - /* add in a new CacheSet */ - CacheSet emptySet; - caches[data.pid] = emptySet; - - pit = caches.find(data.pid); - } - - if (isMultiCache()) - { - (pit->second)[data.cid.subid] = data; - } - else - { - (pit->second)[0] = data; - } - - /* tell the strapper we've loaded one */ - if (mStrapper) - { - mStrapper->refreshCacheStore(data); - } - return; -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - -CacheStrapper::CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId) - :p3Config(), mServiceCtrl(sc), mFtServiceId(ftServiceId), - csMtx("CacheStrapper") -{ - return; -} - - -void CacheStrapper::addCachePair(CachePair set) -{ - caches[set.id.type] = set; -} - - - /**************** from pqimonclient ********************/ - -void CacheStrapper::statusChange(const std::list &plist) -{ - std::list::const_iterator it; - for(it = plist.begin(); it != plist.end(); ++it) - { - if(it->actions & RS_SERVICE_PEER_CONNECTED) - { - /* grab all the cache ids and add */ - - std::map hashs; - std::map::iterator cit; - - handleCacheQuery(it->id, hashs); - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(cit = hashs.begin(); cit != hashs.end(); ++cit) - { - mCacheUpdates.push_back(std::make_pair(it->id, cit->second)); - } - } - } -} - - /**************** from pqimonclient ********************/ - -void CacheStrapper::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - /* we've received an update - * send to all online peers + self intersected with online peers. - */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -#endif - const RsPeerId& ownid = mServiceCtrl->getOwnId() ; - std::set ids; - - // Need to use ftServiceID as packets are passed through there. - mServiceCtrl->getPeersConnected(mFtServiceId, ids); - ids.insert(ownid) ; - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) - if(destination_peers.find(*it) != destination_peers.end()) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() Send To: " << *it << std::endl; -#endif - mCacheUpdates.push_back(std::make_pair(*it, data)); - } - - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -} - -// void CacheStrapper::refreshCache(const RsCacheData &data) -// { -// /* we've received an update -// * send to all online peers + self -// */ -// #ifdef CS_DEBUG -// std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -// #endif -// -// std::string ownid = mServiceCtrl->getOwnId() ; -// std::set ids; -// mServiceCtrl->getOnlineList(ids); -// ids.push_back(ownid) ; -// -// { -// RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ -// for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) -// if(*it == ownid || isPeerPartipating(*it)) -// mCacheUpdates.push_back(std::make_pair(*it, data)); -// } -// IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -// } - -void CacheStrapper::refreshCacheStore(const RsCacheData & /* data */ ) -{ - /* indicate to save data */ - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - -} - -bool CacheStrapper::getCacheUpdates(std::list > &updates) -{ - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - updates = mCacheUpdates; - mCacheUpdates.clear(); - - return true; -} - - - - /* pass to correct CacheSet */ -void CacheStrapper::recvCacheResponse(RsCacheData &data, time_t /* ts */) -{ - /* find cache store */ - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(data.cid.type))) - { - /* error - don't have this type of cache */ - return; - } - - /* notify the CacheStore */ - (it2 -> second).store -> availableCache(data); - -} - -void CacheStrapper::handleCacheQuery(const RsPeerId& id, std::map &hashs) -{ - /* basic version just iterates through .... - * more complex could decide who gets what! - * - * or that can be handled on a cache by cache basis. - */ - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - (it->second).source -> cachesAvailable(id, hashs); - } - return; -} - -void CacheStrapper::listCaches(std::ostream &out) -{ - /* can overwrite for more control! */ - std::map::iterator it; - out << "CacheStrapper::listCaches() [" << mServiceCtrl->getOwnId(); - out << "] " << " Total Caches: " << caches.size(); - out << std::endl; - for(it = caches.begin(); it != caches.end(); ++it) - { - out << "CacheType: " << it->first; - out << std::endl; - - (it->second).source->listCaches(out); - (it->second).store->listCaches(out); - out << std::endl; - } - return; -} - -void CacheStrapper::listPeerStatus(std::ostream & /* out */) -{ -#if 0 - std::map::iterator it; - out << "CacheStrapper::listPeerStatus() [" << ownId; - out << "] Total Peers: " << status.size() << " Total Caches: " << caches.size(); - out << std::endl; - for(it = status.begin(); it != status.end(); ++it) - { - out << "Peer: " << it->first; - out << " Query: " << (it->second).query; - out << " Answer: " << (it->second).answer; - out << std::endl; - } - return; -#endif -} - - -bool CacheStrapper::findCache(const RsFileHash& hash, RsCacheData &data) const -{ - /* can overwrite for more control! */ - std::map::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if ((it->second).source->findCache(hash, data)) - { - return true; - } - } - return false; -} - -bool CacheStrapper::CacheExist(RsCacheData& data){ - - std::string filename = data.path + "/" + data.name; - FILE* file = NULL; - file = RsDirUtil::rs_fopen(filename.c_str(), "r"); - - if(file == NULL) - return false; - - fclose(file); - return true; -} - -/***************************************************************************/ -/****************************** CONFIGURATION HANDLING *********************/ -/***************************************************************************/ - -/**** OVERLOADED FROM p3Config ****/ - -RsSerialiser *CacheStrapper::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser(); - - /* add in the types we need! */ - rss->addSerialType(new RsCacheConfigSerialiser()); - - return rss; -} - - -bool CacheStrapper::saveList(bool &cleanup, std::list& saveData) -{ - - /* it can delete them! */ - cleanup = true; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::saveList()" << std::endl; -#endif - - /* iterate through the Caches (local first) */ - - std::list::iterator cit; - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - std::map::iterator tit; - std::map ownTmp; - (it->second).source -> cachesAvailable(ownId, ownTmp); - (it->second).store -> getAllStoredCaches(remoteCaches); - - for(tit = ownTmp.begin(); tit != ownTmp.end(); ++tit) - { - if(CacheExist(tit->second)) - ownCaches.push_back(tit->second); - } - } - - for(cit = ownCaches.begin(); cit != ownCaches.end(); ++cit) - { - RsCacheConfig *rscc = new RsCacheConfig(); - - // Fixup lazy behaviour in clients... - // This ensures correct loading later. - // (used to be: rscc->pid = cit->pid;) - rscc->pid = ownId; - - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - for(cit = remoteCaches.begin(); cit != remoteCaches.end(); ++cit) - { - if (cit->pid == ownId) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() discarding Own Remote Cache"; - std::cerr << std::endl; -#endif - continue; /* skip own caches -> will get transferred anyway */ - } - - RsCacheConfig *rscc = new RsCacheConfig(); - - rscc->pid = cit->pid; - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - /* list completed! */ - return true; -} - - -bool CacheStrapper::loadList(std::list& load) -{ - std::list::iterator it; - RsCacheConfig *rscc; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item Count: " << load.size(); - std::cerr << std::endl; -#endif - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - //peerConnectState ownState; - //mPeerMgr->getOwnNetStatus(ownState); - //std::string ownName = ownState.name+" ("+ownState.location+")"; - - std::map > saveFiles; - std::map >::iterator sit; - - for(it = load.begin(); it != load.end(); ++it) - { - /* switch on type */ - if (NULL != (rscc = dynamic_cast(*it))) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item: "; - std::cerr << std::endl; - rscc->print(std::cerr, 10); - std::cerr << std::endl; -#endif - RsCacheData cd; - - cd.pid = RsPeerId(rscc->pid) ; - -#if 0 - if(cd.pid == ownId) - { - cd.pname = ownName; - } - else - { - peerConnectState pca; - mPeerMgr->getFriendNetStatus(rscc->pid, pca); - cd.pname = pca.name+" ("+pca.location+")"; - } -#endif - - cd.cid.type = rscc->cachetypeid; - cd.cid.subid = rscc->cachesubid; - cd.path = RsDirUtil::convertPathToUnix(rscc->path); - cd.name = rscc->name; - cd.hash = rscc->hash; - cd.size = rscc->size; - cd.recvd = rscc->recvd; - - /* store files that we want to keep */ - (saveFiles[cd.path]).insert(cd.name); - - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(cd.cid.type))) - { - /* error - don't have this type of cache */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Can't Find Cache discarding"; - std::cerr << std::endl; -#endif - } - else - { - /* check that the file exists first.... */ - std::string filename = cd.path; - filename += "/"; - filename += cd.name; -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Checking File: " << filename; - std::cerr << std::endl; -#endif - - uint64_t tmp_size ; - bool fileExists = RsDirUtil::checkFile(filename,tmp_size); - if (fileExists) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Exists ... continuing"; - std::cerr << std::endl; -#endif - if (cd.pid == ownId) - { - - /* load local */ - (it2 -> second).source -> loadLocalCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Local"; - std::cerr << std::endl; -#endif - } - else - { - /* load remote */ - (it2 -> second).store -> loadCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Remote"; - std::cerr << std::endl; -#endif - } - } - else - { - std::cerr << "CacheStrapper::loadList() Doesn't Exist dropping: " << filename; - std::cerr << std::endl; - } - - } - - /* cleanup */ - delete (*it); - - } - else - { - /* cleanup */ - delete (*it); - } - } - load.clear() ; - - /* assemble a list of dirs to clean (union of cache dirs) */ - std::list cacheDirs; - std::list::iterator dit; -#ifdef CS_DEBUG - std::set::iterator fit; -#endif - std::map::iterator cit; - for(cit = caches.begin(); cit != caches.end(); ++cit) - { - std::string lcdir = (cit->second).source->getCacheDir(); - std::string rcdir = (cit->second).store->getCacheDir(); - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), lcdir)) - { - cacheDirs.push_back(lcdir); - } - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), rcdir)) - { - cacheDirs.push_back(rcdir); - } - } - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Files To Save:" << std::endl; -#endif - -#ifdef CS_DEBUG - for(sit = saveFiles.begin(); sit != saveFiles.end(); ++sit) - { - std::cerr << "CacheStrapper::loadList() Files To Save in dir: <" << sit->first << ">" << std::endl; - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "\tFile: " << *fit << std::endl; - } - } -#endif - - std::set emptySet; - for(dit = cacheDirs.begin(); dit != cacheDirs.end(); ++dit) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Cleaning cache dir: <" << *dit << ">" << std::endl; -#endif - sit = saveFiles.find(RsDirUtil::convertPathToUnix(*dit)); - - if (sit != saveFiles.end()) - { -#ifdef CS_DEBUG - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "CacheStrapper::loadList() Keeping File: " << *fit << std::endl; - } -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, sit->second); - } - else - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() No Files to save here!" << std::endl; -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, emptySet); - } - } - - return true; - -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - - -/* request from CacheStore */ -bool CacheTransfer::RequestCache(RsCacheData &data, CacheStore *cbStore) -{ - /* check for a previous request -> and cancel - * - * - if duplicate pid, cid -> cancel old transfer - * - if duplicate hash -> Fail Transfer - */ - - std::map::iterator dit; - std::map::iterator sit; - - for(dit = cbData.begin(); dit != cbData.end(); ++dit) - { - if (((dit->second).pid == data.pid) && - ((dit->second).cid.type == data.cid.type) && - ((dit->second).cid.subid == data.cid.subid)) - { - sit = cbStores.find(dit->second.hash); - - /* if identical to previous request, then we don't want to cancel - * a partially transferred cache file - * - * We wouldn't expect to have to request it again, however the feedback loop - * from ftController is not completed (it should callback and tell us if it cancels - * the cache file. XXX TO FIX. - */ - if ((data.hash == dit->second.hash) && - (data.path == dit->second.path) && - (data.size == dit->second.size)) - { - std::cerr << "Re-request duplicate cache... let it continue"; - std::cerr << std::endl; - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - return true; - } - - /* cancel old transfer */ - CancelCacheFile(dit->second.pid, dit->second.path, - dit->second.hash, dit->second.size); - - sit = cbStores.find(dit->second.hash); - cbData.erase(dit); - cbStores.erase(sit); - - break; - } - } - - /* find in store.... */ - sit = cbStores.find(data.hash); - if (sit != cbStores.end()) - { - /* Duplicate Current Request */ - cbStore -> failedCache(data); - return false; - } - - - /* store request */ - cbData[data.hash] = data; - cbStores[data.hash] = cbStore; - - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - /* wait for answer */ - return true; -} - - -/* to be overloaded */ -bool CacheTransfer::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ - (void) id; - (void) path; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::RequestCacheFile() Dummy... saying completed"; - std::cerr << std::endl; -#endif - - /* just tell them we've completed! */ - CompletedCache(hash); - return true; -} - -/* to be overloaded */ -bool CacheTransfer::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash &hash, uint64_t size) -{ - (void) id; - (void) path; - (void) hash; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CancelCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::CancelCacheFile() Dummy fn"; - std::cerr << std::endl; -#endif - - return true; -} - - -/* internal completion -> does cb */ -bool CacheTransfer::CompletedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache(" << hash << ")"; - std::cerr << std::endl; -#endif - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() Failed to find it"; - std::cerr << std::endl; -#endif - - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() callback to store"; - std::cerr << std::endl; -#endif - /* callback */ - (sit -> second) -> downloadedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - -/* internal completion -> does cb */ -bool CacheTransfer::FailedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { - return false; - } - - /* callback */ - (sit -> second) -> failedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - - -bool CacheTransfer::FindCacheFile(const RsFileHash &hash, std::string &path, uint64_t &size) -{ - RsCacheData data; - if (strapper->findCache(hash, data)) - { - path = data.path + "/" + data.name; - size = data.size; - return true; - } - - return false; -} - - - diff --git a/libretroshare/src/dbase/cachestrapper.h b/libretroshare/src/dbase/cachestrapper.h deleted file mode 100644 index 3be595fc4..000000000 --- a/libretroshare/src/dbase/cachestrapper.h +++ /dev/null @@ -1,475 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_CACHE_STRAPPER_H -#define MRK_CACHE_STRAPPER_H - -#include "pqi/p3cfgmgr.h" -#include "pqi/pqiservicemonitor.h" -#include "util/rsthreads.h" - -#include -#include -#include -#include -#include - -/******************* CacheStrapper and Related Classes ******************* - * A generic Cache Update system. - * - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - * - * CacheTransfer: Interface for FileTransfer Class to support. - * - * CacheSource: Base Class for cache data provider. eg. FileIndexMonitor. - * CacheStore: Base Class for data cache. eg. FileCache/Store. - * - * Still TODO: - * (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper. - * (2) CacheStrapper:: Save / Load Cache lists.... - * (3) Clean up lists, maps on shutdown etc. - * (4) Consider Mutexes for multithreaded operations. - * (5) Test the MultiSource/Store capabilities. - * - ******************* CacheStrapper and Related Classes *******************/ - - -class CacheTransfer; /* Interface for File Transfer */ -class CacheSource; /* Interface for local File Index/Monitor */ -class CacheStore; /* Interface for the actual Cache */ -class CacheStrapper; /* Controlling Class */ - -/******************************** CacheId ********************************/ - -/*! - * Use this to identify the type of cache source, strapper, - */ -class CacheId -{ - public: - CacheId() :type(0), subid(0) { return; } - CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; } - uint16_t type; /// cache types, this should be set to services type - uint16_t subid; /// should be initialised when using multicache feature of cachestrapper -}; - - -bool operator<(const CacheId &a, const CacheId &b); - -/*! - * Use for identifying physical files that have been chosen as cache data - * note: this does not actual store the data but serves to locate on network (via hash attribute, - * and on file via path) - */ -class RsCacheData -{ - public: - - RsPeerId pid; /// peer id - /// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore) - CacheId cid; /// cache id - std::string path; /// file system path where physical cache data is located - std::string name; - RsFileHash hash; - uint64_t size; - time_t recvd; /// received timestamp -}; - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d); - -/***************************** CacheTransfer *****************************/ - -/** - * Interface for FileTransfer Class to support cache - */ -class CacheTransfer -{ - public: - - CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; } - virtual ~CacheTransfer() {} - - /*! - * upload side of things .... searches through CacheStrapper. - */ - bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size); - - /*! - * At the download side RequestCache() => overloaded RequestCacheFile() - * the class should then call CompletedCache() or FailedCache() - */ - bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */ - - protected: - - /*! - * to be overloaded - */ - virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - - bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */ - bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */ - - private: - - CacheStrapper *strapper; - - std::map cbData; - std::map cbStores; -}; - - - -/************************ CacheSource/CacheStore *************************/ - -typedef std::map CacheSet; - -/*! - * Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper - * Service is able to use this class for refresh its cache (push cache data) - * and interface to load and check cache availablility among peers (source of cache data) - * Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting - * service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for - * new cache data from peers - * @see p3Distrib - */ -class CacheSource -{ - public: - - CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir); - virtual ~CacheSource() {} - - /*! - * called to determine available cache for peer - - * default acceptable (returns all) - */ - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids)=0; - - /*! - * function called at startup to load from - * configuration file.... - * to be overloaded by inherited class - */ - virtual bool loadLocalCache(const RsCacheData &data); - - /* control Caches available */ - bool refreshCache(const RsCacheData &data,const std::set& destination_peers); - bool refreshCache(const RsCacheData &data); - bool clearCache(CacheId id); - - /* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; } - - /* get private data */ - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /* display */ - void listCaches(std::ostream &out); - - /* search */ - bool findCache(const RsFileHash& hash, RsCacheData &data) const; - - protected: - - uint16_t cacheType; /// for checking of cache type (usually of child class of source) - bool multiCache; /// whether multisource is in use or not. - CacheStrapper *mStrapper; - - /*** MUTEX LOCKING */ - void lockData() const; - void unlockData() const; - - CacheSet caches; /// all local cache data stored here - std::map mOldCaches; /// replaced/cleared caches are pushed here (in case requested) - - private: - - std::string cacheDir; - mutable RsMutex cMutex; - -}; - - -/*! - * Base Class for data cache. eg. FileCache/Store. - * This is best used to deal with external caches from other peers - * @see p3Distrib. pqiMonitor - */ -class CacheStore -{ - public: - - /*! - * - * @param t set to particular rs_service id. see rsserviceids.h - * @param m whether this is multicache service (true) or not(false) - * @param cs cache strapper instance responsible for maintaining the cache service - * @param cft cache transfer instance responsible for rquestiing and tranfering caches - * @param cachedir directory used to store cache related info for cachestore client - * @return - */ - CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir); - virtual ~CacheStore() {} - - /* current stored data */ - - /*! - * - * @param data returns cache data for pid/cid set in data itself - * @return false is unsuccessful and vice versa - */ - bool getStoredCache(RsCacheData &data); /* use pid/cid in data */ - - /*! - * - * @param data all cache store by cachestore is store here - * @return false not returned, only true at the moment - */ - bool getAllStoredCaches(std::list &data); /* use pid/cid in data */ - - /*! - * input from CacheStrapper -> store can then download new data - */ - void availableCache(const RsCacheData &data); - - /*! - * should be called when the download is completed ... cache data is loaded - */ - void downloadedCache(const RsCacheData &data); - - /*! - * called if the download fails, TODO: nothing done yet - */ - void failedCache(const RsCacheData &data); - - /* virtual functions overloaded by cache implementor */ - - /* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; } - - /*! - * @param data cache data is stored here - * @return false is failed (cache does not exist), otherwise true - */ - virtual bool fetchCache(const RsCacheData &data); /* a question? */ - virtual int nameCache(RsCacheData &data); /* fill in the name/path */ - virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* get private data */ - - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /*! - * display, e.g. can pass std::out, cerr, ofstream, etc - */ - void listCaches(std::ostream &out); - - protected: - - /*! - * ** MUTEX LOCKING - */ - void lockData() const; - - /*! - * ** MUTEX LOCKING - */ - void unlockData() const; - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - void locked_storeCacheEntry(const RsCacheData &data); - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - bool locked_getStoredCache(RsCacheData &data); - - private: - - uint16_t cacheType; /* for checking */ - bool multiCache; /* do we care about subid's */ - - CacheStrapper *mStrapper; - CacheTransfer *cacheTransfer; - - std::string cacheDir; - - mutable RsMutex cMutex; - std::map caches; - -}; - - - -/***************************** CacheStrapper *****************************/ - - -/*! - * a convenient to pass cache handles to cachestrapper to maintain the cache service - * Make Sure you get the Ids right! see rsservicesids.h. - * When creating a cache service this data structure is - * source, usually the child class of store and source also serves as both handles - * @see CacheStrapper - */ -class CachePair -{ - public: - - /*! - * Default constructor, all variables set to NULL - */ - CachePair() - :source(NULL), store(NULL), id(0, 0) { return; } - - /*! - * - * @param a cache source for service - * @param b cache store for service - * @param c the cache service id, c.type should be set to service id service-child class of store and source - */ - CachePair(CacheSource *a, CacheStore *b, CacheId c) - :source(a), store(b), id(c) { return; } - - CacheSource *source; - CacheStore *store; - CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use -}; - - -bool operator<(const CachePair &a, const CachePair &b); - -/*! - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - */ - -class p3ServiceControl; - -class CacheStrapper: public pqiServiceMonitor, public p3Config -{ - public: - - /*! - * @param cm handle used by strapper for getting peer connection information (online peers, sslids...) - * @return - */ - CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId); -virtual ~CacheStrapper() { return; } - - /************* from pqiMonitor *******************/ -virtual void statusChange(const std::list &plist); - /************* from pqiMonitor *******************/ - - /* Feedback from CacheSources */ - -/*! - * send data to peers online and self - * @param data - * - */ -void refreshCache(const RsCacheData &data); -void refreshCache(const RsCacheData &data,const std::set& destination_peers); // specify a particular list of destination peers (self not added!) - -/*! - * forces config savelist - * @param data - * @see saveList() - */ -void refreshCacheStore(const RsCacheData &data); - -/*! - * list of Caches to send out - */ -bool getCacheUpdates(std::list > &updates); - -/*! - * add to strapper's cachepair set so a related service's store and source can be maintained - * @param pair the source and store handle for a service - */ -void addCachePair(CachePair pair); - - /*** I/O (2) ***/ -void recvCacheResponse(RsCacheData &data, time_t ts); -void handleCacheQuery(const RsPeerId& id, std::map &data); - - -/*! - * search through CacheSources. - * @return false if cachedate mapping to hash not found - */ -bool findCache(const RsFileHash &hash, RsCacheData &data) const; - - /* display */ -void listCaches(std::ostream &out); - -/*! - * does not do anything - * @param out - * @deprecated - */ -void listPeerStatus(std::ostream &out); - -/** - * Checks if the cache physically exist at path given - * @param data - * @return whether it exists or not - */ -bool CacheExist(RsCacheData& data); - - /* Config */ - protected: - - /* Key Functions to be overloaded for Full Configuration */ -virtual RsSerialiser *setupSerialiser(); -virtual bool saveList(bool &cleanup, std::list&); -virtual bool loadList(std::list& load); - - private: - - /* these are static - so shouldn't need mutex */ - p3ServiceControl *mServiceCtrl; - uint32_t mFtServiceId; - - std::map caches; - - RsMutex csMtx; /* protect below */ - - std::list > mCacheUpdates; -}; - - -#endif diff --git a/libretroshare/src/dbase/cachetest.h b/libretroshare/src/dbase/cachetest.h deleted file mode 100644 index 59d55b5be..000000000 --- a/libretroshare/src/dbase/cachetest.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * RetroShare FileCache Module: cachetest.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_TEST_CACHE_H -#define MRK_TEST_CACHE_H - -#include "cachestrapper.h" - -#define TESTID 0xffff -#define TESTID2 0xffee - -class CacheTestSource: public CacheSource -{ - public: - CacheTestSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID, false, dir) { return; } -}; - -class CacheTestStore: public CacheStore -{ - public: - CacheTestStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID, false, cft, dir) { return; } -}; - - -class CacheTestMultiSource: public CacheSource -{ - public: - CacheTestMultiSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID2, true, dir) { return; } -}; - -class CacheTestMultiStore: public CacheStore -{ - public: - CacheTestMultiStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID2, true, cft, dir) { return; } -}; - - -#endif - diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc deleted file mode 100644 index 3269469f7..000000000 --- a/libretroshare/src/dbase/fimonitor.cc +++ /dev/null @@ -1,1706 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifdef WINDOWS_SYS -#include "util/rsstring.h" -#include "util/rswin.h" -#endif - -#include "rsserver/p3face.h" -#include "dbase/fimonitor.h" -#include "util/rsdir.h" -#include "util/rsmemory.h" -#include "pqi/authssl.h" -#include "serialiser/rsserviceids.h" -#include "retroshare/rsiface.h" -#include "pqi/p3notify.h" -#include "retroshare/rspeers.h" -#include "retroshare/rstypes.h" -#include "util/folderiterator.h" -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// *********** -//#define FIM_DEBUG 1 -// ***********/ - -FileIndexMonitor::FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid,const std::string& config_dir) - :CacheSource(RS_SERVICE_TYPE_FILE_INDEX, true, cs, cachedir), fiMutex("FileIndexMonitor"), fi(pid), - pendingDirs(false), pendingForceCacheWrite(false), - mForceCheck(false), mInCheck(false), hashCache(config_dir+"/" + "file_cache"),useHashCache(true) - -{ - updatePeriod = 15 * 60; // 15 minutes - reference_time = 0 ; -} - -bool FileIndexMonitor::autoCheckEnabled() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod > 0 ; -} - -bool FileIndexMonitor::rememberHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return useHashCache ; -} -void FileIndexMonitor::setRememberHashFiles(bool b) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ -#ifdef FIM_DEBUG - std::cerr << "Setting useHashCache to " << b << std::endl; -#endif - useHashCache = b ; -} -void FileIndexMonitor::setRememberHashFilesDuration(uint32_t days) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "Setting HashCache duration to " << days << std::endl; -#endif - hashCache.setRememberHashFilesDuration(days) ; -} - -uint32_t FileIndexMonitor::rememberHashFilesDuration() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - return hashCache.rememberHashFilesDuration() ; -} - -// Remove any memory of formerly hashed files that are not shared anymore -void FileIndexMonitor::clearHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - hashCache.clear() ; - hashCache.save() ; -} - -HashCache::HashCache(const std::string& path) - : _path(path) -{ - _max_cache_duration_days = 10 ; // 10 days is the default value. - _files.clear() ; - _changed = false ; - - // check for unencrypted - - std::istream *f = NULL ; - uint64_t file_size ; - - if(RsDirUtil::checkFile( _path+".bin",file_size,false ) ) - { - std::cerr << "Encrypted hash cache file present. Reading it." << std::endl; - - // read the binary stream into memory. - // - void *buffer = rs_malloc(file_size) ; - - if(buffer == NULL) - return ; - - FILE *F = fopen( (_path+".bin").c_str(),"rb") ; - if (!F) - { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << (_path+".bin") << std::endl; - free(buffer); - return; - } - if(fread(buffer,1,file_size,F) != file_size) - { - std::cerr << "Cannot read from file " + _path+".bin" << ": something's wrong." << std::endl; - free(buffer) ; - fclose(F) ; - return ; - } - fclose(F) ; - - void *decrypted_data =NULL; - int decrypted_data_size =0; - - if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) - { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; - free(buffer) ; - return ; - } - free(buffer) ; - - std::string s((char *)decrypted_data,decrypted_data_size) ; - f = new std::istringstream(s) ; - - free(decrypted_data) ; - } - else - { - std::cerr << "Encrypted file cache not present. Trying unencrypted..." << std::endl; - - f = new std::ifstream( (_path+".lst").c_str()) ; - - if(!f->good()) - { - delete f ; - std::cerr << "Unencrypted file cache not present either." << std::endl; - return ; - } - } - - std::streamsize max_line_size = 2000 ; // should be enough. Anyway, if we - // miss one entry, we just lose some - // cache itemsn but this is not too - // much of a problem. - char *buff = new char[max_line_size] ; - -#ifdef FIM_DEBUG - std::cerr << "Loading HashCache from file " << path << std::endl ; - int n=0 ; -#endif - - while(!f->eof()) - { - HashCacheInfo info ; - - f->getline(buff,max_line_size,'\n') ; - std::string name(buff) ; - - f->getline(buff,max_line_size,'\n') ; //if(sscanf(buff,"%llu",&info.size) != 1) break ; - - info.size = 0 ; - sscanf(buff, RsDirUtil::scanf_string_for_uint(sizeof(info.size)), &info.size); - - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.time_stamp)),&info.time_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.modf_stamp)),&info.modf_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; info.hash = RsFileHash(std::string(buff)) ; - -#ifdef FIM_DEBUG - std::cerr << " (" << name << ", " << info.size << ", " << info.time_stamp << ", " << info.modf_stamp << ", " << info.hash << std::endl ; - ++n ; -#endif - _files[name] = info ; - } - - delete[] buff ; - delete f ; -#ifdef FIM_DEBUG - std::cerr << n << " entries loaded." << std::endl ; -#endif -} - -void HashCache::save() -{ - if(!_changed) - { -#ifdef FIM_DEBUG - std::cerr << "Hash cache not changed. Not saving." << std::endl ; -#endif - return; - } - -#ifdef FIM_DEBUG - std::cerr << "Saving Hash Cache to file " << _path << "..." << std::endl ; -#endif - - std::ostringstream f ; - for(std::map::const_iterator it(_files.begin()); - it != _files.end(); - ++it) - { - f << it->first << std::endl ; - f << it->second.size << std::endl; - f << it->second.time_stamp << std::endl; - f << it->second.modf_stamp << std::endl; - f << it->second.hash << std::endl; - } - - void *encryptedData = NULL ; - int encDataLen = 0 ; - - if(!AuthSSL::getAuthSSL()->encrypt( - encryptedData, - encDataLen, - f.str().c_str(), - f.str().length(), - AuthSSL::getAuthSSL()->OwnId())) - { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; - return; - } - - FILE *F = fopen( (_path+".bin.tmp").c_str(),"wb" ) ; - - if(!F) - { - std::cerr << "Cannot open encrypted file cache for writing: " << _path+".bin.tmp" << std::endl; - goto save_free; - } - if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) - { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; - fclose(F) ; - goto save_free; - } - - fclose(F) ; - - RsDirUtil::renameFile(_path+".bin.tmp",_path+".bin") ; -#ifdef FIM_DEBUG - std::cerr << "done." << std::endl ; -#endif - - _changed = false; - -save_free: - free(encryptedData); -} - -bool HashCache::find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) -{ -#ifdef FIM_DEBUG - std::cerr << "HashCache: looking for " << full_path << std::endl ; -#endif - time_t now = time(NULL) ; - std::map::iterator it(_files.find(full_path)) ; - - if(it != _files.end() && (uint64_t)time_stamp == it->second.modf_stamp && size == it->second.size) - { - hash = it->second.hash ; - it->second.time_stamp = now ; -#ifdef FIM_DEBUG - std::cerr << "Found in cache." << std::endl ; -#endif - return true ; - } - else - { -#ifdef FIM_DEBUG - std::cerr << "not found in cache." << std::endl ; -#endif - return false ; - } -} -void HashCache::insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) -{ - HashCacheInfo info ; - info.size = size ; - info.modf_stamp = time_stamp ; - info.time_stamp = time(NULL) ; - info.hash = hash ; - - _files[full_path] = info ; - _changed = true ; - -#ifdef FIM_DEBUG - std::cerr << "Entry inserted in cache: " << full_path << ", " << size << ", " << time_stamp << std::endl ; -#endif -} -void HashCache::clean() -{ -#ifdef FIM_DEBUG - std::cerr << "Cleaning HashCache..." << std::endl ; -#endif - time_t now = time(NULL) ; - time_t duration = _max_cache_duration_days * 24 * 3600 ; // seconds - -#ifdef FIM_DEBUG - std::cerr << "cleaning hash cache." << std::endl ; -#endif - - for(std::map::iterator it(_files.begin());it!=_files.end();) - if(it->second.time_stamp + duration < (uint64_t)now) - { -#ifdef FIM_DEBUG - std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; -#endif - std::map::iterator tmp(it) ; - ++tmp ; - _files.erase(it) ; - it=tmp ; - _changed = true ; - } - else - ++it ; - -#ifdef FIM_DEBUG - std::cerr << "Done." << std::endl; -#endif -} - -FileIndexMonitor::~FileIndexMonitor() -{ - /* Data cleanup - TODO */ -} - -int FileIndexMonitor::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchTerms(keywords, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchBoolExp(exp, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ -#ifdef DEBUG - if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) - std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; -#endif - /* translate/filter results */ - - for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) - { - DirDetails cdetails ; - RequestDirDetails (*rit,cdetails,FileSearchFlags(0u)); -#ifdef FIM_DEBUG - std::cerr << "Filtering candidate " << (*rit)->name << ", flags=" << cdetails.flags << ", peer=" << peer_id ; -#endif - - if(!peer_id.isNull()) - { - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ; - - if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags )) - { - cdetails.id.clear() ; - results.push_back(cdetails); -#ifdef FIM_DEBUG - std::cerr << ": kept" << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << ": discarded" << std::endl ; -#endif - } - else - results.push_back(cdetails); - } - - return !results.empty() ; -} - -bool FileIndexMonitor::findLocalFile(const RsFileHash& hash,FileSearchFlags hint_flags, const RsPeerId& peer_id,std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const -{ - std::list results; - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::findLocalFile() Hash: " << hash << std::endl; -#endif - /* search through the fileIndex */ - fi.searchHash(hash, results); - - if ( !results.empty() ) - { - /* find the full path for the first entry */ - FileEntry *fe = results.front(); - DirEntry *de = fe->parent; /* all files must have a valid parent! */ - - locked_findShareFlagsAndParentGroups(fe,storage_flags,parent_groups) ; - - // turn share flags into hint flags - - FileSearchFlags shflh = peer_id.isNull()?(RS_FILE_HINTS_BROWSABLE|RS_FILE_HINTS_NETWORK_WIDE):rsPeers->computePeerPermissionFlags(peer_id,storage_flags,parent_groups) ; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile: Filtering candidate " << fe->name << ", flags=" << storage_flags << ", hint_flags=" << hint_flags << ", peer_id = " << peer_id << std::endl ; -#endif - - if(peer_id.isNull() || (shflh & hint_flags)) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Name: " << fe->name << std::endl; -#endif - std::string shpath = RsDirUtil::removeRootDir(de->path); - std::string basedir = RsDirUtil::getRootDir(de->path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - if (shpath != "") - { - fullpath += shpath + "/"; - } - fullpath += fe->name; - - size = fe->size; - ok = true; - } -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Path: " << fullpath << std::endl; - std::cerr << "FileIndexMonitor::findLocalFile() Found Size: " << size << std::endl; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor::findLocalFile() discarded" << std::endl ; -#endif - } - } /* UNLOCKED DIRS */ - - return ok; -} - -bool FileIndexMonitor::convertSharedFilePath(std::string path, std::string &fullpath) -{ - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() path: " << path << std::endl; -#endif - - std::string shpath = RsDirUtil::removeRootDir(path); - std::string basedir = RsDirUtil::getRootDir(path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - fullpath += shpath; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() Found Path: " << fullpath << std::endl; -#endif - ok = true; - } - - } /* UNLOCKED DIRS */ - - return ok; -} - - -bool FileIndexMonitor::loadLocalCache(const RsCacheData &data) /* called with stored data */ -{ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadLocalCache(): subid = " << data.cid.subid << ", filename=" << data.name << ", peer id = " << data.pid << std::endl; -#endif - - if(!strcmp(data.name.c_str()+data.name.size()-5,".rsfc"))// this trick allows to load the complete file. Not the one being shared. - { // other files are discarded and re-created in case permissions have changed. - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - /* More error checking needed here! */ - - std::string name = data.name ; - - if ( fi.loadIndex(data.path + '/' + name, RsFileHash(), data.size) ) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Success!"; - std::cerr << std::endl; -#endif - fi.root->row = 0; - fi.root->name = data.pid.toStdString(); // XXX Hack here - TODO - - std::string fname_browsable = data.path + '/' + name ; - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - reference_time = buf.st_mtime ; -#ifdef FIM_DEBUG - std::cerr << "Read new reference time of created file " << fname_browsable << ", to " << reference_time << std::endl; -#endif - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - reference_time = 0 ; - } -#ifdef FIM_DEBUG - std::cerr << "Current reference time is now : " << reference_time << std::endl; -#endif - } - else - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Failed!"; - std::cerr << std::endl; -#endif - reference_time = 0 ; - } - - fi.updateMaxModTime() ; - - // The index is re-saved. - // - we might have new friends - // - the cache system removes old cache items so we need to re-create it. - // - locked_saveFileIndexes(false) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor:: not loading cache item " << data.name << std::endl; -#endif - - return false; -} - -bool FileIndexMonitor::updateCache(const RsCacheData &data,const std::set& destination_peers) /* we call this one */ -{ - return refreshCache(data,destination_peers); -} - - -int FileIndexMonitor::getPeriod() const -{ -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() getting watch period" << std::endl; -#endif - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod ; -} - -void FileIndexMonitor::setPeriod(int period) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - updatePeriod = period; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() Setting watch period to " << updatePeriod << std::endl; -#endif -} - -void FileIndexMonitor::data_tick() -{ - if(autoCheckEnabled()) - updateCycle(); - - int i=0 ; - for(;;++i) - { - if(!isRunning()) - return; - - usleep(1*1000*1000); // 1 sec - - /* check dirs if they've changed */ - if (internal_setSharedDirectories()) - break; - - { - RsStackMutex mtx(fiMutex) ; - - if(i >= abs(updatePeriod)) - break ; - } - } - - if(i < abs(updatePeriod) || autoCheckEnabled()) - updateCycle(); -} - -void FileIndexMonitor::updateCycle() -{ - time_t startstamp = time(NULL); - -#ifdef FIM_DEBUG - std::cerr << "Checking directory for new/modified files. Reference time is " << reference_time << std::endl; -#endif - /* iterate through all out-of-date directories */ - bool moretodo = true; - bool fiMods = false; - - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - mInCheck = true; - } - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_EXAMINING_FILES, "") ; - - std::vector to_hash ; - - bool cache_is_new ; - { - RsStackMutex mtx(fiMutex) ; - cache_is_new = useHashCache && hashCache.empty() ; - } - struct stat64 buf; - - while(isRunning() && moretodo) - { - /* sleep a bit for each loop */ -// csoler: I'm disabling this since it causes a very long update cycle when the number -// of directories to go through is very large. -// -// usleep(100*1000); /* 100 msec */ - - /* check if directories have been updated */ - - if (internal_setSharedDirectories()) /* reset start time */ - startstamp = time(NULL); - - /* Handle a Single out-of-date directory */ - - time_t stamp = time(NULL); - - /* lock dirs from now on */ - RsStackMutex mtx(fiMutex) ; - - DirEntry *olddir = fi.findOldDirectory(startstamp); - - if (!olddir) - { - /* finished */ - moretodo = false; - continue; - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Checking: " << olddir->path << std::endl; -#endif - - FileEntry fe; - - /* determine the full root path */ - std::string dirpath = olddir->path; - std::string rootdir = RsDirUtil::getRootDir(olddir->path); - std::string remdir = RsDirUtil::removeRootDir(olddir->path); - std::string realroot = locked_findRealRoot(rootdir); - std::string realpath = realroot; - - if (remdir != "") - realpath += "/" + remdir; - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " RealPath: " << realpath << std::endl; -#endif - - /* check for the dir existance */ - librs::util::FolderIterator dirIt(realpath); - if (!dirIt.isValid()) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Missing Dir: " << realpath << std::endl; - std::cerr << " Root Dir: " << rootdir << std::endl; - std::cerr << " remdir: " << remdir << std::endl; -#endif - if(directoryMap.end() != directoryMap.find(rootdir) && remdir=="") - { -#ifdef FIM_DEBUG - std::cerr << " This is a root directory. Keeping it empty." << std::endl; -#endif - } - else - { - if (!fi.removeOldDirectory(olddir->parent->path, olddir->name, stamp))/* bad directory - delete */ - { - /* bad... drop out of updateCycle() - hopefully the initial cleanup - * will deal with it next time! - otherwise we're in a continual loop - */ - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << "ERROR Failed to Remove: " << olddir->path << std::endl; - } - continue; - } - } - - /* update this dir - as its valid */ - fe.name = olddir->name; - fi.updateDirEntry(olddir->parent->path, fe, stamp); - - /* update the directories and files here */ - std::map::iterator dit; - std::map::iterator fit; - - /* flag existing subdirs as old */ - for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); ++dit) - { - fe.name = (dit->second)->name; - /* set the age as out-of-date so that it gets checked */ - fi.updateDirEntry(olddir->path, fe, 0); - } - - /* now iterate through the directory... - * directories - flags as old, - * files checked to see if they have changed. (rehashed) - */ - - to_hash.push_back(DirContentToHash()) ; - to_hash.back().realpath = realpath ; - to_hash.back().dirpath = dirpath ; - - for(;dirIt.isValid() && isRunning();dirIt.next()) - { - /* check entry type */ - std::string fname; - dirIt.d_name(fname); - std::string fullname = realpath + "/" + fname; -#ifdef FIM_DEBUG - std::cerr << "calling stats on " << fullname <path, fe, 0); - } - else if (S_ISREG(buf.st_mode)) - { - /* is file */ - bool toadd = false; -#ifdef FIM_DEBUG - std::cerr << "Is File: " << fullname << std::endl; -#endif - - fe.name = fname; - fe.size = buf.st_size; - fe.modtime = buf.st_mtime; - - /* check if it exists already */ - fit = olddir->files.find(fname); - if (fit == olddir->files.end()) - { - /* needs to be added */ -#ifdef FIM_DEBUG - std::cerr << "File Missing from List:" << fname << std::endl; -#endif - toadd = true; - } - else - { - /* check size / modtime are the same */ - // if reference_time was not inited, we revert to the old method: we test the saved mod time for the file - // versus the measured mod time. If reference is inited (this is what should always happen) we compare the measured - // mod time with the reference time. - // - if ((fe.size != (fit->second)->size) || (reference_time==0 && fe.modtime != (fit->second)->modtime) || fe.modtime > reference_time) //(fit->second)->modtime)) - { -#ifdef FIM_DEBUG - std::cerr << "File ModTime/Size changed:" << fname << std::endl; - std::cerr << "fe.modtime = " << fe.modtime << std::endl; - std::cerr << "fit.mdtime = " << fit->second->modtime << std::endl; -#endif - toadd = true; - } - else - fe.hash = (fit->second)->hash; /* keep old info */ - } - if (toadd) - { - /* push onto Hash List */ -#ifdef FIM_DEBUG - std::cerr << "Adding to Update List: "; - std::cerr << olddir->path; - std::cerr << fname << std::endl; -#endif - to_hash.back().fentries.push_back(fe); - fiMods = true ; - } - else /* update with new time */ - { -#ifdef FIM_DEBUG - std::cerr << "File Hasn't Changed:" << fname << std::endl; -#endif - fi.updateFileEntry(olddir->path, fe, stamp); - - if(cache_is_new) - hashCache.insert(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash) ; - } - } - } -#ifdef FIM_DEBUG - else - std::cout << "stat error " << errno << std::endl ; -#endif - } - - if(to_hash.back().fentries.empty()) - to_hash.pop_back() ; - - /* now we unlock the lock, and iterate through the - * next files - hashing them, before adding into the system. - */ - /* for safety - blank out data we cannot use (TODO) */ - olddir = NULL; - - /* close directory */ - dirIt.closedir(); - } - - // Now, hash all files at once. - // - if(isRunning() && !to_hash.empty()) - hashFiles(to_hash) ; - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - - int cleanedCount = 0; - - { /* LOCKED DIRS */ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* finished update cycle - cleanup extra dirs/files that - * have not had their timestamps updated. - */ - - cleanedCount = fi.cleanOldEntries(startstamp) ; - -#ifdef FIM_DEBUG - /* print out the new directory structure */ -// fi.printFileIndex(std::cerr); -#endif - /* now if we have changed things -> restore file/hash it/and - * tell the CacheSource - */ - - if (pendingForceCacheWrite) - { - pendingForceCacheWrite = false; - fiMods = true; - } - - if (fiMods) - { - reference_time = locked_saveFileIndexes(true) ; -#ifdef FIM_DEBUG - std::cerr << "Index saved. New reference time is " << reference_time << std::endl; -#endif - } - - fi.updateHashIndex() ; // update hash map that is used to accelerate search. - fi.updateMaxModTime() ; // Update modification times for proper display. - - mInCheck = false; - - if(useHashCache) - { - hashCache.clean() ; - hashCache.save() ; - } - } - - if (cleanedCount > 0) { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } -} - -static std::string friendlyUnit(uint64_t val) -{ - const std::string units[5] = {"B","KB","MB","GB","TB"}; - char buf[50] ; - - double fact = 1.0 ; - - for(unsigned int i=0; i<5; ++i) - if(double(val)/fact < 1024.0) - { - sprintf(buf,"%2.2f",double(val)/fact) ; - return std::string(buf) + " " + units[i]; - } - else - fact *= 1024.0f ; - - sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ; - return std::string(buf) + " TB"; -} - - -void FileIndexMonitor::hashFiles(const std::vector& to_hash) -{ - // Size interval at which we save the file lists - static const uint64_t MAX_SIZE_WITHOUT_SAVING = 10737418240ull ; // 10 GB - - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - time_t stamp = time(NULL); - - // compute total size of files to hash - uint64_t total_size = 0 ; - uint32_t n_files = 0 ; - - for(uint32_t i=0;inotifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - - std::string real_path = RsDirUtil::makePath(to_hash[i].realpath, fe.name); - - // 1st look into the hash cache if this file already exists. - // - if(useHashCache && hashCache.find(real_path,fe.size,fe.modtime,fe.hash)) - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - else if(RsDirUtil::getFileHash(real_path, fe.hash,fe.size, this)) // not found, then hash it. - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* update fileIndex with new time */ - /* update with new time */ - - // Check again that the hashed file hasn't been modified since the beginning of the hashing process. - // If so, drop it. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(real_path, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(real_path.c_str(), &buf)) -#endif - { - if(buf.st_mtime != fe.modtime) - std::cerr << "File " << real_path << " has been modified while being hashed. It will be dropped to avoid data inconsistency" << std::endl; - else - { - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - - hashed_size += to_hash[i].fentries[j].size ; - - // Update the hash cache - // - if(useHashCache) - hashCache.insert(real_path,fe.size,fe.modtime,fe.hash) ; - } - } - } - else - std::cerr << "Failed to Hash File " << fe.name << std::endl; - - size += to_hash[i].fentries[j].size ; - - // don't hit the disk too hard! - usleep(10*1000); // 10 msec - - // Save the hashing result every 60 seconds, so has to save what is already hashed. -#ifdef FIM_DEBUG - std::cerr << "size - last_save_size = " << hashed_size - last_save_size << ", max=" << MAX_SIZE_WITHOUT_SAVING << std::endl ; -#endif - - if(hashed_size > last_save_size + MAX_SIZE_WITHOUT_SAVING) - { - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_SAVE_FILE_INDEX, "") ; - - //Waiting 1 sec before pass to other hash ??is it really needed?? - usleep(1*1000*1000); // 1 sec - - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - fi.updateHashIndex() ; - FileIndexMonitor::locked_saveFileIndexes(true) ; - last_save_size = hashed_size ; - - if(useHashCache) - hashCache.save() ; - } - - // check if thread is running - running = isRunning(); - } - - fi.updateHashIndex() ; - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - -time_t FileIndexMonitor::locked_saveFileIndexes(bool update_cache) -{ - /* store to the cacheDirectory */ - - std::string path = getCacheDir(); - - // Multiple files are saved: for every kind of peers, the set of browsable files will be different. A specific file is - // prepared for all situations, and shared by all peers having the same situation. - // - // A complete file collection is also saved, and serves as memory for the FileIndexMonitor system. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() FileIndex modified ... updating" << std::endl; -#endif - // Make for each peer the list of forbidden shared directories. Make a separate cache file for each different set. - // To figure out which sets are different, we index them by the set of forbidden indexes from the directory list. - // This is probably a bit costly, but we can't suppose that the number of shared directories is bounded. - // - std::list all_friend_ids ; - rsPeers->getFriendList(all_friend_ids); - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle(): got list of all friends." << std::endl ; - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - std::cerr << " " << *it << std::endl; -#endif - - std::map, std::set > peers_per_directory_combination ; - - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "About to save, with the following restrictions:" << std::endl ; - std::cerr << "Peer : " << *it << std::endl; -#endif - - std::set forbidden_dirs ; - for(std::map::const_iterator dit(directoryMap.begin());dit!=directoryMap.end();++dit) - { -#ifdef FIM_DEBUG - std::cerr << " dir=" << dit->first << ", " ; - std::cerr << "parent groups: " ; - for(std::list::const_iterator mit(dit->second.parent_groups.begin());mit!=dit->second.parent_groups.end();++mit) - std::cerr << (*mit) << ", " ; - std::cerr << std::endl;; -#endif - - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(*it,dit->second.shareflags,dit->second.parent_groups) ; - - if(!(permission_flags & RS_FILE_HINTS_BROWSABLE)) - { -#ifdef FIM_DEBUG - std::cerr << "forbidden" << std::endl; -#endif - forbidden_dirs.insert(dit->first) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "autorized" << std::endl; -#endif - } - - peers_per_directory_combination[forbidden_dirs].insert(*it) ; - } - RsPeerId ownId = rsPeers->getOwnId() ; - peers_per_directory_combination[std::set()].insert(ownId) ; // add full configuration to self, i.e. no forbidden directories. - - int n=0 ; - time_t now = time(NULL) ; - time_t mod_time = 0 ; - - for(std::map, std::set >::const_iterator it(peers_per_directory_combination.begin()); - it!=peers_per_directory_combination.end();++it,++n) - { - std::string tmpname_browsable; - - if(it->first.empty()) - rs_sprintf(tmpname_browsable, "fc-own-%ld.rsfc",now,n); - else - rs_sprintf(tmpname_browsable, "fc-own-%ld.%04d",now,n); - - std::string fname_browsable = path + "/" + tmpname_browsable; - -#ifdef FIM_DEBUG - std::cerr << "Sending file list: " << std::endl; - std::cerr << " filename : " << tmpname_browsable << std::endl; - std::cerr << " to peers : " << std::endl; - for(std::set::const_iterator itt(it->second.begin());itt!= it->second.end();++itt) - std::cerr << " " << *itt << std::endl; - std::cerr << " forbidden : " << std::endl; - for(std::set::const_iterator itt(it->first.begin());itt!= it->first.end();++itt) - std::cerr << " " << *itt << std::endl; -#endif - - RsFileHash hash ; - uint64_t size ; - -#ifdef FIM_DEBUG - std::cerr << "writing file " << fname_browsable << std::endl; -#endif - fi.saveIndex(fname_browsable, hash, size,it->first); // save only browsable files - - if(size > 0) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() saved with hash:" << hash << std::endl; -#endif - - /* should clean up the previous cache.... */ - - /* flag as new info */ - RsCacheData data; - data.pid = fi.root->id; - data.cid.type = getCacheType(); - data.cid.subid = n; - data.path = path; - data.name = tmpname_browsable; - data.hash = hash; - data.size = size; - data.recvd = time(NULL); - - for(std::set::const_iterator ff(it->second.begin());ff!=it->second.end();++ff) - _cache_items_per_peer[*ff] = data ; - - data.cid.subid = n; - - if(update_cache) - updateCache(data,it->second); - } - - if(it->first.empty()) - { - // Computes the reference time. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - mod_time = buf.st_mtime ; - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - mod_time = 0 ; - } - } - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() called updateCache()"; - std::cerr << std::endl; -#endif - return mod_time ; -} - -bool FileIndexMonitor::cachesAvailable(const RsPeerId &pid,std::map &ids) -{ - lockData() ; -#ifdef FIM_DEBUG - std::cerr << "In cachesAvailable..." << std::endl; -#endif - - // Go through the list of saved cache items for that particular peer. - // - ids.clear() ; - std::map::const_iterator it(_cache_items_per_peer.find(pid)) ; - RsPeerId ownId = rsPeers->getOwnId(); - - if(it != _cache_items_per_peer.end()) - { - ids[it->second.cid] = it->second ; - - if(pid != ownId) - ids[it->second.cid].cid.subid = 0 ; // Force subid to be 0, so that it's - // not going to be mixed up at the client with other files received if the - // subid changes for that peer. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor: caches available for peer " << pid << ": " << it->second.name << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "No cache item for peer " << pid << std::endl; -#endif - - unlockData() ; - - return true ; -} - -void FileIndexMonitor::updateShareFlags(const SharedDirInfo& dir) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - bool fimods = false ; -#ifdef FIM_DEBUG - std::cerr << "*** FileIndexMonitor: Updating flags for " << dir.filename << " to " << dir.shareflags << std::endl ; -#endif - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - for(std::list::iterator it(pendingDirList.begin());it!=pendingDirList.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing pending dir " << (*it).filename << std::endl ; -#endif - if((*it).filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating to " << (*it).shareflags << "!!" << std::endl ; -#endif - (*it).shareflags = dir.shareflags ; - (*it).parent_groups = dir.parent_groups ; - break ; - } - } - - for(std::map::iterator it(directoryMap.begin());it!=directoryMap.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing " << (*it).second.filename << std::endl ; -#endif - if((*it).second.filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating from " << it->second.shareflags << "!!" << std::endl ; -#endif - (*it).second.shareflags = dir.shareflags ; - (*it).second.parent_groups = dir.parent_groups ; - fimods = true ; - break ; - } - } - } - if(fimods) - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; - } - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - /* interface */ -void FileIndexMonitor::setSharedDirectories(const std::list& dirs) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - std::list checkeddirs; - - std::list::const_iterator it; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setSharedDirectories() :\n"; -#endif - - for(it = dirs.begin(); it != dirs.end(); ++it) - { - -#ifdef FIM_DEBUG - std::cerr << "\t" << (*it).filename; - std::cerr << std::endl; -#endif - - checkeddirs.push_back(*it); - -#ifdef REMOVED_CODE - // this code has been removed because it prevents unmounted shared directories to stay in the list of shared files. It's better - // to keep them showing empty than removing them completely. - // - /* check if dir exists before adding in */ -// std::string path = (*it).filename; -// if (!RsDirUtil::checkDirectory(path)) -// { -//#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::setSharedDirectories()"; -// std::cerr << " Ignoring NonExistant SharedDir: " << path << std::endl; -//#endif -// } -// else -// { -// checkeddirs.push_back(*it); -// } -#endif - } - - { - RsStackMutex stack(fiMutex) ;/* LOCKED DIRS */ - - pendingDirs = true; - pendingDirList = checkeddirs; - } - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - /* interface */ -void FileIndexMonitor::getSharedDirectories(std::list &dirs) -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - /* must provide pendingDirs, as other parts depend on instanteous response */ - if (pendingDirs) - dirs = pendingDirList; - else - { - /* get actual list (not pending stuff) */ - std::map::const_iterator it; - - for(it = directoryMap.begin(); it != directoryMap.end(); ++it) - dirs.push_back(it->second) ; - } -} - - - /* interface */ -void FileIndexMonitor::forceDirectoryCheck() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!mInCheck) - mForceCheck = true; -} - - -void FileIndexMonitor::forceDirListsRebuildAndSend() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; -} - - /* interface */ -bool FileIndexMonitor::inDirectoryCheck() -{ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - return mInCheck; -} - - -bool FileIndexMonitor::internal_setSharedDirectories() -{ - bool changed = false; - - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!pendingDirs) - { - if (mForceCheck) - { - mForceCheck = false; - return true; - } - - return false; - } - - mForceCheck = false; - pendingDirs = false; - pendingForceCacheWrite = true; - - /* clear old directories */ - directoryMap.clear(); - - /* iterate through the directories */ - std::list::iterator it; - std::map::const_iterator cit; - for(it = pendingDirList.begin(); it != pendingDirList.end(); ++it) - { - /* get the head directory */ - std::string root_dir = (*it).filename; - std::string top_dir = it->virtualname; - if (top_dir.empty()) { - top_dir = RsDirUtil::getTopDir(root_dir); - } - - /* if unique -> add, else add modifier */ - bool unique = false; - for(int i = 0; !unique; ++i) - { - std::string tst_dir = top_dir; - if (i > 0) - { - rs_sprintf_append(tst_dir, "-%d", i); - } - if (directoryMap.end()== (cit=directoryMap.find(tst_dir))) - { - unique = true; - /* store calculated name */ - it->virtualname = tst_dir; - /* add it! */ - directoryMap[tst_dir.c_str()] = *it; -#ifdef FIM_DEBUG - std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl; -#endif - } - } - } - - pendingDirList.clear(); - - /* now we've decided on the 'root' dirs set them to the - * fileIndex - */ - std::list topdirs; - for(cit = directoryMap.begin(); cit != directoryMap.end(); ++cit) - { - topdirs.push_back(cit->first); - } - - if (fi.setRootDirectories(topdirs, 0) > 0) - { - changed = true; - } - - locked_saveFileIndexes(true) ; - } - - if (changed) - { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } - - return true; -} - -/* lookup directory function */ -std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const -{ - /**** MUST ALREADY BE LOCKED ****/ - std::string realroot = ""; - - std::map::const_iterator cit; - if (directoryMap.end()== (cit=directoryMap.find(rootdir))) - { - std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: "; - std::cerr << rootdir << std::endl; - } - else - { - realroot = cit->second.filename; - } - - return realroot; -} - -int FileIndexMonitor::RequestDirDetails(const std::string& path, DirDetails& details) const -{ - /* lock it up */ - RsStackMutex mutex(fiMutex) ; - return fi.extractData(path,details) ; -} - -uint32_t FileIndexMonitor::getType(void *ref) const -{ - RsStackMutex mutex(fiMutex) ; - - return fi.getType(ref) ; -} -int FileIndexMonitor::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - - RsStackMutex mutex(fiMutex) ; - -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - /* root case */ - -#ifdef FIM_DEBUG2 - fi.root->checkParentPointers(); -#endif - - // If ref is NULL, we build a root node - - if (ref == NULL) - { -#ifdef FI_DEBUG2 - std::cerr << "FileIndex::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - /* local only */ - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fi.root->name; - stub.ref = fi.root; - details.children.push_back(stub); - details.count = 1; - - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = "root"; - details.hash.clear() ; - details.path = "root"; - details.age = 0; - details.flags.clear() ; - details.min_age = 0 ; - - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - if(!b) - return false ; - - // look for the top level and setup flags accordingly - // The top level directory is the first dir in parents for which - // dir->parent->parent == NULL - - if(ref != NULL) - { - FileEntry *file = (FileEntry *) ref; - locked_findShareFlagsAndParentGroups(file,details.flags,details.parent_groups) ; - } - return true ; -} - -void FileIndexMonitor::locked_findShareFlagsAndParentGroups(FileEntry *file,FileStorageFlags& flags,std::list& parent_groups) const -{ - flags.clear() ; - static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_BROWSABLE_OTHERS | DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_GROUPS | DIR_FLAGS_NETWORK_WIDE_GROUPS ; - - DirEntry *dir = dynamic_cast(file) ; - if(dir == NULL) - dir = dynamic_cast(file->parent) ; - - if(dir != NULL && dir->parent != NULL) - while(dir->parent->parent != NULL) - dir = dir->parent ; - - if(dir != NULL && dir->parent != NULL) - { -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails: top parent name=" << dir->name << std::endl ; -#endif - std::map::const_iterator it = directoryMap.find(dir->name) ; - - if(it == directoryMap.end()) - std::cerr << "*********** ERROR *********** In " << __PRETTY_FUNCTION__ << std::endl ; - else - { - flags = it->second.shareflags ; - flags &= PERMISSION_MASK ; - flags &= ~DIR_FLAGS_NETWORK_WIDE_GROUPS ; // Disabling this flag for now, because it has inconsistent effects. - parent_groups = it->second.parent_groups ; - } -#ifdef FIM_DEBUG2 - std::cerr << "flags = " << flags << std::endl ; -#endif - } -} - - diff --git a/libretroshare/src/dbase/fimonitor.h b/libretroshare/src/dbase/fimonitor.h deleted file mode 100644 index 2cdddb4af..000000000 --- a/libretroshare/src/dbase/fimonitor.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_MONITOR_H -#define FILE_INDEX_MONITOR_H - -#include "dbase/cachestrapper.h" -#include "dbase/findex.h" -#include "util/rsthreads.h" -#include "retroshare/rsfiles.h" - -/****************************************************************************************** - * The Local Monitoring Class: FileIndexMonitor. - * - * This periodically scans the directory tree, and updates any modified directories/files. - * - *****************************************************************************************/ - -/****************************************************************************************** - STILL TODO: - - (1) Implement Hash function. - -bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi); - - (2) Add Shared directory controls to Monitor. - -int FileIndexMonitor::addSharedDirectory(std::path); -int FileIndexMonitor::removeSharedDirectory(std::path); -std::string FileIndexMonitor::findRealRoot(std::string base); - - These must be split into / and the mapping saved. - eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1" - This code has been written already, and can just be moved over. - - FOR LATER: - (2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze) - (3) Add Load/Store interface to FileIndexMonitor. (later) - (4) Integrate with real Thread/Mutex code (last thing to do) - -******************************************************************************************/ - - -class DirContentToHash -{ - public: - std::vector fentries ; - - std::string realpath ; - std::string dirpath ; -}; - -class HashCache -{ - public: - HashCache(const std::string& save_file_name) ; - - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; - - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; - - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; -}; - -/****************************************************************************************** - * FileIndexMonitor - *****************************************************************************************/ - -class FileIndexMonitor: public CacheSource, public RsTickingThread -{ - public: - FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir); - virtual ~FileIndexMonitor(); - - /* external interface for filetransfer */ - bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; - - int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) ; - int SearchBoolExp(Expression *exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - int filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - - /* external interface for local access to files */ - bool convertSharedFilePath(std::string path, std::string &fullpath); - - - /* Interacting with CacheSource */ - /* overloaded from CacheSource */ - virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */ - bool updateCache(const RsCacheData &data,const std::set& dest_peers); /* we call when we have a new cache for others */ - - - /* the FileIndexMonitor inner workings */ - //virtual void run(std::string& currentJob); /* overloaded from RsThread */ - //void updateCycle(std::string& currentJob); - virtual void data_tick(); /* overloaded from RsThread */ - void updateCycle(); - - // Interface for browsing dir hirarchy - int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ; - uint32_t getType(void*) const ; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; - - // set/update shared directories - virtual void setSharedDirectories(const std::list& dirs); - void getSharedDirectories(std::list& dirs); - void updateShareFlags(const SharedDirInfo& info) ; - - void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed - void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files. - bool inDirectoryCheck(); - - /* util fns */ - - // from CacheSource - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids) ; - - protected: - // Sets/gets the duration period within which already hashed files are remembered. - // - void setRememberHashFilesDuration(uint32_t days) ; - uint32_t rememberHashFilesDuration() const ; - void setRememberHashFiles(bool) ; - bool rememberHashFiles() ; - // Remove any memory of formerly hashed files that are not shared anymore - void clearHashFiles() ; - void setPeriod(int insecs); - int getPeriod() const; - - bool autoCheckEnabled() const ; - - private: - /* the mutex should be locked before calling these 3. */ - - // Saves file indexs and update the cache. Returns the name of the main - // file index, which becomes the new reference file for mod times. - // - time_t locked_saveFileIndexes(bool update_cache) ; - - // Finds the share flags associated with this file entry. - void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list &parent_groups) const ; - - std::string locked_findRealRoot(std::string base) const; - - void hashFiles(const std::vector& to_hash) ; - bool hashFile(std::string path, FileEntry &fi); /* To Implement */ - - /* data */ - - mutable RsMutex fiMutex; - - FileIndex fi; - - int updatePeriod; - std::map directoryMap; /* used by findRealRoot */ - - /* flags to kick - if we were busy or sleeping */ - bool pendingDirs; - bool pendingForceCacheWrite; - - /* flags to force Check, to tell if we're in check */ - bool mForceCheck; - bool mInCheck; - - std::list pendingDirList; - bool internal_setSharedDirectories(); - - HashCache hashCache ; - bool useHashCache ; - - std::map _cache_items_per_peer ; // stored the cache items to be sent to each peer. - - // This file is the location of the current index file. When checking for new files, we compare the modification time - // of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer. - // - time_t reference_time ; -}; - - -#endif - - diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc deleted file mode 100644 index 17b5dba0b..000000000 --- a/libretroshare/src/dbase/findex.cc +++ /dev/null @@ -1,1516 +0,0 @@ -/* - * RetroShare FileCache Module: findex.cc - * - * Copyright 2004-2007 by Robert Fernie, Kefei Zhou. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include -#include -#include "dbase/findex.h" -#include "retroshare/rsexpr.h" -#include "util/rsdir.h" -#include "util/rsstring.h" -#include "util/rscompress.h" - -#include -#include -#include -#include -#ifdef __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include -#include - -#include -#include - -// This char is used to separate fields in the file list cache. It is supposed to be -// sufficiently safe on all systems. -// -static const char FILE_CACHE_SEPARATOR_CHAR = '|' ; - -/**** - * #define FI_DEBUG 1 - * #define FI_DEBUG_ALL 1 - ****/ - -static RsMutex FIndexPtrMtx("FIndexPtrMtx") ; -#ifdef __MACH__ -std::unordered_set FileIndex::_pointers ; -#else -std::tr1::unordered_set FileIndex::_pointers ; -#endif -void FileIndex::registerEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.insert(p) ; -} -void FileIndex::unregisterEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.erase(p) ; -} -bool FileIndex::isValid(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - return _pointers.find(p) != _pointers.end() ; -} - -DirEntry::~DirEntry() -{ - /* cleanup */ - std::map::iterator dit; - std::map::iterator fit; - - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - FileIndex::unregisterEntry((void*)dit->second) ; - delete (dit->second); - } - subdirs.clear(); - - for(fit = files.begin(); fit != files.end(); ++fit) - { - FileIndex::unregisterEntry((void*)fit->second) ; - delete (fit->second); - } - files.clear(); -} - - -int DirEntry::checkParentPointers() -{ -#ifdef FI_DEBUG - updateChildRows(); - std::map::iterator dit; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - /* debug check */ - (dit->second)->checkParentPointers(); - } -#endif - return 1; -} - - - -int DirEntry::updateChildRows() -{ - /* iterate through children and set row (parent should be good) */ - std::map::iterator dit; - std::map::iterator fit; - int i = 0; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((dit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING subdir Parent pointer invalid!"; - std::cerr << std::endl; - (dit->second)->parent = this; - } -#endif - (dit->second)->row = i++; - } - - for(fit = files.begin(); fit != files.end(); ++fit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((fit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING file Parent pointer invalid!"; - std::cerr << std::endl; - (fit->second)->parent = this; - } -#endif - (fit->second)->row = i++; - } - return 1; -} - - -int DirEntry::removeDir(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeDir() Cleaning up dir: " << name; - std::cerr << std::endl; -#endif - ndir = (it->second); - - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - -int DirEntry::removeFile(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - - if (files.end() != (it = files.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeFile() removing File: " << name; - std::cerr << std::endl; -#endif - nfile = (it->second); - - files.erase(it); - FileIndex::unregisterEntry((void*)nfile) ; - delete nfile; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeFile() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldDir(const std::string& name, time_t old) -{ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { - ndir = (it->second); - if (ndir->updtime < old) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Removing Old dir: " << name; - std::cerr << std::endl; -#endif - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - - /* update row counters */ - updateChildRows(); - return 1; - } -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Keeping UptoDate dir: " << name; - std::cerr << std::endl; -#endif - return 0; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldEntries(time_t old, bool recursive) -{ - /* remove old dirs from our lists -> then do children and files */ - - /* get all dirs with old time */ - std::list removeList; - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - removeList.push_back(it->second); - } - } - - int count = removeList.size(); - - /* now remove the old entries */ - std::list::iterator rit; - for(rit = removeList.begin(); rit != removeList.end(); ++rit) - { - removeDir((*rit)->name); - } - - if (recursive) - { - /* now handle children */ - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, recursive); - } - } - - /* now handle files similarly */ - std::list removeFileList; - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - if ((fit->second)->updtime < old) - { - removeFileList.push_back(fit->second); - } - } - - count += removeFileList.size(); - - /* now remove the old entries */ - std::list::iterator rfit; - for(rfit = removeFileList.begin(); rfit != removeFileList.end(); ++rfit) - { - removeFile((*rfit)->name); - } - - return count; -} - - -DirEntry *DirEntry::findOldDirectory(time_t old) -{ - /* check if one of our directories is old ... - */ - - /* get all dirs with old time */ - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - return (it->second); - } - } - - /* - * else check chlidren. - */ - - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - DirEntry *olddir = (it->second)->findOldDirectory(old); - if (olddir) - { - return olddir; - } - } - - return NULL; -} - - -DirEntry *DirEntry::findDirectory(const std::string& fpath) -{ - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { -#ifdef FI_DEBUG - std::cerr << "DirEntry::findDirectory() Missing subdir:"; - std::cerr << "\"" << nextdir << "\""; - std::cerr << std::endl; -#endif - return NULL; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - if (rempath == "") - { - return it->second; - } - - - // Adding more lenient directory look up. - // returns lower directory to fpath is a FILE. - DirEntry *subdir = (it->second)->findDirectory(rempath); - if (subdir) - return subdir; - else - return it->second; -} - - -int DirEntry::updateDirectories(const std::string& fpath, int new_pop, int new_modtime) -{ - int ret = 1; - if (path != "") /* if not there -> continue down tree */ - { - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { - return 0; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - ret = (it->second)->updateDirectories(rempath, new_pop, new_modtime); - } - - if (ret) /* if full path is okay -> update and return ok */ - { - /* this is assumes that pop always increases! */ - if (new_pop > pop) - { - pop = new_pop; - } - if (new_modtime > modtime) - { - modtime = new_modtime; - } - } - return ret; -} - -DirEntry *DirEntry::updateDir(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() == (it = subdirs.find(fe.name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Adding Entry"; - std::cerr << std::endl; -#endif - ndir = new DirEntry(); - FileIndex::registerEntry((void*)ndir) ; - ndir -> parent = this; - ndir -> path = path + "/" + fe.name; - ndir -> name = fe.name; - ndir -> pop = 0; - ndir -> modtime = 0; - ndir -> updtime = utime; - - subdirs[fe.name] = ndir; - - /* update row counters */ - updateChildRows(); - return ndir; - } - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Updating Entry"; - std::cerr << std::endl; -#endif - - /* update utime */ - ndir = (it->second); - ndir->updtime = utime; - - return ndir; -} - - -FileEntry *DirEntry::updateFile(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - if (files.end() == (it = files.find(fe.name))) - { - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Adding Entry"; - std::cerr << std::endl; -#endif - - nfile = new FileEntry(); - FileIndex::registerEntry((void*)nfile) ; - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> pop = 0; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - - files[fe.name] = nfile; - - /* update row counters */ - updateChildRows(); - return nfile; - } - - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Updating Entry"; - std::cerr << std::endl; -#endif - - - /* update utime */ - nfile = (it->second); - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - //nfile -> pop = 0; // not handled here. - - return nfile; -} - - -int FileEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - - rs_sprintf_append(out, "file %03d [%ld/%ld] : ", row, updtime, modtime); - - if (parent) - out += parent->path; - else - out += "[MISSING PARENT]"; - - rs_sprintf_append(out, " %s [ s: %lld ] ==> [ %s ]\n", name.c_str(), size, hash.toStdString().c_str()); - - return 1; -} - - -int DirEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - rs_sprintf_append(out, "dir %03d [%ld] : %s\n", row, updtime, path.c_str()); - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->print(out); - } - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - (fit->second)->print(out); - } - return 1; -} - -FileIndex::FileIndex(const RsPeerId& pid) -{ - root = new PersonEntry(pid); - registerEntry(root) ; - _file_hashes.clear() ; -} - -FileIndex::~FileIndex() -{ - FileIndex::unregisterEntry((void*)root) ; - delete root; -} - -int FileIndex::setRootDirectories(const std::list &inlist, time_t updtime) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = 0; - } - - std::list::const_iterator ait; - FileEntry fe; - time_t utime = 1; - for(ait = inlist.begin(); ait != inlist.end(); ++ait) - { - fe.name = (*ait); - - /* see if it exists */ - root->updateDir(fe, utime); - } - - /* remove all dirs with zero time (non recursive) */ - int cleanedCount = root->removeOldEntries(utime, false); - - /* now flag remaining directories with correct update time */ - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = updtime; - } - - // update file hash index. - - updateHashIndex() ; - - return cleanedCount; -} - -void FileIndex::updateHashIndex() -{ - _file_hashes.clear() ; - recursUpdateHashIndex(root) ; -} - -void FileIndex::recursUpdateHashIndex(DirEntry *dir) -{ - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - recursUpdateHashIndex(it->second) ; - - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - _file_hashes[it->second->hash] = it->second ; -} - -void FileIndex::updateMaxModTime() -{ - RecursUpdateMaxModTime(root) ; -} -void FileIndex::RecursUpdateMaxModTime(DirEntry *dir) -{ - time_t max_mod_t = 0 ; - - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - { - RecursUpdateMaxModTime(it->second) ; - max_mod_t = std::max(max_mod_t, it->second->most_recent_time) ; - } - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - max_mod_t = std::max(max_mod_t, it->second->modtime) ; - - dir->most_recent_time = max_mod_t ; -} - -int FileIndex::getRootDirectories(std::list &outlist) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - outlist.push_back(it->first); - } - return 1; -} - -/* update (index building) */ -DirEntry *FileIndex::updateDirEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateDirEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = NULL; - if (fpath == "") - { - parent = root; - } - else - { - parent = root->findDirectory(fpath); - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateDirEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - return parent -> updateDir(fe, utime); -} - - -FileEntry *FileIndex::updateFileEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateFileEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = root->findDirectory(fpath); - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - - return parent -> updateFile(fe, utime); -} - - -DirEntry *FileIndex::findOldDirectory(time_t old) /* finds directories older than old */ -{ - DirEntry *olddir = root->findOldDirectory(old); -#ifdef FI_DEBUG - - std::cerr << "FileIndex::findOldDirectory(" << old << ") -> "; - if (olddir) - { - std::cerr << olddir->path; - } - else - { - std::cerr << "NONE"; - } - std::cerr << std::endl; - -#endif - return olddir; -} - -int FileIndex::removeOldDirectory(const std::string& fpath, const std::string& name, time_t old) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::removeOldDirectory() Path: \""; - std::cerr << fpath << "\"" << " + \"" << name << "\""; - std::cerr << std::endl; -#endif - - /* because of this find - we cannot get a child of - * root (which is what we want!) - */ - - DirEntry *parent = root->findDirectory(fpath); - /* for root directory case ... no subdir. */ - if (fpath == "") - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() removing a root dir"; - std::cerr << std::endl; -#endif - parent = root; - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() NULL parent"; - std::cerr << std::endl; -#endif - return 0; - } - return parent -> removeOldDir(name, old); -} - - -int FileIndex::cleanOldEntries(time_t old) /* removes entries older than old */ -{ - int count = 0; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, true); - } - return count; -} - - - -int FileIndex::printFileIndex(std::ostream &out) -{ - std::string sout ; - printFileIndex(sout) ; - out << sout << std::endl; - - return 1 ; -} -int FileIndex::printFileIndex(std::string &out) -{ - out += "FileIndex::printFileIndex()\n"; - root->print(out); - return 1; -} - -int FileIndex::loadIndex(const std::string& filename, const RsFileHash& expectedHash, uint64_t size) -{ - FILE *file = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; - - if (!file) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex error opening file: " << filename; - std::cerr << std::endl; -#endif - return 0; - } - - std::string s ; - - { - /* load file into memory, close file */ - RsTemporaryMemory compressed_data(size) ; - - if(!compressed_data) - { - std::cerr << "FileIndex::loadIndex(): can't allocate memory for " << size << " bytes." << std::endl; - fclose(file); - return 0; - } - uint64_t bytesread = 0 ; - if(size != (bytesread = fread(compressed_data,1,size,file))) - { - std::cerr << "FileIndex::loadIndex(): can't read " << size << " bytes from file " << filename << ". Only " << bytesread << " actually read." << std::endl; - fclose(file); - return 0; - } - fclose(file); - - RsFileHash tmpout = RsDirUtil::sha1sum((unsigned char *)(compressed_data),size); - - // /* calculate hash */ - // unsigned char sha_buf[SHA_DIGEST_LENGTH]; - // SHA_CTX *sha_ctx = new SHA_CTX; - // SHA1_Init(sha_ctx); - // SHA1_Update(sha_ctx, s.c_str(), s.length()); - // SHA1_Final(&sha_buf[0], sha_ctx); - // delete sha_ctx; - // - // std::string tmpout; - // for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) - // { - // rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); - // } - - if (!expectedHash.isNull() && expectedHash != tmpout) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex expected hash does not match" << std::endl; - std::cerr << "Expected hash: " << expectedHash << std::endl; - std::cerr << "Hash found: " << tmpout << std::endl; -#endif - return 0; - } - // now uncompress the string - // - - uint8_t *uncompressed_data = NULL ; - unsigned int uncompressed_data_size = 0 ; - - if(!RsCompress::uncompress_memory_chunk(compressed_data,size,uncompressed_data,uncompressed_data_size)) - { - std::cerr << "FileIndex::loadIndex() Decompression failed! Fileindex can't be read." << std::endl; - free(uncompressed_data); - return 0 ; - } - s = std::string((char *)uncompressed_data,uncompressed_data_size) ; - - std::cerr << " file = " << filename << std::endl; - std::cerr << " uncompressed size = " << uncompressed_data_size << std::endl; - std::cerr << " compressed size = " << size << std::endl; - std::cerr << " hash = " << tmpout << std::endl; - - free(uncompressed_data) ; - } - -#define FIND_NEXT(s,start,end,c) end = s.find(c, start); if (end == std::string::npos) end = s.length(); - - DirEntry *ndir = NULL; - FileEntry *nfile = NULL; - std::list dirlist; - std::string word; - char ch; - - std::string::size_type pos = 0; - while (pos < s.length()) - { - ch = s[pos]; - ++pos; - if (ch == '-') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - - switch(dirlist.size()) - { - /* parse error: out of directory */ - case 0: - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Ran out of dirs"; - std::cerr << std::endl; -#endif - goto error; - } - /* finished parse, last dir is root */ - case 1: - { - RsPeerId pid = root -> id; - FileIndex::unregisterEntry((void*)root) ; - delete root; /* to clean up old entries */ - root = new PersonEntry(pid); - registerEntry((void*)root) ; - - /* shallow copy of all except id */ - ndir = dirlist.back(); - dirlist.pop_back(); /* empty list */ - (*root) = (*ndir); - - /* now cleanup (can't call standard delete) */ - ndir->subdirs.clear(); - ndir->files.clear(); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - ndir = NULL; - - /* must reset parent pointers now */ - std::map::iterator it; - for(it = root->subdirs.begin(); - it != root->subdirs.end(); ++it) - { - (it->second)->parent = root; - } - - break; - } - /* pop stack */ - default: dirlist.pop_back(); ndir = dirlist.back(); - } - continue; - } - - // Ignore comments - else if (ch == '#') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - } - - else { - std::vector tokens; - /* parse line */ - std::string::size_type lineend; - FIND_NEXT(s, pos, lineend, '\n'); - std::string line = s.substr(pos, lineend - pos); - pos = lineend + 1; - - std::string::size_type start = 0; - while (start < line.length()) - { - std::string::size_type end; - FIND_NEXT(line, start, end, FILE_CACHE_SEPARATOR_CHAR); - tokens.push_back(line.substr(start, end - start)); - start = end + 1; - } - - /* create new file and add it to last directory*/ - if (ch == 'f') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " File token count wrong: " << tokens.size(); - std::cerr << std::endl; - for(unsigned int i = 0; i < tokens.size(); ++i) - { - std::cerr << "\tToken[" << i << "]:" << tokens[i]; - std::cerr << std::endl; - } - -#endif - goto error; - } - nfile = new FileEntry(); - registerEntry((void*)nfile) ; - nfile->name = tokens[0]; - nfile->hash = RsFileHash(tokens[1]); - nfile->size = atoll(tokens[2].c_str()); - nfile->modtime = atoi(tokens[3].c_str()); - nfile->pop = atoi(tokens[4].c_str()); - nfile->updtime = atoi(tokens[5].c_str()); - nfile->parent = ndir; - if (ndir) { - nfile->row = ndir->subdirs.size() + ndir->files.size(); - ndir->files[nfile->name] = nfile; - } - - } - /* create new dir and add to stack */ - else if (ch == 'd') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Dir token count wrong: " << tokens.size(); - std::cerr << std::endl; -#endif - goto error; - } - ndir = new DirEntry(); - registerEntry((void*)ndir) ; - ndir->name = tokens[0]; - ndir->path = tokens[1]; - ndir->size = atoi(tokens[2].c_str()); - ndir->modtime = atoi(tokens[3].c_str()); - ndir->pop = atoi(tokens[4].c_str()); - ndir->updtime = atoi(tokens[5].c_str()); - if (!dirlist.empty()) - { - ndir->parent = (dirlist.back()); - ndir->row = dirlist.back()->subdirs.size(); - dirlist.back()->subdirs[ndir->name] = ndir; - } - dirlist.push_back(ndir); - } - } - } - - updateHashIndex() ; - - return 1; - - /* parse error encountered */ -error: -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << std::endl; -#endif - return 0; -} - - -int FileIndex::saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size,const std::set& forbidden_dirs) -{ - std::string filenametmp = filename + ".tmp" ; - std::string s; - - size = 0 ; - - /* print version and header */ - s += "# FileIndex version 0.1\n"; - s += "# Dir: d name, path, parent, size, modtime, pop, updtime;\n"; - s += "# File: f name, hash, size, modtime, pop, updtime;\n"; - s += "#\n"; - - /* begin recusion */ - root->writeDirInfo(s) ; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { -#ifdef FI_DEBUG - std::cout << "writting root directory: name=" << it->second->name << ", path=" << it->second->path << std::endl ; -#endif - if(forbidden_dirs.find(it->second->name) != forbidden_dirs.end()) - { -#ifdef FI_DEBUG - std::cerr << " will be suppressed." << std::endl ; -#endif - } - else - { -#ifdef FI_DEBUG - std::cerr << " will be saved." << std::endl ; -#endif - (it->second)->saveEntry(s); - } - } - - root->writeFileInfo(s) ; // this should never do anything - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - - // now compress the data. -#ifdef FI_DEBUG - std::cerr << "FileIndex::saveIndex(): compressign data." << std::endl; -#endif - - uint8_t *compressed_data = NULL ; - uint32_t compressed_data_size = 0 ; - - if(!RsCompress::compress_memory_chunk((unsigned char *)s.c_str(),s.length(),compressed_data,compressed_data_size)) - { - std::cerr << "(EE) ERROR in file list compression ! file list can't be saved" << std::endl; - free(compressed_data); - return false ; - } - - fileHash = RsDirUtil::sha1sum((unsigned char *)compressed_data,compressed_data_size); - -#ifdef FI_DEBUG - std::cerr << " file = " << filename << std::endl; - std::cerr << " old size = " << s.length() << std::endl; - std::cerr << " new size = " << compressed_data_size << std::endl; - std::cerr << " hash = " << fileHash << std::endl; -#endif - -// /* calculate sha1 hash */ -// SHA_CTX *sha_ctx = new SHA_CTX; -// SHA1_Init(sha_ctx); -// SHA1_Update(sha_ctx, s.c_str(), s.length()); -// SHA1_Final(&sha_buf[0], sha_ctx); -// delete sha_ctx; -// -// for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) -// { -// rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); -// } - - /* finally, save to file */ - - FILE *file = RsDirUtil::rs_fopen(filenametmp.c_str(), "wb"); - if (file == NULL) - { - std::cerr << "FileIndex::saveIndex error opening file for writting: " << filename << ". Giving up." << std::endl; - return 0; - } - uint32_t outwritten ; - - if(compressed_data_size != (outwritten=fwrite(compressed_data,1,compressed_data_size,file))) - { - std::cerr << "FileIndex::saveIndex error. File not entirely written. Only " << outwritten << " bytes wrote out of " << compressed_data_size << " check for disk full, or disk quotas." << std::endl; - fclose(file); - return 0; - } - - fclose(file); - free(compressed_data) ; - - // Use a temp file name so that the file is never half saved. - // - if(!RsDirUtil::renameFile(filenametmp,filename)) - return false ; - - /* get the size out */ - struct stat64 buf; - - if(-1 == stat64(filename.c_str(), &buf)) - { - std::cerr << "Can't determine size of file " << filename << ": errno = " << errno << std::endl ; - return false ; - } - - size=buf.st_size; - - return true; -} - - -std::string FixName(const std::string& _in) -{ - /* replace any , with _ */ - std::string in(_in) ; - for(unsigned int i = 0; i < in.length(); ++i) - { - if (in[i] == FILE_CACHE_SEPARATOR_CHAR) - { - in[i] = '_'; - } - } - return in; -} - -void DirEntry::writeDirInfo(std::string& s) -{ - /* print node info */ - rs_sprintf_append(s, "d%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName(name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - FixName(path).c_str(), FILE_CACHE_SEPARATOR_CHAR, - size, FILE_CACHE_SEPARATOR_CHAR, - modtime, FILE_CACHE_SEPARATOR_CHAR, - pop, FILE_CACHE_SEPARATOR_CHAR, - updtime, FILE_CACHE_SEPARATOR_CHAR); -} - -void DirEntry::writeFileInfo(std::string& s) -{ - /* print file info */ - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - rs_sprintf_append(s, "f%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName((fit->second)->name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->hash.toStdString().c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->size, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->modtime, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->pop, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->updtime, FILE_CACHE_SEPARATOR_CHAR); - } -} - -/* recusive function for traversing the dir tree in preorder */ -int DirEntry::saveEntry(std::string &s) -{ - writeDirInfo(s) ; - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->saveEntry(s); - } - - writeFileInfo(s) ; - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - return 1; -} - - -int FileIndex::searchHash(const RsFileHash& hash, std::list &results) const -{ -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << std::endl; -#endif - - std::map::const_iterator it = _file_hashes.find(hash) ; - - if(it!=_file_hashes.end() && isValid((void*)it->second)) - results.push_back(it->second) ; - -#ifdef OLD_CODE_PLZ_REMOVE - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - /* add subdirs to stack */ - std::map::iterator it; - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - std::map::iterator fit; - /* search in current dir */ - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - if (hash == (fit->second)->hash) - { - results.push_back(fit->second); -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << " found: " << fit->second->name; - std::cerr << std::endl; -#endif - } - } - } -#endif - - return 0; -} - - -int FileIndex::searchTerms(const std::list& terms, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for (iter = terms.begin(); iter != terms.end(); ++iter) { - std::string::const_iterator it2; - const std::string &str1 = ndir->name; - const std::string &str2 = *iter; - it2 = std::search(str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC()); - if (it2 != str1.end()) { - /* first search to see if its parent is in the results list */ - bool addDir = true; - for (std::list::iterator rit(results.begin()); rit != results.end() && addDir; ++rit) { - DirEntry *de = dynamic_cast(*rit); - if (de && (de == root)) - continue; - if (de && (de == ndir->parent)) - addDir = false; - } - if (addDir) { - results.push_back((FileEntry *) ndir); - break; - } - } - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /* cycle through terms */ - for(iter = terms.begin(); iter != terms.end(); ++iter) - { - /* always ignore case */ - std::string::const_iterator it2 ; - const std::string &str1 = fit->second->name; - const std::string &str2 = (*iter); - - it2 = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end(), CompareCharIC() ); - if (it2 != str1.end()) - { - results.push_back(fit->second); - break; - } - /* original case specific term search ****** - if (fit->second->name.find(*iter) != std::string::npos) - { - results.push_back(fit->second); - break; - } - ************/ - } - } - } //while - - return 0; -} - -int FileIndex::searchBoolExp(Expression * exp, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /*Evaluate the boolean expression and add it to the results if its true*/ - bool ret = exp->eval(fit->second); - if (ret == true){ - results.push_back(fit->second); - } - } - } //while - - return 0; -} - -uint32_t FileIndex::getType(void *ref) -{ - if(ref == NULL) - return DIR_TYPE_ROOT ; - - if(!isValid(ref)) - return DIR_TYPE_ROOT ; - - return static_cast(ref)->type() ; -} - -bool FileIndex::extractData(const std::string& fpath,DirDetails& details) const -{ - void *ref = findRef(fpath) ; - - if(ref == NULL) - return false ; - - return extractData(ref,details) ; -} - -void *FileIndex::findRef(const std::string& fpath) const -{ - DirEntry *parent = root->findDirectory(fpath); - - std::cerr << "findRef() Called on " << fpath << std::endl; - - if (!parent) - { -//#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -//#endif - return NULL; - } - std::cerr << "Found parent directory: " << std::endl; - std::cerr << " parent.name = " << parent->name << std::endl; - std::cerr << " parent.path = " << parent->path << std::endl; - - if(parent->path == fpath) // directory! - { - std::cerr << " fpath is a directory. Returning parent!" << std::endl; - return parent ; - } - else - { - std::cerr << " fpath is a file. Looking into parent directory..." << std::endl; - /* search in current dir */ - for(std::map::iterator fit = parent->files.begin(); fit != parent->files.end(); ++fit) - { - std::cerr << " trying " << parent->path + "/" + fit->second->name << std::endl; - if(parent->path + "/" + fit->second->name == fpath) - { - std::cerr << " found !" << std::endl; - return fit->second ; - } - } - - std::cerr << " (EE) not found !" << std::endl; - return NULL ; - } -} - -bool FileIndex::extractData(void *ref,DirDetails& details) -{ - if(!isValid(ref)) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() asked for an invalid pointer " << (void*)ref << std::endl; -#endif - return false ; - } - - FileEntry *file = static_cast(ref); - DirEntry *dir = (file->hash.isNull())?static_cast(file):NULL ; // This is a hack to avoid doing a dynamic_cast - - details.children.clear() ; - time_t now = time(NULL) ; - - if (dir!=NULL) /* has children --- fill */ - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif - /* extract all the entries */ - for(std::map::const_iterator dit(dir->subdirs.begin()); dit != dir->subdirs.end(); ++dit) - { - DirEntry *dirEntry = dit->second; - - DirStub stub; - stub.type = DIR_TYPE_DIR; - stub.name = dirEntry -> name; - stub.ref = dirEntry; - - details.children.push_back(stub); - } - - for(std::map::const_iterator fit(dir->files.begin()); fit != dir->files.end(); ++fit) - { - FileEntry *fileEntry = fit->second; - - DirStub stub; - stub.type = DIR_TYPE_FILE; - stub.name = fileEntry -> name; - stub.ref = fileEntry; - - details.children.push_back(stub); - } - - if(dir->parent == NULL) - details.type = DIR_TYPE_PERSON ; - else - details.type = DIR_TYPE_DIR; - details.hash.clear() ; - details.count = dir->subdirs.size() + dir->files.size(); - details.min_age = now - dir->most_recent_time ; - } - else - { -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif - details.type = DIR_TYPE_FILE; - details.count = file->size; - details.min_age = now - file->modtime ; - } - -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; -#endif - details.ref = file; - details.hash = file->hash; - details.age = now - file->modtime; - details.flags.clear() ; - - /* find parent pointer, and row */ - details.parent = file->parent ; - - details.prow = (file->parent==NULL)?0:file->parent->row ; - - if(details.type == DIR_TYPE_DIR) - { - details.name = file->name; - details.path = dir->path; - } - else - { - details.name = file->name; - details.path = (file->parent==NULL)?"":file->parent->path; - } - - /* find peer id */ - FileEntry *f ; - for(f=file;f->parent!=NULL;f=f->parent) ; - - details.id = static_cast(f)->id; // The topmost parent is necessarily a personEntrY, so we can avoid a dynamic_cast. - -#ifdef FI_DEBUG - assert(details.parent != details.ref) ; - std::cout << "details: ref=" << (void*)ref << ", prow=" << details.prow << ", parent=" << (void*)details.parent << ", children=" ; - for(std::list::iterator it(details.children.begin());it!=details.children.end();++it) - std::cout << " " << (void*)it->ref ; - std::cout << std::endl ; -#endif - - return true; -} - diff --git a/libretroshare/src/dbase/findex.h b/libretroshare/src/dbase/findex.h deleted file mode 100644 index 87ad257d9..000000000 --- a/libretroshare/src/dbase/findex.h +++ /dev/null @@ -1,278 +0,0 @@ -/* - * RetroShare FileCache Module: findex.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_H -#define FILE_INDEX_H - -#include -#include -#include -#if __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include "retroshare/rstypes.h" - -/****************************************************************************************** - * The Key Data Types for the File Index: - - FileEntry : Information about a single file. - DirEntry : Information about a directory and its children - PersonEntry : Information the root of a FileIndex. - - FileIndex : A Collection of root directories, with a set of functions to manipulate them. - In terms of retroshare, There will be a single 'Local' FileIndex used to store - the shared files, and a set of 'Remote' FileIndices which are used to store - the available files of all the peers. - -******************************************************************************************/ -/****************************************************************************************** - STILL TODO: - - (1) Load/Store a FileIndex to file... - int FileIndex::loadIndex(FILE *input); - int FileIndex::saveIndex(FILE *input); - - This can be done in a recursive manner, or handled completely within FileIndex. - - (2) Search Functions for Partial File Names and Hashes. - - int FileIndex::searchHash(std::string hash, std::list &results); - int FileIndex::searchTerms(std::list terms, std::list &results); - - This is probably best done in a recursive manner. - - The search could also be extended to handle complex Boolean searches such as : - match (size > 100K) && (name contains 'Blue') .... if anyone is interested. - But this can get quite complicated, and can be left to a later date. - -******************************************************************************************/ - - -/****************************************************************************************** - * FileEntry - *****************************************************************************************/ - -#include - -class DirEntry; - -class FileEntry: public RsMemoryManagement::SmallObject -{ - public: - FileEntry() - : size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0) - { return; } - - virtual ~FileEntry() { return; } - virtual uint32_t type() const { return DIR_TYPE_FILE ; } - -virtual int print(std::string &out); - - /* Data */ - std::string name; - RsFileHash hash; - uint64_t size; /* file size */ - time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */ - int pop; /* popularity rating */ - - time_t updtime; /* last updated */ - - /* References for easy manipulation */ - DirEntry *parent; - int row; - std::list parent_groups ; -}; - -/****************************************************************************************** - * DirEntry - *****************************************************************************************/ - -class DirEntry: public FileEntry -{ - public: - - DirEntry() : most_recent_time(0) {} - /* cleanup */ -virtual ~DirEntry(); - - /* update local entries */ -DirEntry * updateDir(const FileEntry& fe, time_t updtime); -FileEntry * updateFile(const FileEntry& fe, time_t updtime); - - - virtual uint32_t type() const { return DIR_TYPE_DIR ; } -int checkParentPointers(); -int updateChildRows(); - - /* remove local entries */ -int removeFile(const std::string& name); -int removeDir(const std::string& name); -int removeOldDir(const std::string& name, time_t old); /* checks ts first */ - - /* recursive cleanup */ -int removeOldEntries(time_t old, bool recursive); - - /* recursive searches */ -DirEntry * findOldDirectory(time_t old); -DirEntry * findDirectory(const std::string& path); - - /* recursive update directory mod/pop values */ -int updateDirectories(const std::string& path, int pop, int modtime); - - /* output */ -int print(std::string &out); - -int saveEntry(std::string &out); -void writeDirInfo(std::string&); -void writeFileInfo(std::string&); - - /* Data */ - std::string path; /* full path (includes name) */ - std::map subdirs; - std::map files; - - time_t most_recent_time; /* last updated */ - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * PersonEntry - *****************************************************************************************/ - -class PersonEntry: public DirEntry -{ - public: - /* cleanup */ - PersonEntry(const RsPeerId& pid) : id(pid) { return; } -virtual ~PersonEntry() { return; } - -DirEntry &operator=(DirEntry &src) -{ - DirEntry *pdest = this; - (*pdest) = src; - return *this; -} - virtual uint32_t type() const { return DIR_TYPE_PERSON ; } - - /* Data */ - RsPeerId id; - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * FileIndex - *****************************************************************************************/ - -class Expression; - -class FileIndex -{ - public: - FileIndex(const RsPeerId& pid); - ~FileIndex(); - - /* control root entries */ - int setRootDirectories(const std::list &inlist, time_t utime); - int getRootDirectories(std::list &outlist); - - /* update (index building) */ - DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime); - FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime); - - DirEntry * findOldDirectory(time_t old); /* finds directories older than old */ - int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old); - - int cleanOldEntries(time_t old); /* removes entries older than old */ - - /* debug */ - int printFileIndex(std::string &out); - int printFileIndex(std::ostream &out); - - /* load/save to file */ - int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size); - int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set& forbidden_roots); - - /* search through this index */ - int searchTerms(const std::list& terms, std::list &results) const; - int searchHash(const RsFileHash& hash, std::list &results) const; - int searchBoolExp(Expression * exp, std::list &results) const; - - /// Recursively compute the maximum modification time of children. - /// Used to easily retrieve mose recent files. - // - void updateMaxModTime() ; - void RecursUpdateMaxModTime(DirEntry *) ; - - PersonEntry *root; - -#ifdef __MACH__ - static std::unordered_set _pointers ; -#else - static std::tr1::unordered_set _pointers ; -#endif - static void registerEntry(void*p) ; - static void unregisterEntry(void*p) ; - static bool isValid(void*p) ; - - /// Fills up details from the data contained in ref. - // - static bool extractData(void *ref,DirDetails& details) ; - static uint32_t getType(void *ref) ; - - void *findRef(const std::string& path) const ; - bool extractData(const std::string& path,DirDetails& details) const ; - - void updateHashIndex() ; - void recursUpdateHashIndex(DirEntry *) ; - - std::map _file_hashes ; -}; - - -#endif - diff --git a/libretroshare/src/dbase/fistore.cc b/libretroshare/src/dbase/fistore.cc deleted file mode 100644 index 00d70a214..000000000 --- a/libretroshare/src/dbase/fistore.cc +++ /dev/null @@ -1,459 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include - -#include "rsserver/p3face.h" -#include "dbase/fistore.h" -#include "retroshare/rsexpr.h" -#include "retroshare/rsfiles.h" -#include "serialiser/rsserviceids.h" -#include "pqi/p3peermgr.h" -#include "pqi/p3notify.h" - -FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, - p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir) - :CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir), - localId(ownid), localindex(NULL), mPeerMgr(cnmgr) -{ - return; -} - -FileIndexStore::~FileIndexStore() -{ - /* clean up the Index */ - return; -} - -/*** - * #define FIS_DEBUG2 1 - * #define FIS_DEBUG 1 - **/ - - /* actual load, once data available */ -int FileIndexStore::loadCache(const RsCacheData &data) -{ - -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl; - std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl; - std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl; - std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl; -#endif - - /* do Callback */ - AboutToModify(); - - /* lock it up */ - lockData(); - - FileIndex *fiold = NULL; - bool local = (data.pid == localId); - - std::map::iterator it; - /* remove old cache */ - if (local) - { - fiold = localindex; - localindex = NULL; - } - else if (indices.end() != (it = indices.find(data.pid))) - { - fiold = it->second; - indices.erase(it); - //delete fi; - } - - if(mPeerMgr->isFriend(data.pid)) - { - // We discard file lists from non friends. This is the place to remove file lists of deleted friends - // from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed - // at next restart. - // - - /* load Cache */ - FileIndex *finew = new FileIndex(data.pid); - - if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size)) - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl; -#endif - /* This is not the place to set the peername. - * It is a hack, which makes it programmatically impossible - * to get the file data out.... - * - * peername should not be used in dbase. - */ - finew->root->name = data.pid.toStdString(); - - if (local) - { - localindex = finew; - } - else - { - indices[data.pid] = finew; - } - delete fiold; - - /* store in tale */ - locked_storeCacheEntry(data); - } - else - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl; -#endif - /* reinstall the old one! */ - delete finew; - if (fiold) - { - if (local) - { - localindex = fiold; - } - else - { - indices[data.pid] = fiold; - } - } - } - } -#ifdef FIS_DEBUG - else - std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ; -#endif - - /* need to correct indices(row) for the roots of the FileIndex */ - int i = 0; - for(it = indices.begin(); it != indices.end(); ++it) - { - (it->second)->root->row = i++; - it->second->FileIndex::updateMaxModTime() ; - } - if (localindex) - { - localindex->root->row = 0; - localindex->updateMaxModTime() ; - } - - unlockData(); - - ModCompleted(); - bool ret = false; - return ret; -} - - - /* Search Interface - For Directory Access */ -int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const -{ - lockData(); - - std::map::const_iterator it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - found = it->second->extractData(path,details) ; - - unlockData(); - - return found ; - -#ifdef OLD_STUFF_TO_REMOVE - /* lock it up */ - lockData(); - - std::map::const_iterator it; - it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - { - //DirEntry *fdir = (it->second).lookupDirectory(path); - /* translate it - */ - bool b = FileIndex::extractData((it->second)->root,details) ; - - found = found && b ; - } - else - found = false; - - unlockData(); - return found; -#endif -} - -int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - std::map::const_iterator pit; - - lockData(); - -// checked by FileIndex::extractData -// if(ref != NULL && !FileIndex::isValid(ref)) -// { -// unlockData() ; -// return false ; -// } - - /* so cast *ref to a DirEntry */ - /* root case */ - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl; - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - (pit->second)->root->checkParentPointers(); - } -#endif - - if (ref == NULL) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - - /* get remote root entries */ - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - /* - */ - FileIndex *fileIndex = pit->second; - - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fileIndex->root->name; - stub.ref = fileIndex->root; - - details.children.push_back(stub); - } - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = ""; - details.hash.clear() ; - details.path = ""; - details.count = indices.size(); - details.age = 0; - details.flags.clear() ; - details.min_age = 0; - - unlockData(); - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - unlockData(); - return b; -} -uint32_t FileIndexStore::getType(void *ref) const -{ - lockData() ; - uint32_t b = FileIndex::getType(ref) ; - unlockData(); - - return b; -} - -int FileIndexStore::SearchHash(const RsFileHash& hash, std::list &results) const -{ - lockData(); - results.clear() ; - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - time_t now = time(NULL); - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Searching: Peer "; - std::cerr << pit->first << std::endl; -#endif - firesults.clear(); - - (pit->second)->searchHash(hash, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - FileDetail fd; - fd.id = pit->first; - fd.name = (*rit)->name; - fd.hash = (*rit)->hash; - fd.path = ""; /* TODO */ - fd.size = (*rit)->size; - fd.age = now - (*rit)->modtime; - fd.rank = (*rit)->pop; - - results.push_back(fd); - } - - } - - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Found " << results.size(); - std::cerr << " Results from " << indices.size() << " Peers" << std::endl; -#endif - - unlockData(); - return results.size(); -} - - -int FileIndexStore::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - results.clear() ; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchKeywords()" << std::endl; -#endif - if(flags & RS_FILE_HINTS_REMOTE) - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - results.push_back(dd); - } - } - - if(flags & RS_FILE_HINTS_LOCAL) - if (localindex) - { - firesults.clear(); - - localindex->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - dd.id.clear() ; - results.push_back(dd); - } - - } - - unlockData(); - return results.size(); -} - - -int FileIndexStore::searchBoolExp(Expression * exp, std::list &results) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::searchBoolExp()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - results.push_back(dd); - } - - } - - /* finally search local files */ - if (localindex) - { - firesults.clear(); - - localindex->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - dd.id.clear() ; - results.push_back(dd); - } - - } - - - unlockData(); - return results.size(); -} - -int FileIndexStore::AboutToModify() -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - -int FileIndexStore::ModCompleted() -{ - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - diff --git a/libretroshare/src/dbase/fistore.h b/libretroshare/src/dbase/fistore.h deleted file mode 100644 index e175adec8..000000000 --- a/libretroshare/src/dbase/fistore.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_FILE_INDEX_STORE_H -#define MRK_FILE_INDEX_STORE_H - - -/********** - * Stores the FileCaches of the Peers - * must implement 'loadCache' to - * - * This class is also accessed by the GUI.... - * and the FileTransfer class. - * - */ - -class p3PeerMgr ; - -#include "dbase/findex.h" -#include "dbase/cachestrapper.h" -#include "retroshare/rsiface.h" - -class FileStoreResult -{ - public: - std::string id; - std::string path; - std::string hash; - std::string name; -}; - -class NotifyCallback -{ - public: - NotifyCallback() { return; } -virtual ~NotifyCallback() { return; } -virtual void AboutToModify() { return; } -virtual void ModCompleted() { return; } -}; - - -class Expression; - -class FileIndexStore: public CacheStore -{ - public: - - FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir); -virtual ~FileIndexStore(); - - /* virtual functions overloaded by cache implementor */ -virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* Search Interface - For FileTransfer Lookup */ - int SearchHash(const RsFileHash &hash, std::list &results) const; - - /* Search Interface - For Search Interface */ - int SearchKeywords(std::list terms, std::list &results,FileSearchFlags flags) const; - - /* Search Interface - for Adv Search Interface */ - int searchBoolExp(Expression * exp, std::list &results) const; - - - /* Search Interface - For Directory Access */ - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const; - int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const; - uint32_t getType(void *ref) const ; - - private: - int AboutToModify(); - int ModCompleted(); - - std::map indices; - - RsPeerId localId; - FileIndex *localindex; - - p3PeerMgr *mPeerMgr ; -}; - - -#endif From 1be599008afe275baa9f1cdefedfd92badec1e71 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 21 Sep 2016 22:31:39 +0200 Subject: [PATCH 83/92] disabled dbase from unittests.pro, which should be cnverted into proper tests for file_sharing. --- tests/unittests/unittests.pro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/unittests.pro b/tests/unittests/unittests.pro index 2409b8845..d0485ca92 100644 --- a/tests/unittests/unittests.pro +++ b/tests/unittests/unittests.pro @@ -353,9 +353,9 @@ SOURCES += libretroshare/gxs/data_service/rsdataservice_test.cc \ ################################ dbase ##################################### -SOURCES += libretroshare/dbase/fisavetest.cc \ - libretroshare/dbase/fitest2.cc \ - libretroshare/dbase/searchtest.cc \ +#SOURCES += libretroshare/dbase/fisavetest.cc \ +# libretroshare/dbase/fitest2.cc \ +# libretroshare/dbase/searchtest.cc \ # libretroshare/dbase/ficachetest.cc \ # libretroshare/dbase/fimontest.cc \ From 7f99bc2b70c9abca4345a95fe8e720acd5cd8c41 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 22 Sep 2016 10:26:05 +0200 Subject: [PATCH 84/92] fixed compilation after removal of dbase/ --- libretroshare/src/plugins/rscacheservice.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libretroshare/src/plugins/rscacheservice.h b/libretroshare/src/plugins/rscacheservice.h index 6e928da61..82f43e0de 100644 --- a/libretroshare/src/plugins/rscacheservice.h +++ b/libretroshare/src/plugins/rscacheservice.h @@ -1,6 +1,5 @@ #pragma once -#include #include "plugins/pluginmanager.h" // This code needs to be re-written to work with GXS. For now it is obsolete. @@ -9,7 +8,7 @@ // The following class abstracts the construction of a cache service. The user only has to // supply RS with a type ID. If the ID is already in use, RS will complain. // -class RsCacheService: public CacheSource, public CacheStore, public p3Config +class RsCacheService: /* public CacheSource, public CacheStore, */ public p3Config { public: RsCacheService(uint16_t type,uint32_t tick_delay_in_seconds, RsPluginHandler* pgHandler) ; From 0cc4ebd89c34c66442185eefc918bf0d09ffb2fc Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 22 Sep 2016 21:47:58 +0200 Subject: [PATCH 85/92] added comments in the file lists code. Removed unused functions. Added a few missing mutexes. --- .../src/file_sharing/dir_hierarchy.cc | 33 +++++++++++++- .../src/file_sharing/directory_storage.cc | 15 ------- .../src/file_sharing/directory_storage.h | 43 +++++++++++++------ libretroshare/src/file_sharing/hash_cache.h | 19 ++++++-- libretroshare/src/file_sharing/p3filelists.cc | 37 ++++++++++++---- libretroshare/src/file_sharing/p3filelists.h | 4 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 + 7 files changed, 108 insertions(+), 45 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index ad1004715..61785753f 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -142,14 +142,18 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; +#endif if( !removeDirectory(d.subdirs[i])) i++ ; } else { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; +#endif should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; ++i; @@ -157,7 +161,9 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; +#endif DirEntry *de = new DirEntry(it->first) ; @@ -258,7 +264,9 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En if(it == subfiles.end()) // file does not exist anymore => delete { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; +#endif delete mNodes[d.subfiles[i]] ; mNodes[d.subfiles[i]] = NULL ; @@ -281,7 +289,9 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; +#endif d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); @@ -297,8 +307,9 @@ bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; } - +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; +#endif RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; mFileHashes[hash] = file_index ; @@ -317,7 +328,9 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex FileEntry& fe(*static_cast(mNodes[file_index])) ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; +#endif fe.file_hash = hash; fe.file_size = size; @@ -358,7 +371,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl; +#endif d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; @@ -375,7 +390,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::iterator it = existing_subdirs.find(subdirs_hash[i]) ; DirectoryStorage::EntryIndex dir_index = 0; @@ -384,7 +401,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI { dir_index = it->second ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " already exists, at index " << dir_index << std::endl; +#endif existing_subdirs.erase(it) ; } @@ -401,7 +420,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI mDirHashes[subdirs_hash[i]] = dir_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " created, at new index " << dir_index << std::endl; +#endif } d.subdirs.push_back(dir_index) ; @@ -411,7 +432,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI for(std::map::const_iterator it = existing_subdirs.begin();it!=existing_subdirs.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; +#endif if(!checkIndex(it->second,FileStorageNode::TYPE_DIR)) { @@ -436,13 +459,17 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI const FileEntry& f(subfiles_array[i]) ; DirectoryStorage::EntryIndex file_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " subfile name = " << subfiles_array[i].file_name << ": " ; +#endif if(it != existing_subfiles.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_FILE) { file_index = it->second ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " already exists, at index " << file_index << std::endl; +#endif if(!updateFile(file_index,f.file_hash,f.file_name,f.file_size,f.file_modtime)) std::cerr << "(EE) Cannot update file with index " << it->second <<" and hash " << f.file_hash << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; @@ -456,7 +483,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI mNodes[file_index] = new FileEntry(f.file_name,f.file_size,f.file_modtime,f.file_hash) ; mFileHashes[f.file_hash] = file_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " created, at new index " << file_index << std::endl; +#endif } d.subfiles.push_back(file_index) ; @@ -465,7 +494,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI for(std::map::const_iterator it = existing_subfiles.begin();it!=existing_subfiles.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; +#endif if(!checkIndex(it->second,FileStorageNode::TYPE_FILE)) { diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 45cb3ccca..9a8b76c9e 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -173,21 +173,6 @@ int DirectoryStorage::searchHash(const RsFileHash& hash, std::list & return mFileHierarchy->searchHash(hash,results); } -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; - -void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) -{ - entry_tag = data[offset++] ; -} -void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size) -{ -} - void DirectoryStorage::load(const std::string& local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index b76d73ebd..2d8119445 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -49,23 +49,28 @@ class DirectoryStorage void save() const ; + // These functions are to be used by file transfer and file search. + virtual int searchTerms(const std::list& terms, std::list &results) const ; virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; - bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; - bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; + // gets/sets the various time stamps: + // + bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; // last modification time, computed recursively over all subfiles and directories + bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; // last modification time for that index only + bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; // last time the entry was updated. This is only used on the RemoteDirectoryStorage side. bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ; bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ; bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ; - uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. + uint32_t getEntryType(const EntryIndex& indx) ; // WARNING: returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. - // It gives access to sub-files and sub-directories below. + // It gives access to sub-files and sub-directories below. When using it, the client should make sure + // that the DirectoryStorage is properly locked, since the iterator cannot lock it. // class DirIterator { @@ -120,20 +125,28 @@ class DirectoryStorage time_t modtime; }; - EntryIndex root() const ; // returns the index of the root directory entry. - const RsPeerId& peerId() const { return mPeerId ; } - int parentRow(EntryIndex e) const ; - bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; + EntryIndex root() const ; // returns the index of the root directory entry. This is generally 0. + const RsPeerId& peerId() const { return mPeerId ; } // peer ID of who owns that file list. + int parentRow(EntryIndex e) const ; // position of the current node, in the array of children at its parent node. Used by GUI for display. + bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; // returns the index of the children node at position "row" in the children nodes. Used by GUI for display. + // Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with + // just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache. + // bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; + // Updates relevant information for the file at the given index. + bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); - bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; - bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; + // Returns the hash of the directory at the given index and reverse. This hash is set as random the first time it is used (when updating directories). It will be + // used by the sync system to designate the directory without referring to index (index could be used to figure out the existance of hidden directories) + + bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; // constant cost + bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; // log cost. void print(); void cleanup(); @@ -144,9 +157,6 @@ class DirectoryStorage private: - void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; - void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; - // debug void locked_check(); @@ -195,6 +205,11 @@ public: LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {} virtual ~LocalDirectoryStorage() {} + /*! + * \brief [gs]etSharedDirectoryList + * Gets/sets the list of shared directories. Each directory is supplied with a virtual name (the name the friends will see), and sharing flags/groups. + * \param lst + */ void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 0f92f09c1..1afa756ca 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -30,6 +30,11 @@ #include "util/rsthreads.h" #include "retroshare/rsfiles.h" +/*! + * \brief The HashStorageClient class + * Used by clients of the hash cache for receiving hash results when done. This is asynchrone of course since hashing + * might be quite costly. + */ class HashStorageClient { public: @@ -59,7 +64,7 @@ public: * \param mod_time Actual file modification time * \param known_hash Returned hash for the file. * \param c Hash cache client to which the hash should be sent once calculated - * \param client_param Param to be passed to the client callback + * \param client_param Param to be passed to the client callback. Useful if the client needs a file ID. * * \return true if the supplied hash info is up to date. */ @@ -75,9 +80,9 @@ public: } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } // duration for which the hash is kept even if the file is not shared anymore uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } - void clear() { mFiles.clear(); mChanged=true; } + void clear() { mFiles.clear(); mChanged=true; } // drop all known hashes. Not something to do, except if you want to rehash the entire database bool empty() const { return mFiles.empty() ; } // Functions called by the thread @@ -86,8 +91,14 @@ public: friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ; private: + /*! + * \brief clean + * This function is responsible for removing old hashes, etc + */ void clean() ; + // loading/saving the entire hash database to a file + void locked_save() ; void locked_load() ; @@ -98,7 +109,7 @@ private: uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries std::map mFiles ; // stored as (full_path, hash_info) - std::string mFilePath ; + std::string mFilePath ; // file where the hash database is stored bool mChanged ; struct FileHashJob diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0e734e56a..409dd4637 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -223,8 +223,10 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; -#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up. - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + // This is a hash to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up, + // so the first notify is ignored, and no other notify will happen next. + + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } return 0; @@ -614,7 +616,9 @@ void p3FileDatabase::requestDirUpdate(void *ref) P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; #endif - if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) + RS_STACK_MUTEX(mFLSMtx) ; + + if(locked_generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) { #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Succeed." << std::endl; @@ -638,7 +642,7 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe return true ; } - else if(row < mRemoteDirectories.size()) + else if((uint32_t)row < mRemoteDirectories.size()) { convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; @@ -752,7 +756,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - bool res = storage->extractData(e,d); + if(!storage->extractData(e,d)) + { + P3FILELISTS_ERROR() << "(EE) request on index " << e << ", for directory ID=" << storage->peerId() << " failed. This should not happen." << std::endl; + return false ; + } // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -824,6 +832,7 @@ void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the di } bool p3FileDatabase::inDirectoryCheck() { + RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->inDirectoryCheck(); } void p3FileDatabase::setWatchEnabled(bool b) @@ -975,8 +984,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. // // Remote dirs store -// 1 - recursive modif time of the files/dir in the friend's time -// 2 - update TS in the local time +// 1 - recursive max modif time of the files/dir in the friend's time +// 2 - modif time of the files/dir in the friend's time +// 3 - update TS in the local time // // The update algorithm is the following: // @@ -1275,20 +1285,29 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { + RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS std::cerr << " removing directory with index " << entry_index << " because it does not exist." << std::endl; +#endif mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { + RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS std::cerr << " Directory is up to date. Setting local TS." << std::endl; +#endif mRemoteDirectories[fi]->setDirectoryUpdateTime(entry_index,time(NULL)) ; } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { +#ifdef DEBUG_P3FILELISTS std::cerr << " Item contains directory data. Deserialising/Updating." << std::endl; +#endif + RS_STACK_MUTEX(mFLSMtx) ; if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed @@ -1328,7 +1347,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. - if(generateAndSendSyncRequest(rds,e)) + if(locked_generateAndSendSyncRequest(rds,e)) { #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; @@ -1363,7 +1382,7 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId return r ^ random_bias; } -bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) +bool p3FileDatabase::locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) { RsFileHash entry_hash ; time_t now = time(NULL) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 08afd0e88..67834bfa7 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -117,7 +117,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; - // proxy method used by the web UI. + // proxy method used by the web UI. Dont't delete! int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; // set/update shared directories @@ -173,7 +173,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub * \param e Entry index to update * \return true if the request is correctly sent. */ - bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); + bool locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index f48394cca..036dd96e2 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -522,7 +522,9 @@ QModelIndexList SharedFilesDialog::getSelected() void RemoteSharedFilesDialog::expanded(const QModelIndex& indx) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Expanding at " << indx.row() << " and " << indx.column() << " ref=" << indx.internalPointer() << ", pointer at 1: " << proxyModel->mapToSource(indx).internalPointer() << std::endl; +#endif model->updateRef(proxyModel->mapToSource(indx)) ; } From 9f76b7601a1d9d478bc762bd88facaa632fdd693 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 23 Sep 2016 19:01:14 +0200 Subject: [PATCH 86/92] forced closedir in order to avoid leak of file descriptor in FolderIterator --- libretroshare/src/util/folderiterator.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 9e8ba66d3..35a661882 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -150,9 +150,6 @@ uint8_t FolderIterator::file_type() { return mType ; } bool FolderIterator::closedir() { - if(!validity) - return false; - validity = false; #ifdef WINDOWS_SYS From ca281e4bb871ba981742e573ce41bf2edb51ad5e Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 18:32:36 +0200 Subject: [PATCH 87/92] Fixed error with 32 bit GCC Both uint32_t and size_t are resolved to unsigned int, it makes 2 lines conflicted. This happens in filelist_io.cc, lines: 31-32, 36-37, 41-42 --- libretroshare/src/file_sharing/filelist_io.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 90297ab0e..33a1bc5a1 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -29,17 +29,17 @@ #include "filelist_io.h" template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; } -template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const size_t & val) { return setRawUInt64(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast(buff),size,&offset,&val) ; } -template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,size_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint64_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast(buff),size,&offset,val) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast(buff),size,offset) ; } template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; } -template<> uint32_t FileListIO::serial_size(const size_t & ) { return 8 ; } +template<> uint32_t FileListIO::serial_size(const uint64_t & ) { return 8 ; } template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; } template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); } From 529ab64ae383209c3ca56626821e1d9f9aa1926d Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 18:43:48 +0200 Subject: [PATCH 88/92] Fixed missing time.h includes --- libretroshare/src/file_sharing/dir_hierarchy.cc | 1 + libretroshare/src/file_sharing/directory_storage.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 61785753f..023206d29 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -23,6 +23,7 @@ * */ #include +#include #include "util/rsdir.h" #include "util/rsprint.h" #include "retroshare/rsexpr.h" diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 9a8b76c9e..0f0c4e2cb 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -23,6 +23,7 @@ * */ #include +#include #include "serialiser/rstlvbinary.h" #include "retroshare/rspeers.h" #include "util/rsdir.h" From 70c8e14d164b69fb181069a913dd300e32845436 Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 19:00:07 +0200 Subject: [PATCH 89/92] Fix folderiterator.cc:23: error: cannot convert '_stat64*' to '_stati64*' for argument '2' to 'int _wstati64(const wchar_t*, _stati64*)' --- libretroshare/src/util/folderiterator.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 35a661882..bf9bfd68b 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -4,6 +4,10 @@ #include #include +#ifdef WINDOWS_SYS +#include "util/rswin.h" +#endif + #include "folderiterator.h" #include "rsstring.h" From 71a5bffed887f2bf3422c7fc4002a30882ab9745 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 23 Sep 2016 20:07:57 +0200 Subject: [PATCH 90/92] added small hack against re-hashing due to stat64 depending on local time on windows --- libretroshare/src/file_sharing/hash_cache.cc | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 3f96abec1..8311aa77a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -191,9 +191,29 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t time_t now = time(NULL) ; std::map::iterator it = mFiles.find(full_path) ; - if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) + // On windows we compare the time up to +/- 3600 seconds. This avoids re-hashing files in case of daylight saving change. + // + // See: + // https://support.microsoft.com/en-us/kb/190315 + // + if(it != mFiles.end() +#ifdef WINDOWS_SYS + && ( (uint64_t)mod_time == it->second.modf_stamp || (uint64_t)mod_time+3600 == it->second.modf_stamp ||(uint64_t)mod_time == it->second.modf_stamp+3600) +#else + && (uint64_t)mod_time == it->second.modf_stamp +#endif + && size == it->second.size) { it->second.time_stamp = now ; + +#ifdef WINDOWS_SYS + if(it->second.time_stamp != (uint64_t)mod_time) + { + std::cerr << "(WW) detected a 1 hour shift in file modification time. This normally happens to many files at once, when daylight saving time shifts (file=\"" << full_path << "\")." << std::endl; + it->second.time_stamp = (uint64_t)mod_time; + } +#endif + known_hash = it->second.hash; #ifdef HASHSTORAGE_DEBUG std::cerr << "Found in cache." << std::endl ; From 8500b5e9f562ee4ebce0e396d7156004b3214c2d Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 21:56:41 +0200 Subject: [PATCH 91/92] Fixed inconsistency in integer sizes that caused segmentation fault while hashing files also broken message on the GUI --- libretroshare/src/file_sharing/hash_cache.cc | 2 +- libretroshare/src/file_sharing/hash_cache.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 3f96abec1..2ae365885 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -146,7 +146,7 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); std::string tmpout; - rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 1afa756ca..3860aeabc 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -129,7 +129,7 @@ private: RsMutex mHashMtx ; bool mRunning; - uint32_t mHashCounter; + uint64_t mHashCounter; uint32_t mInactivitySleepTime ; uint64_t mTotalSizeToHash ; uint64_t mTotalHashedSize ; From e03bd6cbacca61e71deb58cb50411a956edea2aa Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 24 Sep 2016 13:56:02 +0200 Subject: [PATCH 92/92] fixed bug causing call to closedir on invalid handle --- libretroshare/src/util/folderiterator.cc | 9 +++++++-- libretroshare/src/util/folderiterator.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index bf9bfd68b..415aeb085 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -44,10 +44,10 @@ FolderIterator::FolderIterator(const std::string& folderName) utf16Name += L"/*.*"; handle = FindFirstFileW(utf16Name.c_str(), &fileInfo); - validity = handle != INVALID_HANDLE_VALUE; + is_open = validity = handle != INVALID_HANDLE_VALUE; #else handle = opendir(folderName.c_str()); - validity = handle != NULL; + is_open = validity = handle != NULL; next(); #endif } @@ -156,6 +156,11 @@ bool FolderIterator::closedir() { validity = false; + if(!is_open) + return true ; + + is_open = false ; + #ifdef WINDOWS_SYS return FindClose(handle) != 0; #else diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 1dcc07bc3..21ebacd06 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -50,6 +50,7 @@ public: time_t file_modtime() ; private: + bool is_open; bool validity; #ifdef WINDOWS_SYS