From 1383846364412e6ef2951156281c8ff5d16136b1 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 27 Sep 2018 13:42:49 +0200 Subject: [PATCH] added extra list view in own files. --- .../src/file_sharing/dir_hierarchy.cc | 7 +- .../src/file_sharing/file_sharing_defaults.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 112 ++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 9 +- libretroshare/src/ft/ftextralist.cc | 29 +++-- libretroshare/src/ft/ftextralist.h | 7 ++ libretroshare/src/ft/ftserver.cc | 3 + libretroshare/src/retroshare/rstypes.h | 1 + retroshare-gui/src/gui/RemoteDirModel.cpp | 4 +- 9 files changed, 149 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 743176e38..b6208eb57 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -865,7 +865,7 @@ void InternalFileHierarchyStorage::print() const for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) @@ -1183,6 +1183,11 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) } free(buffer) ; + std::string err_str ; + + if(!check(err_str)) + std::cerr << "(EE) Error while loading file hierarchy " << fname << std::endl; + return true ; } catch(read_error& e) diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 6633e348f..75f19075a 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -25,6 +25,7 @@ static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 m static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. But we only update for real if something has changed. static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP = 60 ; // 60 sec. +static const uint32_t DELAY_BETWEEN_EXTRA_FILES_CACHE_UPDATES = 2 ; // 2 sec. static const uint32_t DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR = 60*24*86400 ; // delete non empty remoe directories after 60 days of inactivity static const uint32_t DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR = 5*24*86400 ; // delete empty remote directories after 5 days of inactivity diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 156490e18..c6ee048be 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -38,6 +38,7 @@ //#define DEBUG_P3FILELISTS 1 #define DEBUG_CONTENT_FILTERING 1 +#define DEBUG_FILE_HIERARCHY 1 static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; @@ -71,6 +72,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + mLastExtraFilesCacheUpdate = 0; mLastCleanupTime = 0 ; mLastDataRecvTS = 0 ; mTrustFriendNodesForBannedFiles = TRUST_FRIEND_NODES_FOR_BANNED_FILES_DEFAULT; @@ -884,6 +886,8 @@ void p3FileDatabase::requestDirUpdate(void *ref) } } +// Finds the pointer to the sub-element #row under element ref. + bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result, FileSearchFlags flags ) const { @@ -891,10 +895,10 @@ bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result, { if(flags & RS_FILE_HINTS_LOCAL) { - if(row != 0) + if(row != 0 && row != 1) return false ; - convertEntryIndexToPointer(0,0,result); + convertEntryIndexToPointer(0,row,result); return true ; } @@ -913,12 +917,18 @@ bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result, convertPointerToEntryIndex(ref,e,fi); // check consistency - if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 1 && (flags & RS_FILE_HINTS_LOCAL))) { P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; return false ; } - DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + + if(fi==1 && (flags & RS_FILE_HINTS_LOCAL)) // extra list + { + convertEntryIndexToPointer(row+1,1,result); + return true; + } + DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); // Case where the index is the top of a single person. Can be us, or a friend. @@ -950,6 +960,65 @@ int p3FileDatabase::getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& s } } +void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIndex e,DirDetails& d) const +{ + // update the cache of extra files if last requested too long ago + + time_t now = time(NULL); + + if(mLastExtraFilesCacheUpdate + DELAY_BETWEEN_EXTRA_FILES_CACHE_UPDATES <= now) + { + mExtraFiles->getExtraFileList(mExtraFilesCache); + mLastExtraFilesCacheUpdate = now; + } + + // + + if(e == 0) // "virtual extra files directory" => create a dir with as many child as they are extra files + { + d.parent = NULL ; + + d.prow = 0;//fi-1 ; + d.type = DIR_TYPE_DIR; + d.hash.clear() ; + d.count = mExtraFilesCache.size(); + d.max_mtime = time(NULL); + d.mtime = time(NULL); + d.name = "Temporary shared files"; + d.path = "/"; + d.ref = ref ; + + for(uint32_t i=0;i(i,1,stub.ref); // local shared files from extra list + + d.children.push_back(stub); + } + } + else // extra file. Just query the corresponding data from ftExtra + { + d.prow = 1;//fi-1 ; + d.type = DIR_TYPE_FILE; + + FileInfo& f(mExtraFilesCache[(int)e-1]) ; + + d.hash = f.hash; + d.count = 0; + d.max_mtime = time(NULL); + d.mtime = time(NULL); + d.name = f.fname; + d.path = "/"; + d.ref = ref ; + + convertEntryIndexToPointer(0,1,d.parent) ; + } + + d.id = RsPeerId(); +} + // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { @@ -975,10 +1044,8 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags { d.ref = NULL ; d.type = DIR_TYPE_ROOT; - d.count = 1; d.parent = NULL; d.prow = -1; - d.ref = NULL; d.name = "root"; d.hash.clear() ; d.path = ""; @@ -989,13 +1056,26 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags if(flags & RS_FILE_HINTS_LOCAL) { void *p; - convertEntryIndexToPointer(0,0,p); + { + convertEntryIndexToPointer(0,0,p); // root of own directories DirStub stub; stub.type = DIR_TYPE_PERSON; stub.name = mServCtrl->getOwnId().toStdString(); stub.ref = p; d.children.push_back(stub); + } + { + convertEntryIndexToPointer(0,1,p); // local shared files from extra list + DirStub stub; + stub.type = DIR_TYPE_PERSON; + stub.name = "Temporary shared files"; + stub.ref = p; + + d.children.push_back(stub); + } + + } else for(uint32_t i=0;i(ref,e,fi); // check consistency - if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 1 && (flags & RS_FILE_HINTS_LOCAL))) { P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; return false ; } - DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + + if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list + { + getExtraFilesDirDetails(ref,e,d); + +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + P3FILELISTS_DEBUG() << d << std::endl; +#endif + return true; + } + + + DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); // Case where the index is the top of a single person. Can be us, or a friend. diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index a8ac377ea..8e35c0bf2 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -45,6 +45,7 @@ #pragma once #include "ft/ftsearch.h" +#include "ft/ftextralist.h" #include "retroshare/rsfiles.h" #include "services/p3service.h" @@ -87,9 +88,10 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // [...] more to add here }; - explicit p3FileDatabase(p3ServiceControl *mpeers) ; + explicit p3FileDatabase(p3ServiceControl *mpeers); ~p3FileDatabase(); + void setExtraList(ftExtraList *f) { mExtraFiles = f ; } /*! * \brief forceSyncWithPeers * @@ -169,6 +171,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub bool hashingProcessPaused(); protected: + void getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIndex e,DirDetails& d) const; int filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const; std::string makeRemoteFileName(const RsPeerId& pid) const; @@ -219,8 +222,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; - LocalDirectoryUpdater *mLocalDirWatcher ; + ftExtraList *mExtraFiles; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc @@ -268,6 +271,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub std::map mPrimaryBanList ; // primary list (user controlled) of files banned from FT search and forwarding. map std::map mPeerBannedFiles ; // records of which files other peers ban, stored as H(H(f)) std::set mBannedFileList ; // list of banned hashes. This include original hashs and H(H(f)) when coming from friends. + mutable std::vector mExtraFilesCache; // cache for extra files, to avoid requesting them too often. + mutable time_t mLastExtraFilesCacheUpdate ; bool mTrustFriendNodesForBannedFiles ; bool mBannedFileListNeedsUpdate; time_t mLastPrimaryBanListChangeTimeStamp; diff --git a/libretroshare/src/ft/ftextralist.cc b/libretroshare/src/ft/ftextralist.cc index cf5a4f603..a8ad41b23 100644 --- a/libretroshare/src/ft/ftextralist.cc +++ b/libretroshare/src/ft/ftextralist.cc @@ -96,7 +96,7 @@ void ftExtraList::hashAFile() FileDetails details; { - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); if (mToHash.empty()) return; @@ -116,7 +116,7 @@ void ftExtraList::hashAFile() if (RsDirUtil::hashFile(details.info.path, details.info.fname, details.info.hash, details.info.size)) { - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); /* stick it in the available queue */ mFiles[details.info.hash] = details; @@ -146,7 +146,7 @@ bool ftExtraList::addExtraFile(std::string path, const RsFileHash& hash, std::cerr << std::endl; #endif - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); FileDetails details; @@ -179,7 +179,7 @@ bool ftExtraList::removeExtraFile(const RsFileHash& hash, TransferRequestFlags f std::cerr << std::endl; #endif - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); mHashOfHash.erase(makeEncryptedHash(hash)) ; @@ -230,7 +230,7 @@ bool ftExtraList::cleanupOldFiles() std::cerr << std::endl; #endif - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); time_t now = time(NULL); @@ -286,7 +286,7 @@ bool ftExtraList::hashExtraFile(std::string path, uint32_t period, TransferRequ #endif /* add into queue */ - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); FileDetails details(path, period, flags); details.info.age = time(NULL) + period; @@ -305,7 +305,7 @@ bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info) RsFileHash hash; { /* Find in the path->hash map */ - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); std::map::iterator it; if (mHashedList.end() == (it = mHashedList.find(path))) @@ -420,7 +420,7 @@ bool ftExtraList::saveList(bool &cleanup, std::list& sList) std::cerr << std::endl; #endif - RsStackMutex stack(extMutex); + RS_STACK_MUTEX(extMutex); std::map::const_iterator it; @@ -489,8 +489,8 @@ bool ftExtraList::loadList(std::list& load) /* add into system */ FileDetails file; - RsStackMutex stack(extMutex); - + RS_STACK_MUTEX(extMutex); + FileDetails details; details.info.path = fi->file.path; @@ -513,3 +513,12 @@ bool ftExtraList::loadList(std::list& load) return true; } +void ftExtraList::getExtraFileList(std::vector& files) const +{ + RS_STACK_MUTEX(extMutex); + + files.clear(); + + for(auto it(mFiles.begin());it!=mFiles.end();++it) + files.push_back(it->second.info); +} diff --git a/libretroshare/src/ft/ftextralist.h b/libretroshare/src/ft/ftextralist.h index 5832b33fd..8f6b3cbd2 100644 --- a/libretroshare/src/ft/ftextralist.h +++ b/libretroshare/src/ft/ftextralist.h @@ -137,6 +137,12 @@ public: **/ virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; + /*! + * \brief getExtraFileList + * Retrieves the list for display purposes + */ + void getExtraFileList(std::vector& files) const ; + /*** * Thread Main Loop **/ @@ -146,6 +152,7 @@ public: * Configuration - store extra files. * **/ + protected: virtual RsSerialiser *setupSerialiser(); virtual bool saveList(bool &cleanup, std::list&); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 182964446..cf81cbcc8 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -174,8 +174,11 @@ void ftServer::SetupFtServer() void ftServer::connectToFileDatabase(p3FileDatabase *fdb) { mFileDatabase = fdb ; + mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_LOCAL); // due to a bug in addSearchModule, modules can only be added one by one. Using | between flags wont work. mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_REMOTE); + + mFileDatabase->setExtraList(mFtExtra); } void ftServer::connectToTurtleRouter(p3turtle *fts) { diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index b1b35748c..9b480f6f8 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -160,6 +160,7 @@ struct PeerBandwidthLimits : RsSerializable #define DIR_TYPE_PERSON 0x02 #define DIR_TYPE_DIR 0x04 #define DIR_TYPE_FILE 0x08 +#define DIR_TYPE_EXTRA 0x10 /* flags for Directry request - * two types; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 22fb0af5b..8eee722c7 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -1158,9 +1158,9 @@ void FlatStyle_RDM::postMods() bool RetroshareDirModel::requestDirDetails(void *ref, bool remote,DirDetails& d) const { -#ifdef RDM_DEBUG +//#ifdef RDM_DEBUG std::cerr << "RequestDirDetails:: ref = " << ref << ", remote=" << remote << std::endl; -#endif +//#endif // We look in cache and re-use the last result if the reference and remote are the same.