From edc602f68f67e5549af3288d5b4bb7a137a6f71b Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 20 Aug 2016 16:23:11 +0200 Subject: [PATCH] added infrastructure for synchronisation of file lists --- .../src/file_sharing/directory_storage.cc | 22 +- .../src/file_sharing/directory_storage.h | 6 +- .../src/file_sharing/directory_updater.cc | 11 +- .../src/file_sharing/directory_updater.h | 10 +- libretroshare/src/file_sharing/p3filelists.cc | 198 ++++++++--- libretroshare/src/file_sharing/p3filelists.h | 23 +- .../src/file_sharing/rsfilelistitems.cc | 336 ++++++++++++++++++ .../src/file_sharing/rsfilelistitems.h | 132 +++++++ libretroshare/src/libretroshare.pro | 18 +- libretroshare/src/serialiser/itempriorities.h | 5 + 10 files changed, 692 insertions(+), 69 deletions(-) create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.cc create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.h diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 72aaf6f52..7e8cea686 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -303,6 +303,20 @@ class InternalFileHierarchyStorage return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } + bool searchHash(const RsFileHash& hash,std::list& results) + { + std::map::const_iterator it = mHashes.find(hash); + + if( it != mHashes.end() ) + { + results.clear(); + results.push_back(it->second) ; + return true ; + } + else + return false; + } + bool check() // checks consistency of storage. { return true; @@ -390,7 +404,7 @@ private: return true ; } - std::map mHashes ; // used for fast search access + std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? }; /******************************************************************************************************************/ @@ -498,6 +512,12 @@ bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash return mFileHierarchy->updateHash(index,hash); } +int DirectoryStorage::searchHash(const RsFileHash& hash, std::list &results) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchHash(hash,results); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9f2b63be9..637a8e88e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -23,10 +23,10 @@ class DirectoryStorage void save() const ; virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} - virtual int searchHash(const RsFileHash& hash, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } + virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - void getFileDetails(EntryIndex i) ; + bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -47,6 +47,8 @@ class DirectoryStorage // info about the directory that is pointed by the iterator std::string name() const ; + time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified. + time_t last_update_time() const ; // last time this directory was updated private: EntryIndex mParentIndex ; // index of the parent dir. uint32_t mDirTabIndex ; // index in the vector of subdirs. diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 7f427ff7d..731d92902 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,12 +6,12 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +//=============================================================================================================// +// Local Directory Updater // +//=============================================================================================================// -void RemoteDirectoryUpdater::tick() -{ - // use the stored iterator -} +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) @@ -139,4 +139,3 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } - diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index d146a61b5..1248aeefb 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -13,6 +13,8 @@ class DirectoryUpdater virtual ~DirectoryUpdater(){} }; +#warning: simplify, if we don't keep the remote directory updater + class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread { public: @@ -35,11 +37,3 @@ private: time_t mLastSweepTime; }; -class RemoteDirectoryUpdater: public DirectoryUpdater -{ - public: - RemoteDirectoryUpdater() {} - virtual ~RemoteDirectoryUpdater() {} - - virtual void tick() ; -}; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9a8496470..7772af5c6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -3,6 +3,7 @@ #include "file_sharing/p3filelists.h" #include "file_sharing/directory_storage.h" #include "file_sharing/directory_updater.h" +#include "file_sharing/rsfilelistitems.h" #include "retroshare/rsids.h" #include "retroshare/rspeers.h" @@ -25,17 +26,16 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) { // loads existing indexes for friends. Some might be already present here. // - mDirectories.clear() ; // we should load them! - - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ; - mDirectories.push_back(mLocalSharedDirs) ; + mRemoteDirectories.clear() ; // we should load them! + mOwnId = mpeers->getOwnId() ; + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); mHashCache = new HashStorage("hash_cache.bin") ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; - mRemoteDirWatcher = NULL; // not used yet mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + mLastRemoteDirSweepTS = 0 ; } void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) @@ -60,14 +60,13 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(uint32_t i=0;iroot()) ; + + mLastRemoteDirSweepTS = now; + } + return 0; } @@ -153,19 +163,12 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } -void p3FileDatabase::tickRecv() -{ -} -void p3FileDatabase::tickSend() -{ - // go through the list of out requests and send them to the corresponding friends, if they are online. -} - bool p3FileDatabase::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + NOT_IMPLEMENTED(); return true ; @@ -194,19 +197,19 @@ void p3FileDatabase::cleanup() friend_set.insert(*it) ; } - for(uint32_t i=1;ipeerId()) == friend_set.end()) + for(uint32_t i=0;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; - delete mDirectories[i]; - mDirectories[i] = NULL ; + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - friend_set.erase(mDirectories[i]->peerId()); + friend_set.erase(mRemoteDirectories[i]->peerId()); - mFriendIndexMap.erase(mDirectories[i]->peerId()); + mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); mFriendIndexTab[i].clear(); } @@ -218,12 +221,12 @@ void p3FileDatabase::cleanup() uint32_t friend_index = locked_getFriendIndex(*it) ; - if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) + if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) continue ; P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; - mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -244,7 +247,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { // allocate a new index for that friend, and tell that we should save. uint32_t found = 0 ; - for(uint32_t i=1;isecond) - mDirectories.resize(it->second+1,NULL) ; + if(mRemoteDirectories.size() <= it->second) + mRemoteDirectories.resize(it->second,NULL) ; return it->second; } @@ -372,14 +375,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags stub.ref = p; d.children.push_back(stub); } - else for(uint32_t i=1;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i,p); DirStub stub; stub.type = DIR_TYPE_PERSON; - stub.name = mDirectories[i]->peerId().toStdString(); + stub.name = mRemoteDirectories[i]->peerId().toStdString(); stub.ref = p; d.children.push_back(stub); } @@ -396,7 +399,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - bool res = mDirectories[fi]->extractData(e,d) ; + bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -414,11 +417,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags } else { - d.prow = mDirectories[fi]->parentRow(e) ; + d.prow = mRemoteDirectories[fi]->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - d.id = mDirectories[fi]->peerId(); + if(flags & RS_FILE_HINTS_LOCAL) + d.id = mOwnId ; + else + d.id = mRemoteDirectories[fi]->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -451,7 +457,7 @@ uint32_t p3FileDatabase::getType(void *ref) const if(e == 0) return DIR_TYPE_PERSON ; - return mDirectories[fi]->getEntryType(e) ; + return mRemoteDirectories[fi]->getEntryType(e) ; } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed @@ -536,13 +542,14 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F if(res.empty()) return false; - EntryIndex indx = *res.begin() ; // no need to report dupicates + EntryIndex indx = *res.begin() ; // no need to report duplicates mLocalSharedDirs->getFileInfo(indx,info) ; return true; } - else + + if(hintflags & RS_FILE_HINTS_REMOTE) { NOT_IMPLEMENTED(); return false; @@ -596,3 +603,114 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& RS_STACK_MUTEX(mFLSMtx) ; return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; } + +//==============================================================================================================================// +// Update of remote directories // +//==============================================================================================================================// + +void p3FileDatabase::tickRecv() +{ + RsItem *item ; + + while( NULL != (item = recvItem()) ) + { + switch(item->PacketSubType()) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; + break ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast(item) ) ; + break ; + default: + std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; + } + + delete item ; + } +} + +void p3FileDatabase::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. +} + +void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) +{ + // look at item TS. If local is newer, send the full directory content. + + time_t recurs_max_modf_TS; + + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + { + // send full update of directory content + } + else + { + // send last recurs update TS. + } +} + +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +{ + // update the directory content for the specified friend. + + // set the update TS, and the remote modif TS accordingly + + // notify the GUI if the hierarchy has changed +} + +void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) +{ + time_t now = time(NULL) ; + + // get the info for this entry + + // compare TS + + // if not up to date, request update, and return (content is not certified, so no need to recurs yet). + // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. + + time_t recurs_max_modf_TS,last_update_TS ; + + if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + return ; + } + + if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) + { + // check if a request already exists and is not too old either: no need to re-ask. + + DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ; + + std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; + + if(it != mPendingSyncRequests.end()) + { + std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + return ; + } + + std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl; + + DirSyncRequestData data ; + + data.request_TS = now ; + + mPendingSyncRequests[sync_req_id] = data ; + + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + item->entry_index = e ; + item->known_TS = recurs_max_modf_TS ; + + sendItem(item) ; + + return ; + } +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 0fa36088e..8160095aa 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -37,6 +37,9 @@ class LocalDirectoryUpdater ; class RemoteDirectoryStorage ; class LocalDirectoryStorage ; +class RsFileListsSyncReqItem ; +class RsFileListsSyncDirItem ; + class HashStorage ; class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService @@ -135,22 +138,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub private: p3ServiceControl *mServCtrl ; + RsPeerId mOwnId ; + + typedef uint64_t DirSyncRequestId ; + + static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. // - std::list mFastRequestQueue ; - std::list mSlowRequestQueue ; + std::map mFastRequestQueue ; + std::map mSlowRequestQueue ; // Directory storage hierarchies // // The remote one is the reference for the PeerId index below: // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) - std::vector mDirectories ; // mDirectories[0]=mLocalSharedDirs + std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; - RemoteDirectoryUpdater *mRemoteDirWatcher ; // not used yet. LocalDirectoryUpdater *mLocalDirWatcher ; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc @@ -160,9 +167,17 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t locked_getFriendIndex(const RsPeerId& pid); const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; + void handleDirSyncRequest(RsFileListsSyncReqItem *) ; + void handleDirSyncContent(RsFileListsSyncDirItem *) ; + std::map mFriendIndexMap ; std::vector mFriendIndexTab; + // TS for friend list update + time_t mLastRemoteDirSweepTS ; + + void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // HashStorage *mHashCache ; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc new file mode 100644 index 000000000..986b4284c --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -0,0 +1,336 @@ +#include "file_sharing/rsfilelistitems.h" + +RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ + + switch(getRsItemSubType(rstype)) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncReqItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: return deserialFileListsSyncDirItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size); + + default: + { + std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl; + return NULL; + + } + } +} + +uint32_t RsFileListsSerialiser::size(RsItem *item) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serial_size() ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const +{ + tlvsize = serial_size() ; + offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize)) + { + std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl; + return false ; + } +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl; +#endif + offset += 8; + + return true ; +} + +bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + ok &= directory_content_data.SetTlv(data,size,&offset) ; + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +//============================================================================================================================// +// Deserialisation // +//============================================================================================================================// + +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + ok &= item->directory_content_data.GetTlv(data,*size,&offset) ; + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; +#endif + return false; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl; +#endif + return false; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + return true ; +} + +//============================================================================================================================// +// Sizes // +//============================================================================================================================// + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + + return s; +} + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + s += directory_content_data.TlvSize(); + + return s; +} + +void RsFileListsSyncReqItem::clear() +{ +} +void RsFileListsSyncDirItem::clear() +{ + directory_content_data.TlvClear(); +} +std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncReqItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncReqItem", indent); + + return out; +} + +std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncDirItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + printIndent(out , int_Indent); out << "Data size: " << directory_content_data.bin_len << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncDirItem", indent); + + return out; +} diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h new file mode 100644 index 000000000..6eb463b7d --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -0,0 +1,132 @@ +#pragma once + +/* + * libretroshare/src/serialiser: rsnxssitems.h + * + * RetroShare Serialiser. + * + * Copyright 2012 Christopher Evi-Parker, Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include +#include + +#include "serialiser/rsserviceids.h" +#include "serialiser/rsserial.h" +#include "serialiser/rstlvbase.h" +#include "serialiser/rstlvitem.h" +#include "serialiser/rstlvkeys.h" +#include "gxs/rsgxsdata.h" + +// These items have "flag type" numbers, but this is not used. + +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM = 0x02; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03; + +/*! + * Base class for filelist sync items + */ +class RsFileListsItem : public RsItem +{ +public: + RsFileListsItem(uint8_t subtype) + : RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype) + { + setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST); // this is the default. Someitems may be faster, on demand. + return; + } + virtual ~RsFileListsItem(){} + + virtual bool serialise(void *data,uint32_t& size) const = 0 ; + virtual uint32_t serial_size() const = 0 ; + virtual void clear() = 0; + virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0; + + bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const; + + static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ; + static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ; + static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0004 ; + static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ; +}; + +/*! + * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date. + */ +class RsFileListsSyncReqItem : public RsFileListsItem +{ +public: + + RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash +}; + +class RsFileListsSyncDirItem : public RsFileListsItem +{ +public: + + RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. +}; + +class RsFileListsSerialiser : public RsSerialType +{ +public: + + RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} + + virtual ~RsFileListsSerialiser() {} + + virtual uint32_t size(RsItem *item); + virtual bool serialise(RsItem *item, void *data, uint32_t *size); + virtual RsItem* deserialise(void *data, uint32_t *size); + +private: + RsFileListsSyncReqItem *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + + bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type); +}; + + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 70994b8c7..ea7344976 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -37,16 +37,18 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ - file_sharing/hash_cache.h \ - file_sharing/filelist_io.h \ - file_sharing/directory_storage.h \ - file_sharing/directory_updater.h + file_sharing/hash_cache.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ + file_sharing/directory_updater.h \ + file_sharing/filelistitems.h SOURCES *= file_sharing/p3filelists.cc \ - file_sharing/hash_cache.cc \ - file_sharing/filelist_io.cc \ - file_sharing/directory_storage.cc \ - file_sharing/directory_updater.cc + file_sharing/hash_cache.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ + file_sharing/directory_updater.cc \ + file_sharing/filelistitems.cc } dsdv { diff --git a/libretroshare/src/serialiser/itempriorities.h b/libretroshare/src/serialiser/itempriorities.h index 8809754a9..f5008eef0 100644 --- a/libretroshare/src/serialiser/itempriorities.h +++ b/libretroshare/src/serialiser/itempriorities.h @@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST = 2 ; // same priority. const uint8_t QOS_PRIORITY_RS_DISC_SERVICES = 2 ; const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT = 1 ; +// File database +// +const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST = 7 ; +const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST = 3 ; + // Heartbeat. // const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE = 8 ;