diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7e8cea686..8ea522b6f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -53,7 +53,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -65,7 +65,8 @@ class InternalFileHierarchyStorage std::vector subdirs ; std::vector subfiles ; - time_t most_recent_time; + time_t dir_modtime ; // this accounts for deleted files, etc. + time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. }; // class stuff @@ -221,7 +222,10 @@ class InternalFileHierarchyStorage std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; - static_cast(mNodes[file_index])->file_hash = hash ; + RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + + old_hash = hash ; + return true; } bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) @@ -243,6 +247,25 @@ class InternalFileHierarchyStorage return true; } + // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + + time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) + { + DirEntry& d(*static_cast(mNodes[dir_index])) ; + + time_t largest_modf_time = d.dir_modtime ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + + for(uint32_t i=0;i(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ; for(int i=0;irecursUpdateLastModfTime(EntryIndex(0)) ; + + std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +} + std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const { /**** MUST ALREADY BE LOCKED ****/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 637a8e88e..fbc6b3297 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -137,6 +137,7 @@ public: void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; + void updateTimeStamps(); /*! * \brief getFileInfo Converts an index info a full file info structure. * \param i index in the directory structure diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 731d92902..f9fada262 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,13 +10,15 @@ // Local Directory Updater // //=============================================================================================================// -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { mLastSweepTime = 0; + mLastTSUpdateTime = 0; } void LocalDirectoryUpdater::data_tick() @@ -28,8 +30,13 @@ void LocalDirectoryUpdater::data_tick() sweepSharedDirectories() ; mLastSweepTime = now; } - else - usleep(10*1000*1000); + + if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime) + { + mSharedDirectories->updateTimeStamps() ; + mLastTSUpdateTime = now ; + } + usleep(10*1000*1000); } void LocalDirectoryUpdater::forceUpdate() @@ -93,13 +100,13 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p switch(dirIt.file_type()) { case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + subfiles[dirIt.file_name()].size = dirIt.file_size(); + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; - break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + break; default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } @@ -139,3 +146,9 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 1248aeefb..94b9a2080 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -35,5 +35,6 @@ private: LocalDirectoryStorage *mSharedDirectories ; time_t mLastSweepTime; + time_t mLastTSUpdateTime; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 7772af5c6..5ef9af42a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -17,9 +17,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; -static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; -static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; +static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; +static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") @@ -608,6 +609,17 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // Update of remote directories // //==============================================================================================================================// +// Algorithm: +// +// Local dirs store the last modif time of the file, in local time +// - the max time is computed upward until the root of the hierarchy +// - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. +// +// Remote dirs store the last modif time of the files/dir in the friend's time +// - local node sends the last known modf time to friends, +// - friends respond with either a full directory content, or an acknowledge that the time is right +// + void p3FileDatabase::tickRecv() { RsItem *item ; @@ -637,15 +649,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) { // look at item TS. If local is newer, send the full directory content. - time_t recurs_max_modf_TS; + time_t recurs_max_modf_TS, last_update_TS; - if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) - { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; - return ; - } + P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->last_known_recurs_modf_TS < recurs_max_modf_TS) { // send full update of directory content } @@ -655,7 +669,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) } } -void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item) { // update the directory content for the specified friend. @@ -677,9 +691,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS,last_update_TS ; - if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl; return ; } @@ -705,9 +719,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r mPendingSyncRequests[sync_req_id] = data ; - RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ; item->entry_index = e ; - item->known_TS = recurs_max_modf_TS ; + item->last_known_recurs_modf_TS = recurs_max_modf_TS ; + item->PeerId(rds->peerId()) ; sendItem(item) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 8160095aa..58a635d8d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -173,8 +173,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub std::map mFriendIndexMap ; std::vector mFriendIndexTab; - // TS for friend list update - time_t mLastRemoteDirSweepTS ; + // Directory synchronization + // + struct DirSyncRequestData + { + time_t request_TS ; + uint32_t flags ; + }; + + time_t mLastRemoteDirSweepTS ; // TS for friend list update + std::map mPendingSyncRequests ; // pending requests, waiting for an answer void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 986b4284c..198bcc9db 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -1,3 +1,5 @@ +#include "serialiser/rsbaseserial.h" + #include "file_sharing/rsfilelistitems.h" RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) @@ -8,8 +10,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) /* get the type and size */ uint32_t rstype = getRsItemId(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) - return NULL; /* wrong type */ + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ switch(getRsItemSubType(rstype)) { @@ -28,26 +30,26 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) uint32_t RsFileListsSerialiser::size(RsItem *item) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serial_size() ; + if(flst_item != NULL) + return flst_item->serial_size() ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serialise(data,*size) ; + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } @@ -185,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void return item; } -RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem(); uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. @@ -234,7 +236,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su uint32_t rstype = getRsItemId(data); uint32_t rssize = getRsItemSize(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) { #ifdef RSSERIAL_DEBUG std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; @@ -265,11 +267,6 @@ uint32_t RsFileListsSyncReqItem::serial_size()const uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS @@ -278,16 +275,11 @@ uint32_t RsFileListsSyncReqItem::serial_size()const return s; } -uint32_t RsFileListsSyncReqItem::serial_size()const +uint32_t RsFileListsSyncDirItem::serial_size()const { uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 6eb463b7d..5ddd1f063 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -75,7 +75,7 @@ class RsFileListsSyncReqItem : public RsFileListsItem { public: - RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -93,20 +93,20 @@ class RsFileListsSyncDirItem : public RsFileListsItem { public: - RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} - virtual void clear(); - virtual std::ostream &print(std::ostream &out, uint16_t indent); + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); - virtual bool serialise(void *data,uint32_t& size) const; - virtual uint32_t serial_size() const ; + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash - RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. }; class RsFileListsSerialiser : public RsSerialType diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index ea7344976..82b0ea27b 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -41,14 +41,14 @@ file_lists { file_sharing/filelist_io.h \ file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ - file_sharing/filelistitems.h + file_sharing/rsfilelistitems.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ file_sharing/filelist_io.cc \ file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ - file_sharing/filelistitems.cc + file_sharing/rsfilelistitems.cc } dsdv {