From d8a24c8c3a5714923d62181c2e9a28bcdb488cf4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 26 Aug 2016 16:29:02 +0200 Subject: [PATCH] added serialisation of dir entries for exchange with friends. Fixed a few bugs in row and ref calculation --- .../src/file_sharing/directory_storage.cc | 211 +++++++++++++++++- .../src/file_sharing/directory_storage.h | 30 ++- libretroshare/src/file_sharing/filelist_io.h | 3 + libretroshare/src/file_sharing/p3filelists.cc | 76 +++++-- .../src/file_sharing/rsfilelistitems.cc | 9 +- .../src/file_sharing/rsfilelistitems.h | 2 +- libretroshare/src/rsserver/rstypes.cc | 2 +- 7 files changed, 298 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8ea522b6f..f63f0df6a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -2,6 +2,7 @@ #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" +#include "filelist_io.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -53,7 +54,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -67,6 +68,8 @@ class InternalFileHierarchyStorage time_t dir_modtime ; // this accounts for deleted files, etc. time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + + time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; // class stuff @@ -246,6 +249,36 @@ class InternalFileHierarchyStorage return true; } + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + recurs_max_modf_TS = d.most_recent_time ; + local_update_TS = d.dir_update_time ; + + return true; + } + bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + d.most_recent_time = recurs_max_modf_TS ; + d.dir_update_time = local_update_TS ; + + return true; + } // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. @@ -340,9 +373,58 @@ class InternalFileHierarchyStorage return false; } - bool check() // checks consistency of storage. + bool check(std::string& error_string) const // checks consistency of storage. { - return true; + // recurs go through all entries, check that all + + std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. + hits[0] = 1 ; // because 0 is never the child of anyone + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + // stamp the kids + const DirEntry& de = *static_cast(mNodes[i]) ; + + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subdirs[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subdirs[j]] = 1; + } + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subfiles[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subfiles[j]] = 1; + } + + } + + for(uint32_t i=0;igetDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} +bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + locked_check() ; + return res ; } bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + locked_check() ; + return res ; } bool DirectoryStorage::removeDirectory(const EntryIndex& indx) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->removeDirectory(indx); + bool res = mFileHierarchy->removeDirectory(indx); + + locked_check(); + return res ; +} + +void DirectoryStorage::locked_check() +{ + std::string error ; + if(!mFileHierarchy->check(error)) + std::cerr << "Check error: " << error << std::endl; } bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) @@ -773,8 +879,95 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) /* find parent pointer, and row */ - std::cerr << "LocalDirectoryStorage::extractData(): indx=" << indx << " Returning this:" << std::endl; - std::cerr << d << std::endl; - return true; } + +bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + if(dir == NULL) + { + std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl; + return false; + } + + unsigned char *section_data = NULL; + uint32_t section_size = 0; + uint32_t section_offset = 0; + + // we need to send: + // - the name of the directory, its TS + // - the index entry for each subdir (the updte TS are exchanged at a higher level) + // - the file info for each subfile + // + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + + // serialise number of subdirs and number of subfiles + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subfiles.size() )) return false ; + + // serialise subdirs entry indexes + + for(uint32_t i=0;isubdirs.size();++i) + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subdirs[i] )) return false ; + + // serialise directory subfiles, with info for each of them + + for(uint32_t i=0;isubfiles.size();++i) + { + unsigned char *file_section_data = NULL ; + uint32_t file_section_offset = 0 ; + uint32_t file_section_size = 0; + + const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ; + + if(file == NULL) + { + std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl; + continue ; + } + + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subfiles[i] )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file->file_modtime)) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ; + + free(file_section_data) ; + } + + std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; + + return true ; +} + +/******************************************************************************************************************/ +/* Remote Directory Storage */ +/******************************************************************************************************************/ + +bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; + + NOT_IMPLEMENTED(); + + return true ; +} + + + + + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 309bff328..dd3353ad1 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -9,7 +9,9 @@ #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } +class RsTlvBinaryData ; class InternalFileHierarchyStorage ; +class RsTlvBinaryData ; class DirectoryStorage { @@ -26,8 +28,8 @@ class DirectoryStorage virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; - bool setUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; + bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -109,6 +111,9 @@ class DirectoryStorage void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + // debug + void locked_check(); + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. std::string mFileName; @@ -125,6 +130,17 @@ class RemoteDirectoryStorage: public DirectoryStorage public: RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} + + /*! + * \brief deserialiseDirEntry + * Loads a serialised directory content coming from a friend. The directory entry needs to exist already, + * as it is created when updating the parent. + * + * \param indx index of the directory to update + * \param bindata binary data to deserialise from + * \return false when the directory cannot be found. + */ + bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; }; class LocalDirectoryStorage: public DirectoryStorage @@ -150,6 +166,16 @@ public: virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; + /*! + * \brief serialiseDirEntry + * Produced a serialised directory content listing suitable for export to friends. + * + * \param indx index of the directory to serialise + * \param bindata binary data created by serialisation + * \return false when the directory cannot be found. + */ + bool serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) ; + private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 9c5818025..a2ee26916 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -20,6 +20,9 @@ static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; class FileListIO { diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 6fa89bd35..768ce370c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -113,7 +113,7 @@ int p3FileDatabase::tick() last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI -// RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } if(mUpdateFlags) @@ -129,7 +129,7 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } - if(mLastRemoteDirSweepTS + 5 < now) + if(mLastRemoteDirSweepTS + 30 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -227,6 +227,9 @@ void p3FileDatabase::cleanup() P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + if(mRemoteDirectories.size() <= friend_index) + mRemoteDirectories.resize(friend_index+1,NULL) ; + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; @@ -324,7 +327,7 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi // [ 10 bits | 22 bits ] // // This means that the whoel software has the following build-in limitation: - // * 1024 friends + // * 1023 friends // * 4M shared files. uint32_t fe = (uint32_t)e ; @@ -346,8 +349,21 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags { RS_STACK_MUTEX(mFLSMtx) ; + d.children.clear(); + // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) - // or at the top of our own list of shred directories, depending on the flags. + // or at the top of our own list of shared directories, depending on the flags. + + // Friend index is used as follows: + // 0 : own id + // 1...n : other friends + // + // entry_index: starts at 0. + // + // The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL. + // So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories. + // + // We should also check the consistency between flags and the content of ref. if (ref == NULL) { @@ -363,7 +379,6 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.age = 0; d.flags.clear() ; d.min_age = 0 ; - d.children.clear(); if(flags & RS_FILE_HINTS_LOCAL) { @@ -379,7 +394,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags else for(uint32_t i=0;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p); DirStub stub; stub.type = DIR_TYPE_PERSON; @@ -390,6 +405,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; + return true ; } @@ -398,9 +416,17 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags convertPointerToEntryIndex(ref,e,fi); + // check consistency + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + { + std::cerr << "remote request on local index or local request on remote index. This should not happen." << std::endl; + return false ; + } + DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + // Case where the index is the top of a single person. Can be us, or a friend. - bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; + bool res = storage->extractData(e,d); // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -410,22 +436,18 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags for(uint32_t i=0;iparentRow(e) ; + d.prow = storage->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - if(flags & RS_FILE_HINTS_LOCAL) - d.id = mOwnId ; - else - d.id = mRemoteDirectories[fi]->peerId(); + d.id = storage->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -670,7 +692,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -686,7 +708,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; - ritem->last_known_recurs_modf_TS = local_update_time ; + ritem->last_known_recurs_modf_TS = local_recurs_max_time ; } } } @@ -725,14 +747,14 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Uptating." << std::endl; + P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; - mRemoteDirectories[fi]->deserialiseUpdateEntry(item->entry_index,item->directory_content_data) ; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data) ; + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); #warning should notify the GUI here // notify the GUI if the hierarchy has changed @@ -750,7 +772,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS_remote_time,local_update_TS; - if(!rds->getUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; @@ -798,6 +820,18 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r locked_recursSweepRemoteDirectory(rds,*it); } +p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) +{ + uint64_t r = e ; + + // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + + for(uint32_t i=0;i::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; - std::cerr << " Children : " ; for(uint32_t i=0;i