diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d0d40d7bf..f7e8fe61a 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -393,9 +393,9 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(Direct bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) { - std::map::const_iterator it = mHashes.find(hash); + std::map::const_iterator it = mFileHashes.find(hash); - if( it != mHashes.end() ) + if( it != mFileHashes.end() ) { results.clear(); results.push_back(it->second) ; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f0b109c7d..3115543d5 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -117,5 +117,19 @@ private: bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? + // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // is used for fast search access for FT. + // Note: We should try something faster than std::map. hash_map?? + + std::map mFileHashes ; + + // The directory hashes are the sha1sum of the + // full public path to the directory. + // The later is used by synchronisation items in order + // to avoid sending explicit EntryIndex values. + // This is kept separate from mFileHashes because the two are used + // in very different ways. + // + std::map mDirHashes ; }; + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 302c24be5..7d98730ac 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -94,13 +94,16 @@ class DirectoryStorage const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; - bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; } + bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; } + void print(); void cleanup(); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 600e9edec..d53f3c578 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -200,7 +200,7 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; -#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up. +#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up. RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } @@ -864,6 +864,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - local node sends the last known modf time to friends, // - friends respond with either a full directory content, or an acknowledge that the time is right // +// Directories are designated by their hash, instead of their index. This allows to hide the non shared directories +// behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between +// friends is concerned (We obviously could not do that for GUI display, which has a small and constant cost). void p3FileDatabase::tickRecv() { @@ -898,12 +901,20 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { RS_STACK_MUTEX(mFLSMtx) ; - P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ; + EntryIndex entry_index = DirectoryStorage::NO_INDEX; + + if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + + uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ; ritem->PeerId(item->PeerId()) ; ritem->request_id = item->request_id; - ritem->entry_index = item->entry_index ; + ritem->entry_hash = item->entry_hash ; std::list node_groups; FileStorageFlags node_flags; @@ -913,15 +924,15 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } - else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl; + std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -931,7 +942,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->last_known_recurs_modf_TS = local_recurs_max_time; // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer. - mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ; + mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ; } else { @@ -950,13 +961,21 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { - P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; + + EntryIndex entry_index = DirectoryStorage::NO_INDEX; // remove the original request from pending list { RS_STACK_MUTEX(mFLSMtx) ; + if(entry_index == DirectoryStorage::NO_INDEX) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; if(it == mPendingSyncRequests.end()) @@ -984,12 +1003,19 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(mRemoteDirectories[fi] == NULL) mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); + + if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; + return ; + } + std::cerr << " entry index is " << entry_index ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { - P3FILELISTS_DEBUG() << " removing directory with index " << item->entry_index << " because it does not exist." << std::endl; - mRemoteDirectories[fi]->removeDirectory(item->entry_index); + P3FILELISTS_DEBUG() << " removing directory with index " << entry_index << " because it does not exist." << std::endl; + mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } @@ -997,13 +1023,13 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; - if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data)) + if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; @@ -1051,7 +1077,13 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - item->entry_index = e ; + + if(!rds->getHashFromIndex(e,item->entry_hash) ) + { + std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + return; + } + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; item->request_id = sync_req_id ; item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; @@ -1080,14 +1112,16 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { + static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; - // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way + // that cannot be brute-forced or reverse-engineered, which explains the random bias. for(uint32_t i=0;ientry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -206,7 +206,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash - ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -274,7 +274,7 @@ uint32_t RsFileListsSyncRequestItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -287,7 +287,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -308,7 +308,7 @@ std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t inde printRsItemBase(out, "RsFileListsSyncReqItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; @@ -323,7 +323,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind printRsItemBase(out, "RsFileListsSyncDirItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 1ff00d57a..5644d4648 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -84,10 +84,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // index of the directory to sync - uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash }; class RsFileListsSyncResponseItem : public RsFileListsItem @@ -102,10 +102,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. };