half implemented the hash-based communication for dirs between friends

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -393,9 +393,9 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(Direct
 
 bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results)
 {
-    std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mHashes.find(hash);
+    std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mFileHashes.find(hash);
 
-    if( it != mHashes.end() )
+    if( it != mFileHashes.end() )
     {
         results.clear();
         results.push_back(it->second) ;

libretroshare/src/file_sharing/dir_hierarchy.h

@@ -117,5 +117,19 @@ private:
 
     bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir);
 
-    std::map<RsFileHash,DirectoryStorage::EntryIndex> mHashes ; // used for fast search access. We should try something faster than std::map. hash_map??
+    // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data.
+    // is used for fast search access for FT.
+    // Note: We should try something faster than std::map. hash_map??
+
+    std::map<RsFileHash,DirectoryStorage::EntryIndex> mFileHashes ;
+
+    // The directory hashes are the sha1sum of the
+    // full public path to the directory.
+    // The later is used by synchronisation items in order
+    // to avoid sending explicit EntryIndex values.
+    // This is kept separate from mFileHashes because the two are used
+    // in very different ways.
+    //
+    std::map<RsFileHash,DirectoryStorage::EntryIndex> mDirHashes ;
 };
+

libretroshare/src/file_sharing/directory_storage.h

@@ -94,13 +94,16 @@ class DirectoryStorage
         const RsPeerId& peerId() const { return mPeerId ; }
         int parentRow(EntryIndex e) const ;
 
-        bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::__cxx11::string, time_t> &subdirs) ;
+        bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::string, time_t> &subdirs) ;
         bool updateSubFilesList(const EntryIndex& indx, const std::map<std::string, FileTS> &subfiles, std::map<std::string, FileTS> &new_files) ;
         bool removeDirectory(const EntryIndex& indx) ;
 
         bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname,  uint64_t size, time_t modf_time) ;
         bool updateHash(const EntryIndex& index,const RsFileHash& hash);
 
+        bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; }
+        bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; }
+
         void print();
         void cleanup();
 

libretroshare/src/file_sharing/p3filelists.cc

@@ -200,7 +200,7 @@ int p3FileDatabase::tick()
 
         mLastRemoteDirSweepTS = now;
 
-#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up.
+#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up.
             RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
     }
 
@@ -864,6 +864,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string&
 //		- local node sends the last known modf time to friends,
 //		- friends respond with either a full directory content, or an acknowledge that the time is right
 //
+//  Directories are designated by their hash, instead of their index. This allows to hide the non shared directories
+//  behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between
+//  friends is concerned (We obviously could not do that for GUI display, which has a small and constant cost).
 
 void p3FileDatabase::tickRecv()
 {
@@ -898,12 +901,20 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
-        P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
+        P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
 
-        uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ;
+        EntryIndex entry_index = DirectoryStorage::NO_INDEX;
+
+        if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index))
+        {
+            std::cerr << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
+            return;
+        }
+
+        uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ;
         ritem->PeerId(item->PeerId()) ;
         ritem->request_id = item->request_id;
-        ritem->entry_index = item->entry_index ;
+        ritem->entry_hash = item->entry_hash ;
 
         std::list<RsNodeGroupId> node_groups;
         FileStorageFlags node_flags;
@@ -913,15 +924,15 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
             P3FILELISTS_DEBUG() << "  Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
             ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
         }
-        else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
+        else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
         {
-            std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl;
+            std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
             ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
         }
         else
         {
             time_t local_recurs_max_time,local_update_time;
-            mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time);
+            mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time);
 
             if(item->last_known_recurs_modf_TS < local_recurs_max_time)
             {
@@ -931,7 +942,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
                 ritem->last_known_recurs_modf_TS = local_recurs_max_time;
 
                 // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
-                mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ;
+                mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
             }
             else
             {
@@ -950,13 +961,21 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
 
 void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
 {
-    P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl;
+    P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl;
+
+    EntryIndex entry_index = DirectoryStorage::NO_INDEX;
 
     // remove the original request from pending list
 
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
+        if(entry_index == DirectoryStorage::NO_INDEX)
+        {
+            std::cerr << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
+            return;
+        }
+
         std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.find(item->request_id) ;
 
         if(it == mPendingSyncRequests.end())
@@ -984,12 +1003,19 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
 
         if(mRemoteDirectories[fi] == NULL)
             mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId()));
+
+        if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index))
+        {
+            std::cerr << std::endl << "  (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl;
+            return ;
+        }
+        std::cerr << "  entry index is " << entry_index ;
     }
 
     if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED)
     {
-        P3FILELISTS_DEBUG() << "  removing directory with index " << item->entry_index << " because it does not exist." << std::endl;
+        P3FILELISTS_DEBUG() << "  removing directory with index " << entry_index << " because it does not exist." << std::endl;
-        mRemoteDirectories[fi]->removeDirectory(item->entry_index);
+        mRemoteDirectories[fi]->removeDirectory(entry_index);
 
         mRemoteDirectories[fi]->print();
     }
@@ -997,13 +1023,13 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
     {
         P3FILELISTS_DEBUG() << "  Directory is up to date. Setting local TS." << std::endl;
 
-        mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL));
+        mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL));
     }
     else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT)
     {
         P3FILELISTS_DEBUG() << "  Item contains directory data. Deserialising/Updating." << std::endl;
 
-        if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data))
+        if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data))
             RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);						 // notify the GUI if the hierarchy has changed
         else
             std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl;
@@ -1051,7 +1077,13 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
         P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl;
 
         RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
-        item->entry_index =  e ;
+
+        if(!rds->getHashFromIndex(e,item->entry_hash) )
+        {
+            std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl;
+            return;
+        }
+
         item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ;
         item->request_id = sync_req_id ;
         item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ;
@@ -1080,14 +1112,16 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
 
 p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e)
 {
+    static uint64_t random_bias = RSRandom::random_u64();
     uint64_t r = e ;
 
-    // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair.
+    // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way
+    // that cannot be brute-forced or reverse-engineered, which explains the random bias.
 
     for(uint32_t i=0;i<RsPeerId::SIZE_IN_BYTES;++i)
         r += (0x3ff92892a94 * peer_id.toByteArray()[i] + r) ^ 0x39e83784378aafe3;
 
-    return r;
+    return r ^ random_bias;
 }
 
 

libretroshare/src/file_sharing/rsfilelistitems.cc

@@ -91,7 +91,7 @@ bool RsFileListsSyncRequestItem::serialise(void *data, uint32_t& size) const
 
     /* RsFileListsSyncMsgItem */
 
-    ok &= setRawUInt32(data, size, &offset, entry_index);
+    ok &= entry_hash.serialise(data, size, offset);
     ok &= setRawUInt32(data, size, &offset, flags      );
     ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
     ok &= setRawUInt64(data, size, &offset, request_id);
@@ -127,7 +127,7 @@ bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const
 
     /* RsFileListsSyncMsgItem */
 
-    ok &= setRawUInt32(data, size, &offset, entry_index);
+    ok &= entry_hash.serialise(data, size, offset);
     ok &= setRawUInt32(data, size, &offset, flags      );
     ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
     ok &= setRawUInt64(data, size, &offset, request_id);
@@ -168,7 +168,7 @@ RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestI
 
     RsFileListsSyncRequestItem* item = new RsFileListsSyncRequestItem();
 
-    ok &= getRawUInt32(data, *size, &offset, &item->entry_index);
+    ok &= item->entry_hash.deserialise(data, *size, offset);
     ok &= getRawUInt32(data, *size, &offset, &item->flags);
     ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
     ok &= getRawUInt64(data, *size, &offset, &item->request_id);
@@ -206,7 +206,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons
     uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
     uint64_t request_id;                // use to determine if changes that have occured since last hash
 
-    ok &= getRawUInt32(data, *size, &offset, &item->entry_index);
+    ok &= item->entry_hash.deserialise(data, *size, offset);
     ok &= getRawUInt32(data, *size, &offset, &item->flags);
     ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
     ok &= getRawUInt64(data, *size, &offset, &item->request_id);
@@ -274,7 +274,7 @@ uint32_t RsFileListsSyncRequestItem::serial_size()const
 
     uint32_t s = 8; //header size
 
-    s += 4; // entry index
+    s += RsFileHash::serial_size(); // entry hash
     s += 4; // flags
     s += 4; // last_known_recurs_modf_TS
     s += 8; // request_id
@@ -287,7 +287,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const
 
     uint32_t s = 8; //header size
 
-    s += 4; // entry index
+    s += RsFileHash::serial_size(); // entry hash
     s += 4; // flags
     s += 4; // last_known_recurs_modf_TS
     s += 8; // request_id
@@ -308,7 +308,7 @@ std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t inde
     printRsItemBase(out, "RsFileListsSyncReqItem", indent);
     uint16_t int_Indent = indent + 2;
 
-    printIndent(out , int_Indent); out << "Entry index:  " << entry_index << std::endl;
+    printIndent(out , int_Indent); out << "Entry hash:   " << entry_hash << std::endl;
     printIndent(out , int_Indent); out << "Flags:        " << (uint32_t) flags << std::endl;
     printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
     printIndent(out , int_Indent); out << "request id:   " << std::hex << request_id  << std::dec << std::endl;
@@ -323,7 +323,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind
     printRsItemBase(out, "RsFileListsSyncDirItem", indent);
     uint16_t int_Indent = indent + 2;
 
-    printIndent(out , int_Indent); out << "Entry index:  " << entry_index << std::endl;
+    printIndent(out , int_Indent); out << "Entry hash:   " << entry_hash << std::endl;
     printIndent(out , int_Indent); out << "Flags:        " << (uint32_t) flags << std::endl;
     printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
     printIndent(out , int_Indent); out << "request id:   " << std::hex << request_id  << std::dec << std::endl;

libretroshare/src/file_sharing/rsfilelistitems.h

@@ -84,10 +84,10 @@ public:
 	virtual bool serialise(void *data,uint32_t& size) const;	
 	virtual uint32_t serial_size() const ; 			
         
-    uint32_t entry_index ;              // index of the directory to sync
+    RsFileHash entry_hash ;               // hash of the directory to sync
-    uint32_t flags;                     // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
+    uint32_t   flags;                     // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
-    uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
+    uint32_t   last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
-    uint64_t request_id;                // use to determine if changes that have occured since last hash
+    uint64_t   request_id;                // use to determine if changes that have occured since last hash
 };
 
 class RsFileListsSyncResponseItem : public RsFileListsItem
@@ -102,10 +102,10 @@ public:
     virtual bool serialise(void *data,uint32_t& size) const;
     virtual uint32_t serial_size() const ;
 
-    uint32_t entry_index ;              // advises whether to use sync hash
+    RsFileHash entry_hash ;               // hash of the directory to sync
-    uint32_t flags;                     // is it a partial/final item (used for large items only)
+    uint32_t   flags;                     // is it a partial/final item (used for large items only)
-    uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
+    uint32_t   last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
-    uint64_t request_id;                // use to determine if changes that have occured since last hash
+    uint64_t   request_id;                // use to determine if changes that have occured since last hash
 
     RsTlvBinaryData directory_content_data ;	// encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
 };