fixed a few bugs in hash-based sync. Not there yet.

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -1,3 +1,5 @@
+#include "util/rsdir.h"
+
 #include "dir_hierarchy.h"
 #include "filelist_io.h"
 
@@ -28,6 +30,36 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
     de->dir_modtime=0;
 
     mNodes.push_back(de) ;
+#warning not very elegant. We should remove the leading /
+    mDirHashes[computeDirHash("/")] = 0 ;
+}
+
+RsFileHash InternalFileHierarchyStorage::computeDirHash(const std::string& dir_path)
+{
+    return RsDirUtil::sha1sum((unsigned char*)dir_path.c_str(),dir_path.length()) ;
+}
+bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
+{
+    if(!checkIndex(index,FileStorageNode::TYPE_DIR))
+        return false ;
+
+    DirEntry& d = *static_cast<DirEntry*>(mNodes[index]) ;
+
+    hash = computeDirHash( d.dir_parent_path + "/" + d.dir_name ) ;
+
+    std::cerr << "Computing dir hash from index " << index << ". Dir=\"" << d.dir_parent_path + "/" + d.dir_name << "\" hash=" << hash << std::endl;
+
+    return true;
+}
+bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const
+{
+    std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mDirHashes.find(hash) ;
+
+    if(it == mDirHashes.end())
+        return false;
+
+    index = it->second;
+    return true;
 }
 
 int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e)

libretroshare/src/file_sharing/dir_hierarchy.h

@@ -81,6 +81,13 @@ public:
 
     time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index);
 
+    // hash stuff
+
+    static RsFileHash computeDirHash(const std::string& dir_path);
+
+    bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ;
+    bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ;
+
     // file/dir access and modification
     bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
 

libretroshare/src/file_sharing/directory_storage.cc

@@ -193,6 +193,8 @@ bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo&
 
 bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
 {
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+
     d.children.clear() ;
     time_t now = time(NULL) ;
 
@@ -266,6 +268,18 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
 
     return true;
 }
+
+bool DirectoryStorage::getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    return mFileHierarchy->getDirHashFromIndex(index,hash) ;
+}
+bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    return mFileHierarchy->getIndexFromDirHash(hash,index) ;
+}
+
 /******************************************************************************************************************/
 /*                                           Local Directory Storage                                              */
 /******************************************************************************************************************/
@@ -477,7 +491,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
         {
             RsFileHash hash ;
 
-            if(!getHashFromIndex(dir->subdirs[i],hash))
+            if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
             {
                 std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl;
                 continue ;
@@ -664,7 +678,7 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co
     {
         DirectoryStorage::EntryIndex file_index ;
 
-        if(!getIndexFromHash(subfiles_hash[i],file_index))
+        if(!getIndexFromDirHash(subfiles_hash[i],file_index))
         {
             std::cerr << "(EE) Cannot optain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl;
             continue;

libretroshare/src/file_sharing/directory_storage.h

@@ -101,8 +101,8 @@ class DirectoryStorage
         bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname,  uint64_t size, time_t modf_time) ;
         bool updateHash(const EntryIndex& index,const RsFileHash& hash);
 
-        bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; }
+        bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ;
-        bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; }
+        bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ;
 
         void print();
         void cleanup();

libretroshare/src/file_sharing/p3filelists.cc

@@ -12,7 +12,7 @@
 
 #include "rsserver/p3face.h"
 
-#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: "
+#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << ": p3FileLists: "
 
 static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED     = 0x0000 ;
 static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED  = 0x0001 ;
@@ -187,7 +187,7 @@ int p3FileDatabase::tick()
         {
             if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end())
             {
-                std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl;
+                P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl;
 #ifdef DEBUG_FILE_HIERARCHY
                 mRemoteDirectories[i]->print();
 #endif
@@ -210,20 +210,20 @@ int p3FileDatabase::tick()
 void p3FileDatabase::startThreads()
 {
     RS_STACK_MUTEX(mFLSMtx) ;
-    std::cerr << "Starting directory watcher thread..." ;
+    P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ;
     mLocalDirWatcher->start();
-    std::cerr << "Done." << std::endl;
+    P3FILELISTS_DEBUG() << "Done." << std::endl;
 }
 void p3FileDatabase::stopThreads()
 {
     RS_STACK_MUTEX(mFLSMtx) ;
-    std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ;
+    P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ;
     mHashCache->fullstop();
-    std::cerr << "Done." << std::endl;
+    P3FILELISTS_DEBUG() << "Done." << std::endl;
 
-    std::cerr << "Stopping directory watcher thread..." ; std::cerr.flush() ;
+    P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ;
     mLocalDirWatcher->fullstop();
-    std::cerr << "Done." << std::endl;
+    P3FILELISTS_DEBUG() << "Done." << std::endl;
 }
 
 void p3FileDatabase::tickWatchers()
@@ -367,7 +367,7 @@ void p3FileDatabase::cleanup()
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
-        std::cerr << "p3FileDatabase::cleanup()" << std::endl;
+        P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl;
 
         // look through the list of friend directories. Remove those who are not our friends anymore.
         //
@@ -429,7 +429,7 @@ void p3FileDatabase::cleanup()
         for(std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();)
             if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now)
             {
-                std::cerr << "  removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl;
+                P3FILELISTS_DEBUG() << "  removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl;
 
                 std::map<DirSyncRequestId,DirSyncRequestData>::iterator tmp(it);
                 ++tmp;
@@ -438,7 +438,7 @@ void p3FileDatabase::cleanup()
             }
             else
             {
-                std::cerr << "  keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl;
+                P3FILELISTS_DEBUG() << "  keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl;
                 ++it ;
             }
     }
@@ -905,7 +905,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
 
         EntryIndex entry_index = DirectoryStorage::NO_INDEX;
 
-        if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index))
+        if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
         {
             std::cerr << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
             return;
@@ -970,12 +970,6 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
-        if(entry_index == DirectoryStorage::NO_INDEX)
-        {
-            std::cerr << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
-            return;
-        }
-
         std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.find(item->request_id) ;
 
         if(it == mPendingSyncRequests.end())
@@ -1004,7 +998,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
         if(mRemoteDirectories[fi] == NULL)
             mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId()));
 
-        if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index))
+        if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index))
         {
             std::cerr << std::endl << "  (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl;
             return ;
@@ -1078,7 +1072,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
 
         RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
 
-        if(!rds->getHashFromIndex(e,item->entry_hash) )
+        if(!rds->getDirHashFromIndex(e,item->entry_hash) )
         {
             std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl;
             return;

retroshare-gui/src/gui/SharedFilesDialog.cpp

@@ -85,6 +85,8 @@
 //
 #define DONT_USE_SEARCH_IN_TREE_VIEW 1
 
+//#define DEBUG_SHARED_FILES_DIALOG 1
+
 const QString Image_AddNewAssotiationForFile = ":/images/kcmsystem24.png";
 
 class SFDSortFilterProxyModel : public QSortFilterProxyModel
@@ -842,7 +844,9 @@ void SharedFilesDialog::saveExpandedPaths(std::set<std::string>& expanded_indexe
     if(ui.dirTreeView->model() == NULL)
         return ;
 
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "Saving expanded items. " << std::endl;
+#endif
     for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row)
     {
         std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString();
@@ -855,7 +859,9 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set<std::string>& expand
     if(ui.dirTreeView->model() == NULL)
         return ;
 
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "Restoring expanded items. " << std::endl;
+#endif
     for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row)
     {
         std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString();
@@ -866,29 +872,39 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set<std::string>& expand
 void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set<std::string>& exp)
 {
     std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString();
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl;
+#endif
 
     if(ui.dirTreeView->isExpanded(index))
     {
+#ifdef DEBUG_SHARED_FILES_DIALOG
         std::cerr << "Index " << local_path << " is expanded." << std::endl;
+#endif
         if(index.isValid())
             exp.insert(local_path) ;
 
         for(int row=0;row<ui.dirTreeView->model()->rowCount(index);++row)
             recursSaveExpandedItems(index.child(row,0),local_path,exp) ;
     }
+#ifdef DEBUG_SHARED_FILES_DIALOG
     else
         std::cerr << "Index is not expanded." << std::endl;
+#endif
 }
 
 void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set<std::string>& exp)
 {
     std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString();
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl;
+#endif
 
     if(exp.find(local_path) != exp.end())
     {
+#ifdef DEBUG_SHARED_FILES_DIALOG
         std::cerr << "re expanding index " << local_path << std::endl;
+#endif
         ui.dirTreeView->setExpanded(index,true) ;
 
         for(int row=0;row<ui.dirTreeView->model()->rowCount(index);++row)
@@ -904,7 +920,9 @@ void  SharedFilesDialog::postModDirectories(bool local)
     }
     std::set<std::string> expanded_indexes;
     saveExpandedPaths(expanded_indexes) ;
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl;
+#endif
 
     /* Notify both models, only one is visible */
 	tree_model->postMods();
@@ -916,7 +934,9 @@ void  SharedFilesDialog::postModDirectories(bool local)
     if (ui.filterPatternLineEdit->text().isEmpty() == false)
 		FilterItems();
 
+#ifdef DEBUG_SHARED_FILES_DIALOG
     std::cerr << "****** updated directories! ******" << std::endl;
+#endif
 	QCoreApplication::flush();
 }