reducing linear cost of allocateNewIndex to constant. Should improve huge lags when receiving big file lists for the first time

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -299,8 +299,7 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
             std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl;
 #endif
 
-            delete mNodes[d.subfiles[i]] ;
+            deleteNode(d.subfiles[i]) ;
-            mNodes[d.subfiles[i]] = NULL ;
 
             d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ;
             d.subfiles.pop_back();
@@ -374,23 +373,29 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex
     return true;
 }
 
+void InternalFileHierarchyStorage::deleteNode(uint32_t index)
+{
+    if(mNodes[index] != NULL)
+    {
+        delete mNodes[index] ;
+        mFreeNodes.push_back(index) ;
+        mNodes[index] = NULL ;
+    }
+}
+
 DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
 {
-    int found = -1;
+    while(!mFreeNodes.empty())
-    for(uint32_t j=0;j<mNodes.size();++j)
-        if(mNodes[j] == NULL)
-        {
-            found = j;
-            break;
-        }
-
-    if(found < 0)
     {
-        mNodes.push_back(NULL) ;
+        uint32_t index = mFreeNodes.front();
-        return mNodes.size()-1 ;
+        mFreeNodes.pop_front();
+
+        if(mNodes[index] == NULL)
+            return DirectoryStorage::EntryIndex(index) ;
     }
-    else
+
-        return found ;
+	mNodes.push_back(NULL) ;
+	return mNodes.size()-1 ;
 }
 
 bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
@@ -534,8 +539,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI
             std::cerr << "(EE) Cannot delete node of index " << it->second << " because it is not a file. Inconsistency error!" << std::endl;
             continue ;
         }
-        delete mNodes[it->second] ;
+        deleteNode(it->second) ;
-        mNodes[it->second] = NULL ;
     }
 
     // now update row and parent index for all subnodes
@@ -749,6 +753,8 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co
     std::vector<uint32_t> hits(mNodes.size(),0) ;	// count hits of children. Should be 1 for all in the end. Otherwise there's an error.
     hits[0] = 1 ;	// because 0 is never the child of anyone
 
+    mFreeNodes.clear();
+
     for(uint32_t i=0;i<mNodes.size();++i)
         if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR)
         {
@@ -796,13 +802,15 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co
                 }
             }
         }
+		else if(mNodes[i] == NULL)
+            mFreeNodes.push_back(i) ;
 
     for(uint32_t i=0;i<hits.size();++i)
         if(hits[i] == 0 && mNodes[i] != NULL)
         {
             error_string += " - Orphean node!" ;
-            delete mNodes[i] ;
+
-            mNodes[i] = NULL ;
+            deleteNode(i) ;
         }
 
     return error_string.empty();;
@@ -896,12 +904,9 @@ bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::Entry
         recursRemoveDirectory(d.subdirs[i]);
 
     for(uint32_t i=0;i<d.subfiles.size();++i)
-    {
+        deleteNode(d.subfiles[i]);
-        delete mNodes[d.subfiles[i]] ;
+
-        mNodes[d.subfiles[i]] = NULL ;
+    deleteNode(dir) ;
-    }
-    delete mNodes[dir] ;
-    mNodes[dir] = NULL ;
 
     mDirHashes.erase(hash) ;
 
@@ -1014,6 +1019,8 @@ bool InternalFileHierarchyStorage::load(const std::string& fname)
     uint32_t buffer_size = 0 ;
     uint32_t buffer_offset = 0 ;
 
+    mFreeNodes.clear();
+
     try
     {
         if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_size) )

libretroshare/src/file_sharing/dir_hierarchy.h

@@ -122,6 +122,7 @@ public:
     bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
 
     uint32_t mRoot ;
+    std::list<uint32_t > mFreeNodes ;	// keeps a list of free nodes in order to make insert effcieint
     std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
 
     void compress() ;					// use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
@@ -159,6 +160,10 @@ private:
 
     DirectoryStorage::EntryIndex allocateNewIndex();
 
+    // Deletes an existing entry in mNodes, and keeps record of the indices that get freed.
+
+    void deleteNode(DirectoryStorage::EntryIndex);
+
     // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
     // phase. That allows to keep file information when moving them around.
 

libretroshare/src/file_sharing/p3filelists.cc

@@ -543,7 +543,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
             mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
 
 #ifdef DEBUG_P3FILELISTS
-            P3FILELISTS_DEBUG() << "  adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl;
+            P3FILELISTS_DEBUG() << "  adding missing remote dir entry for friend " << pid << ", with index " << it->second << std::endl;
 #endif
         }
 
@@ -570,7 +570,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
             mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
 
 #ifdef DEBUG_P3FILELISTS
-            P3FILELISTS_DEBUG() << "  adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl;
+            P3FILELISTS_DEBUG() << "  adding missing remote dir entry for friend " << pid << ", with index " << it->second << std::endl;
 #endif
         }