re-wrote cleanupDirectory() function. As this a cause of the slow startup.

Startup time is massively improved!

Please test on Windows!



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6757 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/cachestrapper.cc

@@ -961,8 +961,8 @@ bool CacheStrapper::loadList(std::list<RsItem *>& load)
 	//mPeerMgr->getOwnNetStatus(ownState);
 	//std::string ownName = ownState.name+" ("+ownState.location+")";
 
-	std::map<std::string, std::list<std::string> > saveFiles;
+	std::map<std::string, std::set<std::string> > saveFiles;
-	std::map<std::string, std::list<std::string> >::iterator sit;
+	std::map<std::string, std::set<std::string> >::iterator sit;
 
 	for(it = load.begin(); it != load.end(); it++)
 	{
@@ -1001,7 +1001,7 @@ bool CacheStrapper::loadList(std::list<RsItem *>& load)
 			cd.recvd = rscc->recvd;
 
 			/* store files that we want to keep */
-			(saveFiles[cd.path]).push_back(cd.name);
+			(saveFiles[cd.path]).insert(cd.name);
 
 			std::map<uint16_t, CachePair>::iterator it2;
 			if (caches.end() == (it2 = caches.find(cd.cid.type)))
@@ -1107,7 +1107,7 @@ bool CacheStrapper::loadList(std::list<RsItem *>& load)
 	}
 #endif
 
-	std::list<std::string> emptyList;
+	std::set<std::string> emptySet;
 	for(dit = cacheDirs.begin(); dit != cacheDirs.end(); dit++)
 	{
 #ifdef CS_DEBUG 
@@ -1122,14 +1122,14 @@ bool CacheStrapper::loadList(std::list<RsItem *>& load)
 				std::cerr << "CacheStrapper::loadList() Keeping File: " << *fit << std::endl;
 			}
 #endif
-			RsDirUtil::cleanupDirectory(*dit, sit->second);
+			RsDirUtil::cleanupDirectoryFaster(*dit, sit->second);
 		}
 		else
 		{
 #ifdef CS_DEBUG 
 			std::cerr << "CacheStrapper::loadList() No Files to save here!" << std::endl;
 #endif
-			RsDirUtil::cleanupDirectory(*dit, emptyList);
+			RsDirUtil::cleanupDirectoryFaster(*dit, emptySet);
 		}
 	}
 

libretroshare/src/util/rsdir.cc

@@ -487,7 +487,8 @@ bool	RsDirUtil::checkCreateDirectory(const std::string& dir)
 	return 1;
 }
 
-bool 	RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::list<std::string> &keepFiles)
+
+bool 	RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::set<std::string> &keepFiles)
 {
 
 	/* check for the dir existance */
@@ -499,7 +500,6 @@ bool 	RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::list<s
 	DIR *dir = opendir(cleandir.c_str());
 #endif
 
-	std::list<std::string>::const_iterator it;
 
 	if (!dir)
 	{
@@ -541,7 +541,7 @@ bool 	RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::list<s
 				librs::util::ConvertUtf16ToUtf8(wfname, fname);
 #endif
 				/* check if we should keep it */
-				if (keepFiles.end() == (it = std::find(keepFiles.begin(), keepFiles.end(), fname)))
+				if (keepFiles.end() == std::find(keepFiles.begin(), keepFiles.end(), fname))
 				{
 					/* can remove */
 #ifdef WINDOWS_SYS
@@ -564,6 +564,126 @@ bool 	RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::list<s
 	return true;
 }
 
+
+
+/* faster cleanup - first construct two sets - then iterate over together */
+bool 	RsDirUtil::cleanupDirectoryFaster(const std::string& cleandir, const std::set<std::string> &keepFiles)
+{
+
+	/* check for the dir existance */
+#ifdef WINDOWS_SYS
+	std::map<std::string, std::wstring> fileMap;
+	std::map<std::string, std::wstring>::const_iterator fit;
+
+	std::wstring wcleandir;
+	librs::util::ConvertUtf8ToUtf16(cleandir, wcleandir);
+	_WDIR *dir = _wopendir(wcleandir.c_str());
+#else
+	std::map<std::string, std::string> fileMap;
+	std::map<std::string, std::string>::const_iterator fit;
+
+	DIR *dir = opendir(cleandir.c_str());
+#endif
+
+	if (!dir)
+	{
+		return false;
+	}
+
+#ifdef WINDOWS_SYS
+	struct _wdirent *dent;
+	struct _stat buf;
+
+	while(NULL != (dent = _wreaddir(dir)))
+	{
+		const std::wstring &wfname = dent -> d_name;
+		std::wstring wfullname = wcleandir + L"/" + wfname;
+
+		if (-1 != _wstat(wfullname.c_str(), &buf)) 
+		{ 
+			/* only worry about files */
+			if (S_ISREG(buf.st_mode))
+			{
+				std::string fname;
+				librs::util::ConvertUtf16ToUtf8(wfname, fname);
+				fileMap[fname] = wfullname;
+			}
+		}
+	}
+#else
+	struct dirent *dent;
+	struct stat buf;
+
+	while(NULL != (dent = readdir(dir)))
+	{
+		const std::string &fname = dent -> d_name;
+		std::string fullname = cleandir + "/" + fname;
+
+		if (-1 != stat(fullname.c_str(), &buf))
+		{ 
+			/* only worry about files */
+			if (S_ISREG(buf.st_mode))
+			{
+				fileMap[fname] = fullname;
+			}
+		}
+	}
+#endif
+
+
+
+	std::set<std::string>::const_iterator kit;
+
+	fit = fileMap.begin();
+	kit = keepFiles.begin();
+
+	while(fit != fileMap.end() && kit != keepFiles.end())
+	{
+		if (fit->first < *kit)  // fit is not in keep list;
+		{
+#ifdef WINDOWS_SYS
+			_wremove(fit->second..c_str());
+#else
+			remove(fit->second.c_str());
+#endif
+			++fit;
+		}
+		else if (*kit < fit->first) // keepitem doesn't exist.
+		{
+			++kit;
+		}
+		else // in keep list.
+		{
+			++fit;
+			++kit;
+		}
+	}
+
+	// cleanup extra that aren't in keep list.
+	while(fit != fileMap.end())
+	{
+#ifdef WINDOWS_SYS
+		_wremove(fit->second..c_str());
+#else
+		remove(fit->second.c_str());
+#endif
+		++fit;
+	}
+
+	/* close directory */
+#ifdef WINDOWS_SYS
+	_wclosedir(dir);
+#else
+	closedir(dir);
+#endif
+
+	return true;
+}
+
+
+
+
+
 /* slightly nicer helper function */
 bool RsDirUtil::hashFile(const std::string& filepath, 
 		std::string &name, std::string &hash, uint64_t &size)

libretroshare/src/util/rsdir.h

@@ -30,6 +30,7 @@
 
 #include <string>
 #include <list>
+#include <set>
 #include <stdint.h>
 
 class RsThread;
@@ -85,7 +86,9 @@ bool 		fileExists(const std::string& file);
 bool    	checkFile(const std::string& filename,bool disallow_empty_file = false);
 bool    	checkDirectory(const std::string& dir);
 bool    	checkCreateDirectory(const std::string& dir);
-bool    	cleanupDirectory(const std::string& dir, const std::list<std::string> &keepFiles);
+
+bool    	cleanupDirectory(const std::string& dir, const std::set<std::string> &keepFiles);
+bool    	cleanupDirectoryFaster(const std::string& dir, const std::set<std::string> &keepFiles);
 
 bool 		hashFile(const std::string& filepath,   std::string &name, std::string &hash, uint64_t &size);
 bool 		getFileHash(const std::string& filepath,std::string &hash, uint64_t &size, RsThread *thread = NULL);