improved FolderIterator class

libretroshare/src/dbase/fimonitor.cc

@@ -789,7 +789,7 @@ void 	FileIndexMonitor::updateCycle()
 		to_hash.back().realpath = realpath ;
 		to_hash.back().dirpath = dirpath ;
 
-		while(isRunning() && dirIt.readdir())
+        for(;dirIt.isValid() && isRunning();dirIt.next())
 		{
 			/* check entry type */
 			std::string fname;

libretroshare/src/file_sharing/directory_storage.cc

@@ -5,6 +5,9 @@ class InternalFileHierarchyStorage
     class FileStorageNode
     {
     public:
+        static const uint32_t TYPE_FILE = 0x0001 ;
+        static const uint32_t TYPE_DIR  = 0x0002 ;
+
         virtual uint32_t type() const =0;
     };
     class FileEntry: public FileStorageNode
@@ -19,12 +22,12 @@ class InternalFileHierarchyStorage
     public:
         virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
 
-        std::set<EntryIndex> subdirs ;
+        std::set<DirectoryStorage::EntryIndex> subdirs ;
-        std::set<EntryIndex> subfiles ;
+        std::set<DirectoryStorage::EntryIndex> subfiles ;
     };
 
     // file/dir access and modification
-    bool findSubDirectory(EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
+    bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
 
     uint32_t root ;
 

libretroshare/src/file_sharing/directory_storage.h

@@ -39,27 +39,44 @@ class DirectoryStorage
 		{
 			public:
                 DirIterator(const DirIterator& d) ;
+                DirIterator(const EntryIndex& d) ;
 
 				DirIterator& operator++() ;
 				EntryIndex operator*() const ;		// current directory entry
 
-                bool operator()() const ;			// used in for loops. Returns true when the iterator is valid.
+                operator bool() const ;			// used in for loops. Returns true when the iterator is valid.
-		};
+
+                // info about the directory that is pointed by the iterator
+
+                const std::string& name() const ;
+        };
 		class FileIterator
 		{
 			public:
                 FileIterator(DirIterator& d);		// crawls all files in specified directory
+                FileIterator(EntryIndex& e);		// crawls all files in specified directory
 
-                uint32_t size() const ;				// number of files in this directory
 				FileIterator& operator++() ;
 				EntryIndex operator*() const ;		// current file entry
 
-                bool operator()() const ;			// used in for loops. Returns true when the iterator is valid.
+                operator bool() const ;			// used in for loops. Returns true when the iterator is valid.
-		};
+
+                // info about the file that is pointed by the iterator
+
+                std::string name() const ;
+                std::string fullpath() const ;
+                uint64_t size() const ;
+                RsFileHash hash() const ;
+                time_t modtime() const ;
+        };
 
         virtual DirIterator root() ;					// returns the index of the root directory entry.
 
-	private:
+        void updateSubDirectoryList(const EntryIndex& indx,const std::list<std::string>& subdirs) ;
+        void updateSubFilesList(const EntryIndex& indx,const std::list<std::string>& subfiles) ;
+        void removeDirectory(const EntryIndex& indx) ;
+
+    private:
 		void load(const std::string& local_file_name) ;
 		void save(const std::string& local_file_name) ;
 

libretroshare/src/file_sharing/directory_updater.cc

@@ -1,19 +1,24 @@
+#include "util/folderiterator.h"
+
 #include "directory_storage.h"
 #include "directory_updater.h"
 
+#define DEBUG_LOCAL_DIR_UPDATER 1
+
 void RemoteDirectoryUpdater::tick()
 {
 	// use the stored iterator
 }
 
-LocalDirectoryUpdater::LocalDirectoryUpdater()
+LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc)
+    : mHashCache(hc)
 {
-    // tell the hash cache that we're the client
-    mHashCache->setClient(this) ;
 }
 
 void LocalDirectoryUpdater::tick()
 {
+    std::cerr << "LocalDirectoryUpdater::tick()" << std::endl;
+
 	// recursive update algorithm works that way:
 	// 	- the external loop starts on the shared directory list and goes through sub-directories
     // 	- at the same time, it updates the local list of shared directories.  A single sweep is performed over the whole directory structure.
@@ -21,28 +26,29 @@ void LocalDirectoryUpdater::tick()
 	// 	- doing so, changing directory names or moving files between directories does not cause a re-hash of the content. 
 	//
     std::list<SharedDirInfo> shared_directory_list ;
+    std::list<std::string> sub_dir_list ;
 
-    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it)
+    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
-        sub_dir_list.push_back( (*it).filename ) ;
+        sub_dir_list.push_back( (*real_dir_it).filename ) ;
 
     // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
 
-    mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
+    mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ;
 
     // now for each of them, go recursively and match both files and dirs
 
-    DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ;
+    DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ;
 
-    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it)
+    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it)
-        recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
+        recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ;
 }
 
-LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx)
+void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx)
 {
+    std::cerr << "  parsing directory " << cumulated_path << ", index=" << indx << std::endl;
+
     // make sure list of subdirs is the same
-
     // make sure list of subfiles is the same
-
     // request all hashes to the hashcache
 
     librs::util::FolderIterator dirIt(cumulated_path);
@@ -60,22 +66,18 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c
 
     while(dirIt.readdir())
     {
-        std::string name ;
+        switch(dirIt.file_type())
-        uint64_t size ;
-        uint8_t type ;
-        time_t mod_time ;
-
-        dirIt.readEntryInformation(type,name,size,mod_time) ;
-
-        switch(type)
         {
-                case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ;
+                case librs::util::FolderIterator::TYPE_FILE:	subfiles.push_back(dirIt.file_name()) ;
-                                                        break;
+                                                                std::cerr << "  adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
-                case librs::util::FolderIterator::TYPE_DIR:  subdirs.push_back(name) ;
+                                                            break;
+
+                case librs::util::FolderIterator::TYPE_DIR:	subdirs.push_back(dirIt.file_name()) ;
+                                                                std::cerr << "  adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
                                                             break;
 
         default:
-            std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl;
+            std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
         }
     }
     // update file and dir lists for current directory.
@@ -87,18 +89,16 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c
 
     for(DirectoryStorage::FileIterator dit(indx);dit;++dit)
     {
-        // ask about the hash. If not present, ask HashCache. If not present, the callback will update it.
+        // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
 
-        if()
+        mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ;
-            if(mHashCache.requestHash(realpath,name,size,modtime,hash),this)
-                mSharedDirectories->updateFileHash(*dit,hash) ;
     }
 
     // go through the list of sub-dirs and recursively update
 
     DirectoryStorage::DirIterator stored_dir_it(indx) ;
 
-    for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it)
+    for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
         recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
 }
 

libretroshare/src/file_sharing/directory_updater.h

@@ -3,7 +3,8 @@
 // 	- local: directories are crawled n disk and files are hashed / requested from a cache
 // 	- remote: directories are requested remotely to a providing client
 //
-class HashCache ;
+#include "file_sharing/hash_cache.h"
+
 class LocalDirectoryStorage ;
 
 class DirectoryUpdater
@@ -19,12 +20,16 @@ class DirectoryUpdater
 		// 
 };
 
-class LocalDirectoryUpdater: public DirectoryUpdater
+class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient
 {
 public:
     LocalDirectoryUpdater(HashCache *hash_cache) ;
     virtual void tick() ;
 
+protected:
+    virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ;
+    void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx);
+
 private:
     HashCache *mHashCache ;
     LocalDirectoryStorage *mSharedDirectories ;

libretroshare/src/file_sharing/hash_cache.h

@@ -2,14 +2,28 @@
 
 #include <map>
 #include "util/rsthreads.h"
+#include "retroshare/rsfiles.h"
+
+class HashCacheClient
+{
+public:
+    virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ;
+};
+
+class FileHashingThread: public RsTickingThread
+{
+public:
+    FileHashingThread() {}
+
+    virtual void data_tick() ;
+};
 
 class HashCache
 {
 public:
     HashCache(const std::string& save_file_name) ;
 
-    void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
+    bool requestHash(const  std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ;
-    bool find(const  std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
 
     struct HashCacheInfo
     {
@@ -21,10 +35,10 @@ public:
     } ;
 
     // interaction with GUI, called from p3FileLists
-    void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
+    void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; }
-    uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
+    uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; }
-    void clear() { _files.clear(); }
+    void clear() { mFiles.clear(); }
-    bool empty() const { return _files.empty() ; }
+    bool empty() const { return mFiles.empty() ; }
 
 private:
     void clean() ;
@@ -33,7 +47,7 @@ private:
     void load() ;
 
     // threaded stuff
-    RsTickingThread mHashingThread ;
+    FileHashingThread mHashingThread ;
     RsMutex mHashMtx ;
 
     // Local configuration and storage
@@ -45,6 +59,6 @@ private:
 
     // current work
 
-    std::map<std::string> mFilesToHash ;
+    std::map<std::string,HashCacheClient *> mFilesToHash ;
 };
 

libretroshare/src/plugins/pluginmanager.cc

@@ -146,7 +146,7 @@ void RsPluginManager::loadPlugins(const std::vector<std::string>& plugin_directo
 			continue ;
 		}
 
-		while(dirIt.readdir())
+        for(;dirIt.isValid();dirIt.next())
 		{
 			std::string fname;
 			dirIt.d_name(fname);

libretroshare/src/rsserver/rsaccounts.cc

@@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map<RsPeerId, AccountDetails> &
 
 	struct stat64 buf;
 
-	while (dirIt.readdir())
+    for(;dirIt.isValid();dirIt.next())
 	{
 		/* check entry type */
 		std::string fname;

libretroshare/src/util/folderiterator.cc

@@ -1,11 +1,17 @@
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
 #include "folderiterator.h"
 #include "rsstring.h"
 
-
 namespace librs { namespace util {
 
 
 FolderIterator::FolderIterator(const std::string& folderName)
+    : mFolderName(folderName)
 {
 #ifdef WINDOWS_SYS
     std::wstring utf16Name;
@@ -18,10 +24,10 @@ FolderIterator::FolderIterator(const std::string& folderName)
 
     handle = FindFirstFileW(utf16Name.c_str(), &fileInfo);
     validity = handle != INVALID_HANDLE_VALUE;
-    isFirstCall = true;
 #else
     handle = opendir(folderName.c_str());
     validity = handle != NULL;
+    next();
 #endif
 }
 
@@ -30,19 +36,71 @@ FolderIterator::~FolderIterator()
     closedir();
 }
 
-bool FolderIterator::readdir() {
+void FolderIterator::next()
+{
+    do {
+        if(!readdir())
+        {
+            validity = false ;
+            break ;
+        }
+
+        d_name(mFileName);
+    } while(mFileName == "." || mFileName == "..") ;
+
+    mFullPath = mFolderName + "/" + mFileName ;
+
+    struct stat64 buf ;
+
+#ifdef WINDOWS_SYS
+    std::wstring wfullname;
+    librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname);
+    if ( 0 == _wstati64(wfullname.c_str(), &buf))
+#else
+    if ( 0 == stat64(mFullPath.c_str(), &buf))
+#endif
+    {
+        mFileModTime = buf.st_mtime ;
+        mStatInfoOk = true;
+
+        if (S_ISDIR(buf.st_mode))
+        {
+            mType = TYPE_DIR ;
+            mFileSize = 0 ;
+            mFileModTime = buf.st_mtime;
+        }
+        else if (S_ISREG(buf.st_mode))
+        {
+            mType = TYPE_FILE ;
+            mFileSize = buf.st_size;
+            mFileModTime = buf.st_mtime;
+        }
+        else
+        {
+            mType = TYPE_UNKNOWN ;
+            mFileSize = 0 ;
+            mFileModTime = 0;
+        }
+    }
+    else
+    {
+            mType = TYPE_UNKNOWN ;
+            mFileSize = 0 ;
+            mFileModTime = 0;
+            validity = false ;
+    }
+}
+
+bool FolderIterator::readdir()
+{
     if(!validity)
         return false;
 
 #ifdef WINDOWS_SYS
-    if(isFirstCall) {
-        isFirstCall = false;
-        return true;
-    }
     return FindNextFileW(handle, &fileInfo) != 0;
 #else
     ent = ::readdir(handle);
-    return ent != 0;
+    return ent != NULL;
 #endif
 }
 
@@ -65,6 +123,11 @@ bool FolderIterator::d_name(std::string& dest)
     return true;
 }
 
+const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
+const std::string& FolderIterator::file_name() { return mFileName ; }
+uint64_t     FolderIterator::file_size()       { return mFileSize ; }
+time_t       FolderIterator::file_modtime()    { return mFileModTime ; }
+
 bool FolderIterator::closedir()
 {
     if(!validity)

libretroshare/src/util/folderiterator.h

@@ -2,6 +2,7 @@
 #define FOLDERITERATOR_H
 
 
+#include <stdint.h>
 #include <iostream>
 #include <cstdio>
 
@@ -24,14 +25,24 @@ public:
     FolderIterator(const std::string& folderName);
     ~FolderIterator();
 
-    bool isValid() const    { return validity; }
+    enum { TYPE_UNKNOWN = 0x00,
+           TYPE_FILE    = 0x01,
+           TYPE_DIR     = 0x02
+         };
 
+    bool isValid() const    { return validity; }
     bool readdir();
+    void next();
 
     bool d_name(std::string& dest);
-
     bool closedir();
 
+    const std::string& file_name() ;
+    const std::string& file_fullpath() ;
+    uint64_t file_size() ;
+    uint64_t file_type() ;
+    time_t   file_modtime() ;
+
 private:
     bool validity;
 
@@ -43,7 +54,15 @@ private:
     DIR* handle;
     struct dirent* ent;
 #endif
+    void updateStatsInfo() ;
 
+    bool mStatInfoOk ;
+    time_t mFileModTime ;
+    uint64_t mFileSize ;
+    uint8_t mType ;
+    std::string mFileName ;
+    std::string mFullPath ;
+    std::string mFolderName ;
 };
 
 

retroshare-nogui/src/introserver.cc

@@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts()
 	mCertCheckTime = now;
         struct stat64 buf;
 
-	while(dirIt.readdir())
+    for(;dirIt.isValid();dirIt.next())
 	{
 		/* check entry type */
 		std::string fname;