Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

improved FolderIterator class

Browse Source
This commit is contained in:
mr-alice 2016-07-23 22:14:43 -04:00
parent 3c976bb7ee
commit 3e48b0fd95
11 changed files with 183 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libretroshare/src/dbase/fimonitor.cc
View File

@ -789,7 +789,7 @@ void FileIndexMonitor::updateCycle()
to_hash.back().realpath = realpath ;
to_hash.back().dirpath = dirpath ;
while(isRunning() && dirIt.readdir())
for(;dirIt.isValid() && isRunning();dirIt.next())
{
/* check entry type */
std::string fname;

9
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -5,6 +5,9 @@ class InternalFileHierarchyStorage
class FileStorageNode
{
public:
static const uint32_t TYPE_FILE = 0x0001 ;
static const uint32_t TYPE_DIR = 0x0002 ;
virtual uint32_t type() const =0;
};
class FileEntry: public FileStorageNode
@ -19,12 +22,12 @@ class InternalFileHierarchyStorage
public:
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
std::set<EntryIndex> subdirs ;
std::set<EntryIndex> subfiles ;
std::set<DirectoryStorage::EntryIndex> subdirs ;
std::set<DirectoryStorage::EntryIndex> subfiles ;
};
// file/dir access and modification
bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
uint32_t root ;

29
libretroshare/src/file_sharing/directory_storage.h
View File

@ -39,27 +39,44 @@ class DirectoryStorage
{
public:
DirIterator(const DirIterator& d) ;
DirIterator(const EntryIndex& d) ;
DirIterator& operator++() ;
EntryIndex operator*() const ; // current directory entry
bool operator()() const ; // used in for loops. Returns true when the iterator is valid.
};
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the directory that is pointed by the iterator
const std::string& name() const ;
};
class FileIterator
{
public:
FileIterator(DirIterator& d); // crawls all files in specified directory
FileIterator(EntryIndex& e); // crawls all files in specified directory
uint32_t size() const ; // number of files in this directory
FileIterator& operator++() ;
EntryIndex operator*() const ; // current file entry
bool operator()() const ; // used in for loops. Returns true when the iterator is valid.
};
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the file that is pointed by the iterator
std::string name() const ;
std::string fullpath() const ;
uint64_t size() const ;
RsFileHash hash() const ;
time_t modtime() const ;
};
virtual DirIterator root() ; // returns the index of the root directory entry.
private:
void updateSubDirectoryList(const EntryIndex& indx,const std::list<std::string>& subdirs) ;
void updateSubFilesList(const EntryIndex& indx,const std::list<std::string>& subfiles) ;
void removeDirectory(const EntryIndex& indx) ;
private:
void load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;

58
libretroshare/src/file_sharing/directory_updater.cc
View File

@ -1,19 +1,24 @@
#include "util/folderiterator.h"
#include "directory_storage.h"
#include "directory_updater.h"
#define DEBUG_LOCAL_DIR_UPDATER 1
void RemoteDirectoryUpdater::tick()
{
// use the stored iterator
}
LocalDirectoryUpdater::LocalDirectoryUpdater()
LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc)
: mHashCache(hc)
{
// tell the hash cache that we're the client
mHashCache->setClient(this) ;
}
void LocalDirectoryUpdater::tick()
{
std::cerr << "LocalDirectoryUpdater::tick()" << std::endl;
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
@ -21,28 +26,29 @@ void LocalDirectoryUpdater::tick()
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
std::list<std::string> sub_dir_list ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it)
sub_dir_list.push_back( (*it).filename ) ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
sub_dir_list.push_back( (*real_dir_it).filename ) ;
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ;
// now for each of them, go recursively and match both files and dirs
DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ;
DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it)
recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it)
recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ;
}
LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx)
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx)
{
std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl;
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
librs::util::FolderIterator dirIt(cumulated_path);
@ -60,22 +66,18 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c
while(dirIt.readdir())
{
std::string name ;
uint64_t size ;
uint8_t type ;
time_t mod_time ;
dirIt.readEntryInformation(type,name,size,mod_time) ;
switch(type)
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ;
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ;
case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ;
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ;
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl;
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update file and dir lists for current directory.
@ -87,18 +89,16 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c
for(DirectoryStorage::FileIterator dit(indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, the callback will update it.
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
if()
if(mHashCache.requestHash(realpath,name,size,modtime,hash),this)
mSharedDirectories->updateFileHash(*dit,hash) ;
mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ;
}
// go through the list of sub-dirs and recursively update
DirectoryStorage::DirIterator stored_dir_it(indx) ;
for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it)
for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
}

9
libretroshare/src/file_sharing/directory_updater.h
View File

@ -3,7 +3,8 @@
// - local: directories are crawled n disk and files are hashed / requested from a cache
// - remote: directories are requested remotely to a providing client
//
class HashCache ;
#include "file_sharing/hash_cache.h"
class LocalDirectoryStorage ;
class DirectoryUpdater
@ -19,12 +20,16 @@ class DirectoryUpdater
//
};
class LocalDirectoryUpdater: public DirectoryUpdater
class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient
{
public:
LocalDirectoryUpdater(HashCache *hash_cache) ;
virtual void tick() ;
protected:
virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ;
void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx);
private:
HashCache *mHashCache ;
LocalDirectoryStorage *mSharedDirectories ;

30
libretroshare/src/file_sharing/hash_cache.h
View File

@ -2,14 +2,28 @@
#include <map>
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
class HashCacheClient
{
public:
virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ;
};
class FileHashingThread: public RsTickingThread
{
public:
FileHashingThread() {}
virtual void data_tick() ;
};
class HashCache
{
public:
HashCache(const std::string& save_file_name) ;
void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ;
struct HashCacheInfo
{
@ -21,10 +35,10 @@ public:
} ;
// interaction with GUI, called from p3FileLists
void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
void clear() { _files.clear(); }
bool empty() const { return _files.empty() ; }
void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; }
uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; }
void clear() { mFiles.clear(); }
bool empty() const { return mFiles.empty() ; }
private:
void clean() ;
@ -33,7 +47,7 @@ private:
void load() ;
// threaded stuff
RsTickingThread mHashingThread ;
FileHashingThread mHashingThread ;
RsMutex mHashMtx ;
// Local configuration and storage
@ -45,6 +59,6 @@ private:
// current work
std::map<std::string> mFilesToHash ;
std::map<std::string,HashCacheClient *> mFilesToHash ;
};

2
libretroshare/src/plugins/pluginmanager.cc
View File

@ -146,7 +146,7 @@ void RsPluginManager::loadPlugins(const std::vector<std::string>& plugin_directo
continue ;
}
while(dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
std::string fname;
dirIt.d_name(fname);

2
libretroshare/src/rsserver/rsaccounts.cc
View File

@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map<RsPeerId, AccountDetails> &
struct stat64 buf;
while (dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
/* check entry type */
std::string fname;

79
libretroshare/src/util/folderiterator.cc
View File

@ -1,11 +1,17 @@
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include "folderiterator.h"
#include "rsstring.h"
namespace librs { namespace util {
FolderIterator::FolderIterator(const std::string& folderName)
: mFolderName(folderName)
{
#ifdef WINDOWS_SYS
std::wstring utf16Name;
@ -18,10 +24,10 @@ FolderIterator::FolderIterator(const std::string& folderName)
handle = FindFirstFileW(utf16Name.c_str(), &fileInfo);
validity = handle != INVALID_HANDLE_VALUE;
isFirstCall = true;
#else
handle = opendir(folderName.c_str());
validity = handle != NULL;
next();
#endif
}
@ -30,19 +36,71 @@ FolderIterator::~FolderIterator()
closedir();
}
bool FolderIterator::readdir() {
void FolderIterator::next()
{
do {
if(!readdir())
{
validity = false ;
break ;
}
d_name(mFileName);
} while(mFileName == "." || mFileName == "..") ;
mFullPath = mFolderName + "/" + mFileName ;
struct stat64 buf ;
#ifdef WINDOWS_SYS
std::wstring wfullname;
librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname);
if ( 0 == _wstati64(wfullname.c_str(), &buf))
#else
if ( 0 == stat64(mFullPath.c_str(), &buf))
#endif
{
mFileModTime = buf.st_mtime ;
mStatInfoOk = true;
if (S_ISDIR(buf.st_mode))
{
mType = TYPE_DIR ;
mFileSize = 0 ;
mFileModTime = buf.st_mtime;
}
else if (S_ISREG(buf.st_mode))
{
mType = TYPE_FILE ;
mFileSize = buf.st_size;
mFileModTime = buf.st_mtime;
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
}
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
validity = false ;
}
}
bool FolderIterator::readdir()
{
if(!validity)
return false;
#ifdef WINDOWS_SYS
if(isFirstCall) {
isFirstCall = false;
return true;
}
return FindNextFileW(handle, &fileInfo) != 0;
#else
ent = ::readdir(handle);
return ent != 0;
return ent != NULL;
#endif
}
@ -65,6 +123,11 @@ bool FolderIterator::d_name(std::string& dest)
return true;
}
const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
const std::string& FolderIterator::file_name() { return mFileName ; }
uint64_t FolderIterator::file_size() { return mFileSize ; }
time_t FolderIterator::file_modtime() { return mFileModTime ; }
bool FolderIterator::closedir()
{
if(!validity)

23
libretroshare/src/util/folderiterator.h
View File

@ -2,6 +2,7 @@
#define FOLDERITERATOR_H
#include <stdint.h>
#include <iostream>
#include <cstdio>
@ -24,14 +25,24 @@ public:
FolderIterator(const std::string& folderName);
~FolderIterator();
bool isValid() const { return validity; }
enum { TYPE_UNKNOWN = 0x00,
TYPE_FILE = 0x01,
TYPE_DIR = 0x02
};
bool isValid() const { return validity; }
bool readdir();
void next();
bool d_name(std::string& dest);
bool closedir();
const std::string& file_name() ;
const std::string& file_fullpath() ;
uint64_t file_size() ;
uint64_t file_type() ;
time_t file_modtime() ;
private:
bool validity;
@ -43,7 +54,15 @@ private:
DIR* handle;
struct dirent* ent;
#endif
void updateStatsInfo() ;
bool mStatInfoOk ;
time_t mFileModTime ;
uint64_t mFileSize ;
uint8_t mType ;
std::string mFileName ;
std::string mFullPath ;
std::string mFolderName ;
};

2
retroshare-nogui/src/introserver.cc
View File

@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts()
mCertCheckTime = now;
struct stat64 buf;
while(dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
/* check entry type */
std::string fname;