added iterators for directory storage class

This commit is contained in:
mr-alice 2016-07-24 23:48:22 -04:00
parent 3e48b0fd95
commit 25c824fd5f
6 changed files with 320 additions and 38 deletions

View File

@ -1,5 +1,22 @@
#include <set>
#include "directory_storage.h"
/******************************************************************************************************************/
/* Internal File Hierarchy Storage */
/******************************************************************************************************************/
template<class T> typename std::set<T>::iterator erase_from_set(typename std::set<T>& s,const typename std::set<T>::iterator& it)
{
typename std::set<T>::iterator tmp(it);
++tmp;
s.erase(it) ;
return tmp;
}
// This class handles the file hierarchy
// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed,
// so that the hierarchy stays completely coherent between calls.
class InternalFileHierarchyStorage
{
class FileStorageNode
@ -8,36 +25,279 @@ class InternalFileHierarchyStorage
static const uint32_t TYPE_FILE = 0x0001 ;
static const uint32_t TYPE_DIR = 0x0002 ;
virtual ~FileStorageNode() {}
virtual uint32_t type() const =0;
};
class FileEntry: public FileStorageNode
{
public:
FileEntry(const std::string& name) : file_name(name) {}
virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
virtual ~FileEntry() {}
// local stuff
std::string file_name ;
uint64_t file_size ;
RsFileHash file_hash ;
time_t file_modtime ;
};
class DirEntry: public FileStorageNode
{
public:
DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {}
virtual ~DirEntry() {}
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
std::set<DirectoryStorage::EntryIndex> subdirs ;
std::set<DirectoryStorage::EntryIndex> subfiles ;
// local stuff
std::string dir_name ;
DirectoryStorage::EntryIndex parent_index;
std::vector<DirectoryStorage::EntryIndex> subdirs ;
std::vector<DirectoryStorage::EntryIndex> subfiles ;
};
// class stuff
InternalFileHierarchyStorage() : mRoot(0)
{
mNodes.push_back(new DirEntry("",0)) ;
}
// high level modification routines
bool isIndexValid(DirectoryStorage::EntryIndex e)
{
return e < mNodes.size() && mNodes[e] != NULL ;
}
bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set<std::string>& subdirs)
{
if(indx >= mNodes.size() || mNodes[indx] == NULL)
return nodeAccessError("updateSubDirectoryList(): Node does not exist") ;
if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ;
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
std::set<std::string> should_create(subdirs);
for(uint32_t i=0;i<d.subdirs.size();)
if(subdirs.find(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) == subdirs.end())
removeDirectory(d.subdirs[i]) ;
else
{
should_create.erase(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) ;
++i;
}
for(std::set<std::string>::const_iterator it(should_create.begin());it!=should_create.end();++it)
{
d.subdirs.push_back(mNodes.size()) ;
mNodes.push_back(new DirEntry(*it,indx));
}
return true;
}
bool removeDirectory(const DirectoryStorage::EntryIndex& indx)
{
// check that it's a directory
if(indx >= mNodes.size() || mNodes[indx] == NULL)
return nodeAccessError("removeDirectory(): Node does not exist") ;
if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
return nodeAccessError("removeDirectory(): Node is not a directory") ;
if(indx == 0)
return nodeAccessError("removeDirectory(): Cannot remove top level directory") ;
// remove from parent
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
DirEntry& parent_dir(*static_cast<DirEntry*>(mNodes[d.parent_index]));
for(uint32_t i=0;i<parent_dir.subdirs.size();++i)
if(parent_dir.subdirs[i] == indx)
return recursRemoveDirectory(indx) ;
return nodeAccessError("removeDirectory(): inconsistency!!") ;
}
bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files)
{
if(indx >= mNodes.size() || mNodes[indx] == NULL)
return nodeAccessError("updateSubFilesList(): Node does not exist") ;
if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
return nodeAccessError("updateSubFilesList(): Node is not a directory") ;
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
new_files = subfiles ;
for(uint32_t i=0;i<d.subfiles.size();)
if(subfiles.find(static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_name) == subfiles.end())
{
d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ;
d.subfiles.pop_back();
}
else
{
new_files.erase(static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_name) ;
++i;
}
for(std::set<std::string>::const_iterator it(new_files.begin());it!=new_files.end();++it)
{
d.subfiles.push_back(mNodes.size()) ;
mNodes.push_back(new FileEntry(*it));
}
return true;
}
void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ;
// file/dir access and modification
bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
uint32_t root ;
uint32_t mRoot ;
std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
void compress() ; // use empty space in the vector, mostly due to deleted entries.
void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
// all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
// are not referenced.
friend class DirectoryStorage ; // only class that can use this.
// low level stuff. Should normally not be used externally.
DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index)
{
if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR)
return DirectoryStorage::NO_INDEX;
if(static_cast<DirEntry*>(mNodes[parent_index])->subfiles.size() <= file_tab_index)
return DirectoryStorage::NO_INDEX;
return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[file_tab_index];
}
DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index)
{
if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR)
return DirectoryStorage::NO_INDEX;
if(static_cast<DirEntry*>(mNodes[parent_index])->subdirs.size() <= dir_tab_index)
return DirectoryStorage::NO_INDEX;
return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[dir_tab_index];
}
bool check() // checks consistency of storage.
{
return true;
}
void print()
{
}
private:
bool nodeAccessError(const std::string& s)
{
std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
return false ;
}
// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
// phase. That allows to keep file information when moving them around.
bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir)
{
DirEntry& d(*static_cast<DirEntry*>(mNodes[dir])) ;
for(uint32_t i=0;i<d.subdirs.size();++i)
recursRemoveDirectory(d.subdirs[i]);
delete mNodes[dir] ;
mNodes[dir] = NULL ;
return true ;
}
};
/******************************************************************************************************************/
/* Iterators */
/******************************************************************************************************************/
DirectoryStorage::DirIterator::DirIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mDirTabIndex = 0;
}
DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mFileTabIndex = 0;
}
DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++()
{
++mDirTabIndex ;
return *this;
}
DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
{
++mFileTabIndex ;
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
{
return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
{
return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ;
}
DirectoryStorage::DirIterator::operator bool() const
{
return **this != DirectoryStorage::NO_INDEX;
}
DirectoryStorage::FileIterator::operator bool() const
{
return **this != DirectoryStorage::NO_INDEX;
}
/******************************************************************************************************************/
/* Directory Storage */
/******************************************************************************************************************/
DirectoryStorage::DirIterator DirectoryStorage::root()
{
return DirIterator(this,EntryIndex(0)) ;
}
void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set<std::string>& subdirs)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->updateSubDirectoryList(indx,subdirs) ;
}
void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
}
void DirectoryStorage::removeDirectory(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->removeDirectory(indx);
}
// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ;
// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ;
// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ;

View File

@ -1,3 +1,5 @@
#pragma once
#include <string>
#include <stdint.h>
#include <list>
@ -23,12 +25,13 @@ class DirectoryStorage
virtual ~DirectoryStorage() {}
typedef uint32_t EntryIndex ;
static const EntryIndex NO_INDEX = 0xffffffff;
void save() const ;
virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const;
virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const;
virtual int searchBoolExp(Expression * exp, std::list<EntryIndex> &results) const;
virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const { return 0;}
virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const { return 0;}
virtual int searchBoolExp(Expression * exp, std::list<EntryIndex> &results) const { return 0; }
void getFileDetails(EntryIndex i) ;
@ -39,25 +42,31 @@ class DirectoryStorage
{
public:
DirIterator(const DirIterator& d) ;
DirIterator(const EntryIndex& d) ;
DirIterator(DirectoryStorage *d,EntryIndex i) ;
DirIterator& operator++() ;
EntryIndex operator*() const ; // current directory entry
EntryIndex operator*() const ;
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the directory that is pointed by the iterator
const std::string& name() const ;
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mDirTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
friend class DirectoryStorage ;
};
class FileIterator
{
public:
FileIterator(DirIterator& d); // crawls all files in specified directory
FileIterator(EntryIndex& e); // crawls all files in specified directory
FileIterator(DirIterator& d); // crawls all files in specified directory
FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory
FileIterator& operator++() ;
EntryIndex operator*() const ; // current file entry
EntryIndex operator*() const ; // current file entry
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
@ -68,14 +77,24 @@ class DirectoryStorage
uint64_t size() const ;
RsFileHash hash() const ;
time_t modtime() const ;
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mFileTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
};
virtual DirIterator root() ; // returns the index of the root directory entry.
void updateSubDirectoryList(const EntryIndex& indx,const std::list<std::string>& subdirs) ;
void updateSubFilesList(const EntryIndex& indx,const std::list<std::string>& subfiles) ;
void updateSubDirectoryList(const EntryIndex& indx,const std::set<std::string>& subdirs) ;
void updateSubFilesList(const EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files) ;
void removeDirectory(const EntryIndex& indx) ;
void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ;
void cleanup();
private:
void load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;
@ -86,6 +105,8 @@ class DirectoryStorage
// storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
InternalFileHierarchyStorage *mFileHierarchy ;
RsMutex mDirStorageMtx ;
};
class RemoteDirectoryStorage: public DirectoryStorage
@ -104,15 +125,6 @@ public:
void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
void getSharedDirectoryList(std::list<SharedDirInfo>& lst) ;
/*!
* \brief addFile
* \param dir
* \param hash
* \param modf_time
*/
void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ;
private:
std::list<SharedDirInfo> mLocalDirs ;

View File

@ -26,10 +26,12 @@ void LocalDirectoryUpdater::tick()
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
std::list<std::string> sub_dir_list ;
std::set<std::string> sub_dir_list ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
sub_dir_list.push_back( (*real_dir_it).filename ) ;
sub_dir_list.insert( (*real_dir_it).filename ) ;
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
@ -61,18 +63,18 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
// collect subdirs and subfiles
std::list<std::string> subfiles ;
std::list<std::string> subdirs ;
std::set<std::string> subfiles ;
std::set<std::string> subdirs ;
while(dirIt.readdir())
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ;
case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ;
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ;
case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ;
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
break;
@ -83,7 +85,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
mSharedDirectories->updateSubFilesList(indx,subfiles) ;
std::set<std::string> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
@ -98,7 +102,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
DirectoryStorage::DirIterator stored_dir_it(indx) ;
for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
for(std::set<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
}

View File

@ -4,13 +4,13 @@
// - remote: directories are requested remotely to a providing client
//
#include "file_sharing/hash_cache.h"
class LocalDirectoryStorage ;
#include "file_sharing/directory_storage.h"
class DirectoryUpdater
{
public:
DirectoryUpdater() ;
DirectoryUpdater() {}
virtual ~DirectoryUpdater(){}
// Does some updating job. Crawls the existing directories and checks wether it has been updated
// recently enough. If not, calls the directry source.
@ -24,6 +24,8 @@ class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient
{
public:
LocalDirectoryUpdater(HashCache *hash_cache) ;
virtual ~LocalDirectoryUpdater() {}
virtual void tick() ;
protected:
@ -38,5 +40,8 @@ private:
class RemoteDirectoryUpdater: public DirectoryUpdater
{
public:
RemoteDirectoryUpdater() {}
virtual ~RemoteDirectoryUpdater() {}
virtual void tick() ;
};

View File

@ -124,9 +124,10 @@ bool FolderIterator::d_name(std::string& dest)
}
const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
const std::string& FolderIterator::file_name() { return mFileName ; }
uint64_t FolderIterator::file_size() { return mFileSize ; }
time_t FolderIterator::file_modtime() { return mFileModTime ; }
const std::string& FolderIterator::file_name() { return mFileName ; }
uint64_t FolderIterator::file_size() { return mFileSize ; }
time_t FolderIterator::file_modtime() { return mFileModTime ; }
uint8_t FolderIterator::file_type() { return mType ; }
bool FolderIterator::closedir()
{

View File

@ -40,7 +40,7 @@ public:
const std::string& file_name() ;
const std::string& file_fullpath() ;
uint64_t file_size() ;
uint64_t file_type() ;
uint8_t file_type() ;
time_t file_modtime() ;
private: