diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 0b7e100e4..a955df832 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,22 @@ +#include #include "directory_storage.h" +/******************************************************************************************************************/ +/* Internal File Hierarchy Storage */ +/******************************************************************************************************************/ + +template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) +{ + typename std::set::iterator tmp(it); + ++tmp; + s.erase(it) ; + return tmp; +} + +// This class handles the file hierarchy +// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, +// so that the hierarchy stays completely coherent between calls. + class InternalFileHierarchyStorage { class FileStorageNode @@ -8,36 +25,279 @@ class InternalFileHierarchyStorage static const uint32_t TYPE_FILE = 0x0001 ; static const uint32_t TYPE_DIR = 0x0002 ; + virtual ~FileStorageNode() {} virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode { public: + FileEntry(const std::string& name) : file_name(name) {} virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + virtual ~FileEntry() {} + // local stuff + std::string file_name ; + uint64_t file_size ; + RsFileHash file_hash ; + time_t file_modtime ; }; class DirEntry: public FileStorageNode { public: + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + virtual ~DirEntry() {} + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + // local stuff + std::string dir_name ; + DirectoryStorage::EntryIndex parent_index; + + std::vector subdirs ; + std::vector subfiles ; }; + // class stuff + InternalFileHierarchyStorage() : mRoot(0) + { + mNodes.push_back(new DirEntry("",0)) ; + } + + // high level modification routines + + bool isIndexValid(DirectoryStorage::EntryIndex e) + { + return e < mNodes.size() && mNodes[e] != NULL ; + } + + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubDirectoryList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + + std::set should_create(subdirs); + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) + removeDirectory(d.subdirs[i]) ; + else + { + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; + ++i; + } + + for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) + { + d.subdirs.push_back(mNodes.size()) ; + mNodes.push_back(new DirEntry(*it,indx)); + } + + return true; + } + bool removeDirectory(const DirectoryStorage::EntryIndex& indx) + { + // check that it's a directory + + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("removeDirectory(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("removeDirectory(): Node is not a directory") ; + + if(indx == 0) + return nodeAccessError("removeDirectory(): Cannot remove top level directory") ; + + // remove from parent + + DirEntry& d(*static_cast(mNodes[indx])) ; + DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); + + for(uint32_t i=0;i& subfiles,std::set& new_files) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubFilesList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubFilesList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + new_files = subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name) == subfiles.end()) + { + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; + d.subfiles.pop_back(); + } + else + { + new_files.erase(static_cast(mNodes[d.subfiles[i]])->file_name) ; + ++i; + } + + for(std::set::const_iterator it(new_files.begin());it!=new_files.end();++it) + { + d.subfiles.push_back(mNodes.size()) ; + mNodes.push_back(new FileEntry(*it)); + } + return true; + } + void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ; + // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e - uint32_t root ; + uint32_t mRoot ; std::vector mNodes;// uses pointers to keep information about valid/invalid objects. - void compress() ; // use empty space in the vector, mostly due to deleted entries. + void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to + // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that + // are not referenced. friend class DirectoryStorage ; // only class that can use this. + + // low level stuff. Should normally not be used externally. + + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; + } + DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + } + + bool check() // checks consistency of storage. + { + return true; + } + + void print() + { + } +private: + bool nodeAccessError(const std::string& s) + { + std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; + return false ; + } + + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning + // phase. That allows to keep file information when moving them around. + + bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir) + { + DirEntry& d(*static_cast(mNodes[dir])) ; + + for(uint32_t i=0;imFileHierarchy ; + mParentIndex = i; + mDirTabIndex = 0; +} + +DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i) +{ + mStorage = s->mFileHierarchy ; + mParentIndex = i; + mFileTabIndex = 0; +} + +DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++() +{ + ++mDirTabIndex ; + + return *this; +} +DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++() +{ + ++mFileTabIndex ; + + return *this; +} +DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const +{ + return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; +} +DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const +{ + return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; +} +DirectoryStorage::DirIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} +DirectoryStorage::FileIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} + +/******************************************************************************************************************/ +/* Directory Storage */ +/******************************************************************************************************************/ + +DirectoryStorage::DirIterator DirectoryStorage::root() +{ + return DirIterator(this,EntryIndex(0)) ; +} + +void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; +} +void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; +} +void DirectoryStorage::removeDirectory(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->removeDirectory(indx); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0fb94037a..90a0a889e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -23,12 +25,13 @@ class DirectoryStorage virtual ~DirectoryStorage() {} typedef uint32_t EntryIndex ; + static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const; - virtual int searchHash(const RsFileHash& hash, std::list &results) const; - virtual int searchBoolExp(Expression * exp, std::list &results) const; + virtual int searchTerms(const std::list& terms, std::list &results) const { return 0;} + virtual int searchHash(const RsFileHash& hash, std::list &results) const { return 0;} + virtual int searchBoolExp(Expression * exp, std::list &results) const { return 0; } void getFileDetails(EntryIndex i) ; @@ -39,25 +42,31 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; - DirIterator(const EntryIndex& d) ; + DirIterator(DirectoryStorage *d,EntryIndex i) ; DirIterator& operator++() ; - EntryIndex operator*() const ; // current directory entry + EntryIndex operator*() const ; operator bool() const ; // used in for loops. Returns true when the iterator is valid. // info about the directory that is pointed by the iterator const std::string& name() const ; + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mDirTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; + + friend class DirectoryStorage ; }; class FileIterator { public: - FileIterator(DirIterator& d); // crawls all files in specified directory - FileIterator(EntryIndex& e); // crawls all files in specified directory + FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory FileIterator& operator++() ; - EntryIndex operator*() const ; // current file entry + EntryIndex operator*() const ; // current file entry operator bool() const ; // used in for loops. Returns true when the iterator is valid. @@ -68,14 +77,24 @@ class DirectoryStorage uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; + + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mFileTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; }; virtual DirIterator root() ; // returns the index of the root directory entry. - void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; - void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; void removeDirectory(const EntryIndex& indx) ; + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + + void cleanup(); + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; @@ -86,6 +105,8 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. InternalFileHierarchyStorage *mFileHierarchy ; + + RsMutex mDirStorageMtx ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -104,15 +125,6 @@ public: void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; - /*! - * \brief addFile - * \param dir - * \param hash - * \param modf_time - */ - void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; - private: std::list mLocalDirs ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c88820fb0..c80968d8b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -26,10 +26,12 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - std::list sub_dir_list ; + + + std::set sub_dir_list ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) - sub_dir_list.push_back( (*real_dir_it).filename ) ; + sub_dir_list.insert( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. @@ -61,18 +63,18 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles - std::list subfiles ; - std::list subdirs ; + std::set subfiles ; + std::set subdirs ; while(dirIt.readdir()) { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ; std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; @@ -83,7 +85,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // update file and dir lists for current directory. mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + std::set new_files ; + mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; // now go through list of subfiles and request the hash to hashcache @@ -98,7 +102,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 35f069b3c..74795ae14 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -4,13 +4,13 @@ // - remote: directories are requested remotely to a providing client // #include "file_sharing/hash_cache.h" - -class LocalDirectoryStorage ; +#include "file_sharing/directory_storage.h" class DirectoryUpdater { public: - DirectoryUpdater() ; + DirectoryUpdater() {} + virtual ~DirectoryUpdater(){} // Does some updating job. Crawls the existing directories and checks wether it has been updated // recently enough. If not, calls the directry source. @@ -24,6 +24,8 @@ class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual ~LocalDirectoryUpdater() {} + virtual void tick() ; protected: @@ -38,5 +40,8 @@ private: class RemoteDirectoryUpdater: public DirectoryUpdater { public: + RemoteDirectoryUpdater() {} + virtual ~RemoteDirectoryUpdater() {} + virtual void tick() ; }; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 747ec93d9..105c7776c 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -124,9 +124,10 @@ bool FolderIterator::d_name(std::string& dest) } const std::string& FolderIterator::file_fullpath() { return mFullPath ; } -const std::string& FolderIterator::file_name() { return mFileName ; } -uint64_t FolderIterator::file_size() { return mFileSize ; } -time_t FolderIterator::file_modtime() { return mFileModTime ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } +uint8_t FolderIterator::file_type() { return mType ; } bool FolderIterator::closedir() { diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 823aa9a9a..dcf8ed3f3 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -40,7 +40,7 @@ public: const std::string& file_name() ; const std::string& file_fullpath() ; uint64_t file_size() ; - uint64_t file_type() ; + uint8_t file_type() ; time_t file_modtime() ; private: