added iterators for directory storage class

libretroshare/src/file_sharing/directory_storage.cc

@@ -1,5 +1,22 @@
+#include <set>
 #include "directory_storage.h"
 
+/******************************************************************************************************************/
+/*                                              Internal File Hierarchy Storage                                   */
+/******************************************************************************************************************/
+
+template<class T> typename std::set<T>::iterator erase_from_set(typename std::set<T>& s,const typename std::set<T>::iterator& it)
+{
+    typename std::set<T>::iterator tmp(it);
+    ++tmp;
+    s.erase(it) ;
+    return tmp;
+}
+
+// This class handles the file hierarchy
+// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed,
+// so that the hierarchy stays completely coherent between calls.
+
 class InternalFileHierarchyStorage
 {
     class FileStorageNode
@@ -8,36 +25,279 @@ class InternalFileHierarchyStorage
         static const uint32_t TYPE_FILE = 0x0001 ;
         static const uint32_t TYPE_DIR  = 0x0002 ;
 
+        virtual ~FileStorageNode() {}
         virtual uint32_t type() const =0;
     };
     class FileEntry: public FileStorageNode
     {
     public:
+        FileEntry(const std::string& name) : file_name(name) {}
         virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
+        virtual ~FileEntry() {}
 
+        // local stuff
+        std::string file_name ;
+        uint64_t    file_size ;
+        RsFileHash  file_hash ;
+        time_t      file_modtime ;
     };
 
     class DirEntry: public FileStorageNode
     {
     public:
+        DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {}
+        virtual ~DirEntry() {}
+
         virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
 
-        std::set<DirectoryStorage::EntryIndex> subdirs ;
+        // local stuff
-        std::set<DirectoryStorage::EntryIndex> subfiles ;
+        std::string dir_name ;
+        DirectoryStorage::EntryIndex parent_index;
+
+        std::vector<DirectoryStorage::EntryIndex> subdirs ;
+        std::vector<DirectoryStorage::EntryIndex> subfiles ;
     };
 
+    // class stuff
+    InternalFileHierarchyStorage() : mRoot(0)
+    {
+        mNodes.push_back(new DirEntry("",0)) ;
+    }
+
+    // high level modification routines
+
+    bool isIndexValid(DirectoryStorage::EntryIndex e)
+    {
+        return e < mNodes.size() && mNodes[e] != NULL ;
+    }
+
+    bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set<std::string>& subdirs)
+    {
+        if(indx >= mNodes.size() || mNodes[indx] == NULL)
+            return nodeAccessError("updateSubDirectoryList(): Node does not exist") ;
+
+        if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
+            return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ;
+
+        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+
+        std::set<std::string> should_create(subdirs);
+
+        for(uint32_t i=0;i<d.subdirs.size();)
+            if(subdirs.find(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) == subdirs.end())
+                removeDirectory(d.subdirs[i]) ;
+            else
+            {
+                should_create.erase(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) ;
+                ++i;
+            }
+
+        for(std::set<std::string>::const_iterator it(should_create.begin());it!=should_create.end();++it)
+        {
+            d.subdirs.push_back(mNodes.size()) ;
+            mNodes.push_back(new DirEntry(*it,indx));
+        }
+
+        return true;
+    }
+    bool removeDirectory(const DirectoryStorage::EntryIndex& indx)
+    {
+        // check that it's a directory
+
+        if(indx >= mNodes.size() || mNodes[indx] == NULL)
+            return nodeAccessError("removeDirectory(): Node does not exist") ;
+
+        if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
+            return nodeAccessError("removeDirectory(): Node is not a directory") ;
+
+        if(indx == 0)
+            return nodeAccessError("removeDirectory(): Cannot remove top level directory") ;
+
+        // remove from parent
+
+        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+        DirEntry& parent_dir(*static_cast<DirEntry*>(mNodes[d.parent_index]));
+
+        for(uint32_t i=0;i<parent_dir.subdirs.size();++i)
+            if(parent_dir.subdirs[i] == indx)
+                    return recursRemoveDirectory(indx) ;
+
+        return nodeAccessError("removeDirectory(): inconsistency!!") ;
+    }
+
+    bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files)
+    {
+        if(indx >= mNodes.size() || mNodes[indx] == NULL)
+            return nodeAccessError("updateSubFilesList(): Node does not exist") ;
+
+        if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR)
+            return nodeAccessError("updateSubFilesList(): Node is not a directory") ;
+
+        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+        new_files = subfiles ;
+
+        for(uint32_t i=0;i<d.subfiles.size();)
+            if(subfiles.find(static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_name) == subfiles.end())
+            {
+                d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ;
+                d.subfiles.pop_back();
+            }
+            else
+            {
+                new_files.erase(static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_name) ;
+                ++i;
+            }
+
+        for(std::set<std::string>::const_iterator it(new_files.begin());it!=new_files.end();++it)
+        {
+            d.subfiles.push_back(mNodes.size()) ;
+            mNodes.push_back(new FileEntry(*it));
+        }
+        return true;
+    }
+    void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
+    void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ;
+
     // file/dir access and modification
     bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
 
-    uint32_t root ;
+    uint32_t mRoot ;
 
     std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
 
-    void compress() ;					// use empty space in the vector, mostly due to deleted entries.
+    void compress() ;					// use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
+                                            // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
+                                            // are not referenced.
 
     friend class DirectoryStorage ;		// only class that can use this.
+
+    // low level stuff. Should normally not be used externally.
+
+    DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index)
+    {
+        if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR)
+            return DirectoryStorage::NO_INDEX;
+
+        if(static_cast<DirEntry*>(mNodes[parent_index])->subfiles.size() <= file_tab_index)
+            return DirectoryStorage::NO_INDEX;
+
+        return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[file_tab_index];
+    }
+    DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index)
+    {
+        if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR)
+            return DirectoryStorage::NO_INDEX;
+
+        if(static_cast<DirEntry*>(mNodes[parent_index])->subdirs.size() <= dir_tab_index)
+            return DirectoryStorage::NO_INDEX;
+
+        return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[dir_tab_index];
+    }
+
+    bool check()	// checks consistency of storage.
+    {
+        return true;
+    }
+
+    void print()
+    {
+    }
+private:
+    bool nodeAccessError(const std::string& s)
+    {
+        std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
+        return false ;
+    }
+
+    // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
+    // phase. That allows to keep file information when moving them around.
+
+    bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir)
+    {
+        DirEntry& d(*static_cast<DirEntry*>(mNodes[dir])) ;
+
+        for(uint32_t i=0;i<d.subdirs.size();++i)
+            recursRemoveDirectory(d.subdirs[i]);
+
+        delete mNodes[dir] ;
+        mNodes[dir] = NULL ;
+
+        return true ;
+    }
 };
 
+/******************************************************************************************************************/
+/*                                                      Iterators                                                 */
+/******************************************************************************************************************/
+
+DirectoryStorage::DirIterator::DirIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
+{
+    mStorage = s->mFileHierarchy ;
+    mParentIndex = i;
+    mDirTabIndex = 0;
+}
+
+DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
+{
+    mStorage = s->mFileHierarchy ;
+    mParentIndex = i;
+    mFileTabIndex = 0;
+}
+
+DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++()
+{
+    ++mDirTabIndex ;
+
+    return *this;
+}
+DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
+{
+    ++mFileTabIndex ;
+
+    return *this;
+}
+DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
+{
+    return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ;
+}
+DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
+{
+    return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ;
+}
+DirectoryStorage::DirIterator::operator bool() const
+{
+    return **this != DirectoryStorage::NO_INDEX;
+}
+DirectoryStorage::FileIterator::operator bool() const
+{
+    return **this != DirectoryStorage::NO_INDEX;
+}
+
+/******************************************************************************************************************/
+/*                                                 Directory Storage                                              */
+/******************************************************************************************************************/
+
+DirectoryStorage::DirIterator DirectoryStorage::root()
+{
+    return DirIterator(this,EntryIndex(0)) ;
+}
+
+void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set<std::string>& subdirs)
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    mFileHierarchy->updateSubDirectoryList(indx,subdirs) ;
+}
+void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files)
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
+}
+void DirectoryStorage::removeDirectory(const EntryIndex& indx)
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    mFileHierarchy->removeDirectory(indx);
+}
+
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH         =  0x01 ;
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME         =  0x02 ;
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE         =  0x03 ;

libretroshare/src/file_sharing/directory_storage.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <string>
 #include <stdint.h>
 #include <list>
@@ -23,12 +25,13 @@ class DirectoryStorage
         virtual ~DirectoryStorage() {}
 
         typedef uint32_t EntryIndex ;
+        static const EntryIndex NO_INDEX = 0xffffffff;
 
 		void save() const ;
 
-		virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const;
+        virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const { return 0;}
-		virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const;
+        virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const { return 0;}
-		virtual int searchBoolExp(Expression * exp, std::list<EntryIndex> &results) const;
+        virtual int searchBoolExp(Expression * exp, std::list<EntryIndex> &results) const { return 0; }
 
 		void getFileDetails(EntryIndex i) ;
 
@@ -39,25 +42,31 @@ class DirectoryStorage
 		{
 			public:
                 DirIterator(const DirIterator& d) ;
-                DirIterator(const EntryIndex& d) ;
+                DirIterator(DirectoryStorage *d,EntryIndex i) ;
 
 				DirIterator& operator++() ;
-				EntryIndex operator*() const ;		// current directory entry
+                EntryIndex operator*() const ;
 
                 operator bool() const ;			// used in for loops. Returns true when the iterator is valid.
 
                 // info about the directory that is pointed by the iterator
 
                 const std::string& name() const ;
+            private:
+                EntryIndex mParentIndex ;		// index of the parent dir.
+                uint32_t mDirTabIndex ;				// index in the vector of subdirs.
+                InternalFileHierarchyStorage *mStorage ;
+
+                friend class DirectoryStorage ;
         };
 		class FileIterator
 		{
 			public:
-                FileIterator(DirIterator& d);		// crawls all files in specified directory
+                FileIterator(DirIterator& d);	// crawls all files in specified directory
-                FileIterator(EntryIndex& e);		// crawls all files in specified directory
+                FileIterator(DirectoryStorage *d,EntryIndex e);		// crawls all files in specified directory
 
 				FileIterator& operator++() ;
-				EntryIndex operator*() const ;		// current file entry
+                EntryIndex operator*() const ;	// current file entry
 
                 operator bool() const ;			// used in for loops. Returns true when the iterator is valid.
 
@@ -68,14 +77,24 @@ class DirectoryStorage
                 uint64_t size() const ;
                 RsFileHash hash() const ;
                 time_t modtime() const ;
+
+            private:
+                EntryIndex mParentIndex ;		// index of the parent dir.
+                uint32_t   mFileTabIndex ;		// index in the vector of subdirs.
+                InternalFileHierarchyStorage *mStorage ;
         };
 
         virtual DirIterator root() ;					// returns the index of the root directory entry.
 
-        void updateSubDirectoryList(const EntryIndex& indx,const std::list<std::string>& subdirs) ;
+        void updateSubDirectoryList(const EntryIndex& indx,const std::set<std::string>& subdirs) ;
-        void updateSubFilesList(const EntryIndex& indx,const std::list<std::string>& subfiles) ;
+        void updateSubFilesList(const EntryIndex& indx,const std::set<std::string>& subfiles,std::set<std::string>& new_files) ;
         void removeDirectory(const EntryIndex& indx) ;
 
+        void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
+        void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ;
+
+        void cleanup();
+
     private:
 		void load(const std::string& local_file_name) ;
 		void save(const std::string& local_file_name) ;
@@ -86,6 +105,8 @@ class DirectoryStorage
         // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
 
         InternalFileHierarchyStorage *mFileHierarchy ;
+
+        RsMutex mDirStorageMtx ;
 };
 
 class RemoteDirectoryStorage: public DirectoryStorage
@@ -104,15 +125,6 @@ public:
     void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
     void getSharedDirectoryList(std::list<SharedDirInfo>& lst) ;
 
-        /*!
-         * \brief addFile
-         * \param dir
-         * \param hash
-         * \param modf_time
-         */
-        void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
-        void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ;
-
 private:
     std::list<SharedDirInfo> mLocalDirs ;
 

libretroshare/src/file_sharing/directory_updater.cc

@@ -26,10 +26,12 @@ void LocalDirectoryUpdater::tick()
 	// 	- doing so, changing directory names or moving files between directories does not cause a re-hash of the content. 
 	//
     std::list<SharedDirInfo> shared_directory_list ;
-    std::list<std::string> sub_dir_list ;
+
+
+    std::set<std::string> sub_dir_list ;
 
     for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
-        sub_dir_list.push_back( (*real_dir_it).filename ) ;
+        sub_dir_list.insert( (*real_dir_it).filename ) ;
 
     // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
 
@@ -61,18 +63,18 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
 
     // collect subdirs and subfiles
 
-    std::list<std::string> subfiles ;
+    std::set<std::string> subfiles ;
-    std::list<std::string> subdirs ;
+    std::set<std::string> subdirs ;
 
     while(dirIt.readdir())
     {
         switch(dirIt.file_type())
         {
-                case librs::util::FolderIterator::TYPE_FILE:	subfiles.push_back(dirIt.file_name()) ;
+                case librs::util::FolderIterator::TYPE_FILE:	subfiles.insert(dirIt.file_name()) ;
                                                                 std::cerr << "  adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
                                                             break;
 
-                case librs::util::FolderIterator::TYPE_DIR:	subdirs.push_back(dirIt.file_name()) ;
+                case librs::util::FolderIterator::TYPE_DIR:	subdirs.insert(dirIt.file_name()) ;
                                                                 std::cerr << "  adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
                                                             break;
 
@@ -83,7 +85,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
     // update file and dir lists for current directory.
 
     mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
-    mSharedDirectories->updateSubFilesList(indx,subfiles) ;
+
+    std::set<std::string> new_files ;
+    mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
 
     // now go through list of subfiles and request the hash to hashcache
 
@@ -98,7 +102,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
 
     DirectoryStorage::DirIterator stored_dir_it(indx) ;
 
-    for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
+    for(std::set<std::string>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
         recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
 }
 

libretroshare/src/file_sharing/directory_updater.h

@@ -4,13 +4,13 @@
 // 	- remote: directories are requested remotely to a providing client
 //
 #include "file_sharing/hash_cache.h"
-
+#include "file_sharing/directory_storage.h"
-class LocalDirectoryStorage ;
 
 class DirectoryUpdater
 {
 	public:
-		DirectoryUpdater() ;
+        DirectoryUpdater() {}
+        virtual ~DirectoryUpdater(){}
 
 		// Does some updating job. Crawls the existing directories and checks wether it has been updated
 		// recently enough. If not, calls the directry source.
@@ -24,6 +24,8 @@ class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient
 {
 public:
     LocalDirectoryUpdater(HashCache *hash_cache) ;
+    virtual ~LocalDirectoryUpdater() {}
+
     virtual void tick() ;
 
 protected:
@@ -38,5 +40,8 @@ private:
 class RemoteDirectoryUpdater: public DirectoryUpdater
 {
     public:
+            RemoteDirectoryUpdater() {}
+        virtual ~RemoteDirectoryUpdater() {}
+
         virtual void tick() ;
 };

libretroshare/src/util/folderiterator.cc

@@ -124,9 +124,10 @@ bool FolderIterator::d_name(std::string& dest)
 }
 
 const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
-const std::string& FolderIterator::file_name() { return mFileName ; }
+const std::string& FolderIterator::file_name()     { return mFileName ; }
-uint64_t     FolderIterator::file_size()       { return mFileSize ; }
+uint64_t           FolderIterator::file_size()     { return mFileSize ; }
-time_t       FolderIterator::file_modtime()    { return mFileModTime ; }
+time_t             FolderIterator::file_modtime()  { return mFileModTime ; }
+uint8_t            FolderIterator::file_type()     { return mType ; }
 
 bool FolderIterator::closedir()
 {

libretroshare/src/util/folderiterator.h

@@ -40,7 +40,7 @@ public:
     const std::string& file_name() ;
     const std::string& file_fullpath() ;
     uint64_t file_size() ;
-    uint64_t file_type() ;
+    uint8_t file_type() ;
     time_t   file_modtime() ;
 
 private: