added squeleton code for own directory update

libretroshare/src/file_sharing/README.txt

@@ -160,14 +160,14 @@ Generating sync events
 Roadmap
 -------
 
-- complete this file until a proper description of the whole thing is achieved.
+[X] complete this file until a proper description of the whole thing is achieved.
-- create a new directory and implement the .h for the basic functionality
+[X] create a new directory and implement the .h for the basic functionality
-- look into existing code in ftServer for the integration, but don't change anything yet
+[ ] look into existing code in ftServer for the integration, but don't change anything yet
-- setup class hierarchy
+[X] setup class hierarchy
-- merge hash cache into file lists.
+[ ] merge hash cache into file lists.
-
+[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible
-- optionally
+[ ] create basic directory functionality with own files: re-hash, and store
-   - change the saving system of FileIndex to make it locally encrypted and compact
+[ ] display own files in GUI, with proper update and working sort
 
 TODO
 ====

libretroshare/src/file_sharing/directory_storage.cc

@@ -0,0 +1,60 @@
+#include "directory_storage.h"
+
+class InternalFileHierarchyStorage
+{
+    class FileStorageNode
+    {
+    public:
+        virtual uint32_t type() const =0;
+    };
+    class FileEntry: public FileStorageNode
+    {
+    public:
+        virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
+
+    };
+
+    class DirEntry: public FileStorageNode
+    {
+    public:
+        virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
+
+        std::set<EntryIndex> subdirs ;
+        std::set<EntryIndex> subfiles ;
+    };
+
+    // file/dir access and modification
+    bool findSubDirectory(EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
+
+    uint32_t root ;
+
+    std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
+
+    void compress() ;					// use empty space in the vector, mostly due to deleted entries.
+
+    friend class DirectoryStorage ;		// only class that can use this.
+};
+
+// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH         =  0x01 ;
+// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME         =  0x02 ;
+// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE         =  0x03 ;
+// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME          =  0x04 ;
+// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS          =  0x05 ;
+// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS   =  0x06 ;
+
+void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size)
+{
+    entry_tag = data[offset++] ;
+}
+void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size)
+{
+}
+
+void DirectoryStorage::load(const std::string& local_file_name)
+{
+    // first load the header, than all fields.
+}
+void DirectoryStorage::save(const std::string& local_file_name)
+{
+    // first write the header, than all fields.
+}

libretroshare/src/file_sharing/directory_storage.h

@@ -14,6 +14,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME          =  0x04 ;
 static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS          =  0x05 ;
 static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS   =  0x06 ;
 
+class InternalFileHierarchyStorage ;
+
 class DirectoryStorage
 {
 	public:
@@ -36,30 +38,37 @@ class DirectoryStorage
 		class DirIterator
 		{
 			public:
-				DirIterator(const DirectoryStorage& d) ;
+                DirIterator(const DirIterator& d) ;
 
 				DirIterator& operator++() ;
 				EntryIndex operator*() const ;		// current directory entry
 
-				bool operator()() const ;				// used in for loops. Returns true when the iterator is valid.
+                bool operator()() const ;			// used in for loops. Returns true when the iterator is valid.
 		};
 		class FileIterator
 		{
 			public:
-				FileIterator(const DirectoryStorage& d) ;
+                FileIterator(DirIterator& d);		// crawls all files in specified directory
 
+                uint32_t size() const ;				// number of files in this directory
 				FileIterator& operator++() ;
 				EntryIndex operator*() const ;		// current file entry
 
-				bool operator()() const ;				// used in for loops. Returns true when the iterator is valid.
+                bool operator()() const ;			// used in for loops. Returns true when the iterator is valid.
 		};
 
+        virtual DirIterator root() ;					// returns the index of the root directory entry.
+
 	private:
 		void load(const std::string& local_file_name) ;
 		void save(const std::string& local_file_name) ;
 
-		void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ;
+        void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ;
-		void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ;
+        void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ;
+
+        // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
+
+        InternalFileHierarchyStorage *mFileHierarchy ;
 };
 
 class RemoteDirectoryStorage: public DirectoryStorage
@@ -74,5 +83,32 @@ class LocalDirectoryStorage: public DirectoryStorage
 public:
     LocalDirectoryStorage() ;
     virtual ~LocalDirectoryStorage() {}
+
+    void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
+    void getSharedDirectoryList(std::list<SharedDirInfo>& lst) ;
+
+        /*!
+         * \brief addFile
+         * \param dir
+         * \param hash
+         * \param modf_time
+         */
+        void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ;
+        void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ;
+
+private:
+    std::list<SharedDirInfo> mLocalDirs ;
+
+    std::map<RsFileHash,EntryIndex> mHashes ;		// used for fast search access
 };
 
+
+
+
+
+
+
+
+
+
+

libretroshare/src/file_sharing/directory_updater.cc

@@ -6,30 +6,99 @@ void RemoteDirectoryUpdater::tick()
 	// use the stored iterator
 }
 
+LocalDirectoryUpdater::LocalDirectoryUpdater()
+{
+    // tell the hash cache that we're the client
+    mHashCache->setClient(this) ;
+}
+
 void LocalDirectoryUpdater::tick()
 {
-#ifdef TODO
 	// recursive update algorithm works that way:
 	// 	- the external loop starts on the shared directory list and goes through sub-directories
-	// 	- at the same time, it updates the list of shared directories
+    // 	- at the same time, it updates the local list of shared directories.  A single sweep is performed over the whole directory structure.
 	// 	- the information that is costly to compute (the hash) is store externally into a separate structure.
 	// 	- doing so, changing directory names or moving files between directories does not cause a re-hash of the content. 
 	//
-	for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it)
+    std::list<SharedDirInfo> shared_directory_list ;
-	{
-		librs::util::FolderIterator dirIt(realpath);
-		if(!dirIt.isValid())
-			mLocalSharedDirs.removeDirectory(it) ;	// this is a complex operation since it needs to *update* it so that it is kept consistent.
 
-		while(dirIt.readdir())
+    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it)
-		{
+        sub_dir_list.push_back( (*it).filename ) ;
 
-		}
+    // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
 
-		if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash))
+    mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
-		{
+
-			;
+    // now for each of them, go recursively and match both files and dirs
-		}
+
-	}
+    DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ;
-#endif
+
+    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it)
+        recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
 }
+
+LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx)
+{
+    // make sure list of subdirs is the same
+
+    // make sure list of subfiles is the same
+
+    // request all hashes to the hashcache
+
+    librs::util::FolderIterator dirIt(cumulated_path);
+
+    if(!dirIt.isValid())
+    {
+        mSharedDirectories->removeDirectory(indx) ;	// this is a complex operation since it needs to *update* it so that it is kept consistent.
+        return ;
+    }
+
+    // collect subdirs and subfiles
+
+    std::list<std::string> subfiles ;
+    std::list<std::string> subdirs ;
+
+    while(dirIt.readdir())
+    {
+        std::string name ;
+        uint64_t size ;
+        uint8_t type ;
+        time_t mod_time ;
+
+        dirIt.readEntryInformation(type,name,size,mod_time) ;
+
+        switch(type)
+        {
+                case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ;
+                                                        break;
+                case librs::util::FolderIterator::TYPE_DIR:  subdirs.push_back(name) ;
+                                                            break;
+
+        default:
+            std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl;
+        }
+    }
+    // update file and dir lists for current directory.
+
+    mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
+    mSharedDirectories->updateSubFilesList(indx,subfiles) ;
+
+    // now go through list of subfiles and request the hash to hashcache
+
+    for(DirectoryStorage::FileIterator dit(indx);dit;++dit)
+    {
+        // ask about the hash. If not present, ask HashCache. If not present, the callback will update it.
+
+        if()
+            if(mHashCache.requestHash(realpath,name,size,modtime,hash),this)
+                mSharedDirectories->updateFileHash(*dit,hash) ;
+    }
+
+    // go through the list of sub-dirs and recursively update
+
+    DirectoryStorage::DirIterator stored_dir_it(indx) ;
+
+    for(std::list<std::string>::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it)
+        recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ;
+}
+

libretroshare/src/file_sharing/directory_updater.h

@@ -3,6 +3,9 @@
 // 	- local: directories are crawled n disk and files are hashed / requested from a cache
 // 	- remote: directories are requested remotely to a providing client
 //
+class HashCache ;
+class LocalDirectoryStorage ;
+
 class DirectoryUpdater
 {
 	public:
@@ -18,8 +21,13 @@ class DirectoryUpdater
 
 class LocalDirectoryUpdater: public DirectoryUpdater
 {
-    public:
+public:
-        virtual void tick() ;
+    LocalDirectoryUpdater(HashCache *hash_cache) ;
+    virtual void tick() ;
+
+private:
+    HashCache *mHashCache ;
+    LocalDirectoryStorage *mSharedDirectories ;
 };
 
 class RemoteDirectoryUpdater: public DirectoryUpdater

libretroshare/src/file_sharing/hash_cache.h

@@ -1,29 +1,50 @@
+#pragma once
+
+#include <map>
+#include "util/rsthreads.h"
+
 class HashCache
 {
-    public:
+public:
-        HashCache(const std::string& save_file_name) ;
+    HashCache(const std::string& save_file_name) ;
 
-        void save() ;
+    void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
-        void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
+    bool find(const  std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
-        bool find(const  std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
-        void clean() ;
 
-        typedef struct
+    struct HashCacheInfo
-        {
+    {
-            uint64_t size ;
+        std::string filename ;
-            uint64_t time_stamp ;
+        uint64_t size ;
-            uint64_t modf_stamp ;
+        uint32_t time_stamp ;
-            RsFileHash hash ;
+        uint32_t modf_stamp ;
-        } HashCacheInfo ;
+        RsFileHash hash ;
+    } ;
 
-        void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
+    // interaction with GUI, called from p3FileLists
-        uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
+    void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
-        void clear() { _files.clear(); }
+    uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
-        bool empty() const { return _files.empty() ; }
+    void clear() { _files.clear(); }
-    private:
+    bool empty() const { return _files.empty() ; }
-        uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries
+
-        std::map<std::string, HashCacheInfo> _files ;
+private:
-        std::string _path ;
+    void clean() ;
-        bool _changed ;
+
+    void save() ;
+    void load() ;
+
+    // threaded stuff
+    RsTickingThread mHashingThread ;
+    RsMutex mHashMtx ;
+
+    // Local configuration and storage
+
+    uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries
+    std::map<std::string, HashCacheInfo> mFiles ;
+    std::string mFilePath ;
+    bool mChanged ;
+
+    // current work
+
+    std::map<std::string> mFilesToHash ;
 };