moved InternalFileHierarchyStorage to separate file. Fixed loading/saving infrastructure (not working yet)

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -0,0 +1,556 @@
+#include "dir_hierarchy.h"
+
+#define DEBUG_DIRECTORY_STORAGE 1
+
+/******************************************************************************************************************/
+/*                                              Internal File Hierarchy Storage                                   */
+/******************************************************************************************************************/
+
+template<class T> typename std::set<T>::iterator erase_from_set(typename std::set<T>& s,const typename std::set<T>::iterator& it)
+{
+    typename std::set<T>::iterator tmp(it);
+    ++tmp;
+    s.erase(it) ;
+    return tmp;
+}
+
+// This class handles the file hierarchy
+// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed,
+// so that the hierarchy stays completely coherent between calls.
+
+InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
+{
+    DirEntry *de = new DirEntry("") ;
+
+    de->row=0;
+    de->parent_index=0;
+    de->dir_modtime=0;
+
+    mNodes.push_back(de) ;
+}
+
+int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e)
+{
+    if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0)
+        return -1 ;
+
+    return mNodes[mNodes[e]->parent_index]->row;
+}
+
+// high level modification routines
+
+bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) const
+{
+    return e < mNodes.size() && mNodes[e] != NULL ;
+}
+bool InternalFileHierarchyStorage::stampDirectory(const DirectoryStorage::EntryIndex& indx)
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+        return false;
+
+    static_cast<DirEntry*>(mNodes[indx])->dir_modtime = time(NULL) ;
+    return true;
+}
+
+bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+        return false;
+
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+
+    std::map<std::string,time_t> should_create(subdirs);
+
+    for(uint32_t i=0;i<d.subdirs.size();)
+        if(subdirs.find(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) == subdirs.end())
+        {
+            std::cerr << "[directory storage] Removing subdirectory " << static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl;
+
+            if( !removeDirectory(d.subdirs[i]))
+                i++ ;
+        }
+        else
+        {
+            std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl;
+
+            should_create.erase(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) ;
+            ++i;
+        }
+
+    for(std::map<std::string,time_t>::const_iterator it(should_create.begin());it!=should_create.end();++it)
+    {
+        std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl;
+
+        DirEntry *de = new DirEntry(it->first) ;
+
+        de->row = mNodes.size();
+        de->parent_index = indx;
+        de->dir_modtime = it->second;
+
+        d.subdirs.push_back(mNodes.size()) ;
+        mNodes.push_back(de) ;
+    }
+
+    return true;
+}
+bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx)	// no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!!
+{
+    // check that it's a directory
+
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+        return false;
+
+    if(indx == 0)
+        return nodeAccessError("checkIndex(): Cannot remove top level directory") ;
+
+#ifdef DEBUG_DIRECTORY_STORAGE
+    std::cerr << "(--) Removing directory " << indx << std::endl;
+    print();
+#endif
+    // remove from parent
+
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+    DirEntry& parent_dir(*static_cast<DirEntry*>(mNodes[d.parent_index]));
+
+    for(uint32_t i=0;i<parent_dir.subdirs.size();++i)
+        if(parent_dir.subdirs[i] == indx)
+        {
+            parent_dir.subdirs[i] = parent_dir.subdirs.back() ;
+            parent_dir.subdirs.pop_back();
+
+            bool res = recursRemoveDirectory(indx) ;
+#ifdef DEBUG_DIRECTORY_STORAGE
+            print();
+            std::string err ;
+            if(!check(err))
+                std::cerr << "(EE) Error after removing subdirectory. Error string=\"" << err << "\", Hierarchy is : " << std::endl;
+#endif
+            return true ;
+        }
+    return nodeAccessError("removeDirectory(): inconsistency!!") ;
+}
+
+bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const
+{
+    if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL)
+        return nodeAccessError("checkIndex(): Node does not exist") ;
+
+    if(! (mNodes[indx]->type() & type))
+        return nodeAccessError("checkIndex(): Node is of wrong type") ;
+
+    return true;
+}
+
+bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+        return false;
+
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+    new_files = subfiles ;
+
+    // remove from new_files the ones that already exist and have a modf time that is not older.
+
+    for(uint32_t i=0;i<d.subfiles.size();)
+    {
+        FileEntry& f(*static_cast<FileEntry*>(mNodes[d.subfiles[i]])) ;
+        std::map<std::string,DirectoryStorage::FileTS>::const_iterator it = subfiles.find(f.file_name) ;
+
+        if(it == subfiles.end())				// file does not exist anymore => delete
+        {
+            std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl;
+
+            delete mNodes[d.subfiles[i]] ;
+            mNodes[d.subfiles[i]] = NULL ;
+
+            d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ;
+            d.subfiles.pop_back();
+            continue;
+        }
+
+        if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)	// file is newer and/or has different size
+        {
+            f.file_hash.clear();																// hash needs recomputing
+            f.file_modtime = it->second.modtime;
+            f.file_size = it->second.size;
+        }
+        new_files.erase(f.file_name) ;
+
+        ++i;
+    }
+
+    for(std::map<std::string,DirectoryStorage::FileTS>::const_iterator it(new_files.begin());it!=new_files.end();++it)
+    {
+        std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl;
+
+        d.subfiles.push_back(mNodes.size()) ;
+        mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime));
+        mNodes.back()->row = mNodes.size()-1;
+        mNodes.back()->parent_index = indx;
+    }
+    return true;
+}
+bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
+{
+    if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
+    {
+        std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
+        return false;
+    }
+
+    std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl;
+
+    RsFileHash& old_hash (static_cast<FileEntry*>(mNodes[file_index])->file_hash) ;
+
+    old_hash = hash ;
+
+    return true;
+}
+bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time)
+{
+    if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
+    {
+        std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
+        return false;
+    }
+
+    FileEntry& fe(*static_cast<FileEntry*>(mNodes[file_index])) ;
+
+    std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl;
+
+    fe.file_hash = hash;
+    fe.file_size = size;
+    fe.file_modtime = modf_time;
+    fe.file_name = fname;
+
+    return true;
+}
+
+bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector<DirectoryStorage::EntryIndex>& subdirs_array,const std::vector<DirectoryStorage::EntryIndex>& subfiles_array)
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+    {
+        std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
+        return false;
+    }
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+
+    d.most_recent_time = most_recent_time;
+    d.dir_modtime      = dir_modtime;
+    d.dir_update_time  = time(NULL);
+    d.dir_name         = dir_name;
+
+    d.subfiles = subfiles_array ;
+    d.subdirs  = subdirs_array ;
+
+    // check that all subdirs already exist. If not, create.
+    for(uint32_t i=0;i<subdirs_array.size();++i)
+    {
+        if(subdirs_array[i] >= mNodes.size() )
+            mNodes.resize(subdirs_array[i]+1,NULL);
+
+        FileStorageNode *& node(mNodes[subdirs_array[i]]);
+
+        if(node != NULL && node->type() != FileStorageNode::TYPE_DIR)
+        {
+            delete node ;
+            node = NULL ;
+        }
+
+        if(node == NULL)
+            node = new DirEntry("");
+
+        ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ;
+        node->row = i ;
+        node->parent_index = indx ;
+    }
+    for(uint32_t i=0;i<subfiles_array.size();++i)
+    {
+        if(subfiles_array[i] >= mNodes.size() )
+            mNodes.resize(subfiles_array[i]+1,NULL);
+
+        FileStorageNode *& node(mNodes[subfiles_array[i]]);
+
+        if(node != NULL && node->type() != FileStorageNode::TYPE_FILE)
+        {
+            delete node ;
+            node = NULL ;
+        }
+
+        if(node == NULL)
+            node = new FileEntry("",0,0);
+
+        node->row = subdirs_array.size()+i ;
+        node->parent_index = indx ;
+    }
+
+    return true;
+}
+
+bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS)
+{
+    if(!checkIndex(index,FileStorageNode::TYPE_DIR))
+    {
+        std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl;
+        return false;
+    }
+
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
+
+    recurs_max_modf_TS = d.most_recent_time ;
+    local_update_TS    = d.dir_update_time ;
+
+    return true;
+}
+bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS)
+{
+    if(!checkIndex(index,FileStorageNode::TYPE_DIR))
+    {
+        std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl;
+        return false;
+    }
+
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
+
+    d.most_recent_time = recurs_max_modf_TS ;
+    d.dir_update_time  = local_update_TS    ;
+
+    return true;
+}
+
+// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
+
+time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index)
+{
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[dir_index])) ;
+
+    time_t largest_modf_time = d.dir_modtime ;
+
+    for(uint32_t i=0;i<d.subfiles.size();++i)
+        largest_modf_time = std::max(largest_modf_time,static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_modtime) ;
+
+    for(uint32_t i=0;i<d.subdirs.size();++i)
+        largest_modf_time = std::max(largest_modf_time,recursUpdateLastModfTime(d.subdirs[i])) ;
+
+    d.most_recent_time = largest_modf_time ;
+
+    return largest_modf_time ;
+}
+
+// Low level stuff. Should normally not be used externally.
+
+const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorage::getNode(DirectoryStorage::EntryIndex indx) const
+{
+    if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR))
+        return mNodes[indx] ;
+    else
+        return NULL ;
+}
+
+const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+        return NULL ;
+
+    return static_cast<DirEntry*>(mNodes[indx]) ;
+}
+const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
+{
+    if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
+        return NULL ;
+
+    return static_cast<FileEntry*>(mNodes[indx]) ;
+}
+uint32_t InternalFileHierarchyStorage::getType(DirectoryStorage::EntryIndex indx) const
+{
+    if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR))
+        return mNodes[indx]->type() ;
+    else
+        return FileStorageNode::TYPE_UNKNOWN;
+}
+
+DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index)
+{
+    if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR))
+        return DirectoryStorage::NO_INDEX;
+
+    if(static_cast<DirEntry*>(mNodes[parent_index])->subfiles.size() <= file_tab_index)
+        return DirectoryStorage::NO_INDEX;
+
+    return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[file_tab_index];
+}
+DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index)
+{
+    if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR))
+        return DirectoryStorage::NO_INDEX;
+
+    if(static_cast<DirEntry*>(mNodes[parent_index])->subdirs.size() <= dir_tab_index)
+        return DirectoryStorage::NO_INDEX;
+
+    return static_cast<DirEntry*>(mNodes[parent_index])->subdirs[dir_tab_index];
+}
+
+bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results)
+{
+    std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mHashes.find(hash);
+
+    if( it != mHashes.end() )
+    {
+        results.clear();
+        results.push_back(it->second) ;
+        return true ;
+    }
+    else
+        return false;
+}
+
+bool InternalFileHierarchyStorage::check(std::string& error_string) const	// checks consistency of storage.
+{
+    // recurs go through all entries, check that all
+
+    std::vector<uint32_t> hits(mNodes.size(),0) ;	// count hits of children. Should be 1 for all in the end. Otherwise there's an error.
+    hits[0] = 1 ;	// because 0 is never the child of anyone
+
+    for(uint32_t i=0;i<mNodes.size();++i)
+        if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR)
+        {
+            // stamp the kids
+            const DirEntry& de = *static_cast<DirEntry*>(mNodes[i]) ;
+
+            for(uint32_t j=0;j<de.subdirs.size();++j)
+            {
+                if(de.subdirs[j] >= mNodes.size())
+                {
+                    error_string = "Node child out of tab!" ;
+                    return false ;
+                }
+                if(hits[de.subdirs[j]] != 0)
+                {
+                    error_string = "Double hit on a single node" ;
+                    return false;
+                }
+                hits[de.subdirs[j]] = 1;
+            }
+            for(uint32_t j=0;j<de.subfiles.size();++j)
+            {
+                if(de.subfiles[j] >= mNodes.size())
+                {
+                    error_string = "Node child out of tab!" ;
+                    return false ;
+                }
+                if(hits[de.subfiles[j]] != 0)
+                {
+                    error_string = "Double hit on a single node" ;
+                    return false;
+                }
+                hits[de.subfiles[j]] = 1;
+            }
+
+        }
+
+    for(uint32_t i=0;i<hits.size();++i)
+        if(hits[i] == 0 && mNodes[i] != NULL)
+        {
+            error_string = "Orphean node!" ;
+            return false;
+        }
+
+    return true;
+}
+
+void InternalFileHierarchyStorage::print() const
+{
+    int nfiles = 0 ;
+    int ndirs = 0 ;
+    int nempty = 0 ;
+    int nunknown = 0;
+
+    for(uint32_t i=0;i<mNodes.size();++i)
+        if(mNodes[i] == NULL)
+        {
+            std::cerr << "  Node " << i << ": empty " << std::endl;
+            ++nempty ;
+        }
+        else if(mNodes[i]->type() == FileStorageNode::TYPE_DIR)
+        {
+            std::cerr << "  Node " << i << ": type=" << mNodes[i]->type() << std::endl;
+            ++ndirs;
+        }
+        else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE)
+        {
+            std::cerr << "  Node " << i << ": type=" << mNodes[i]->type() << std::endl;
+            ++nfiles;
+        }
+        else
+        {
+            ++nunknown;
+            std::cerr << "(EE) Error: unknown type node found!" << std::endl;
+        }
+
+    std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl;
+
+    recursPrint(0,DirectoryStorage::EntryIndex(0));
+}
+void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::EntryIndex node) const
+{
+    std::string indent(2*depth,' ');
+
+    if(mNodes[node] == NULL)
+    {
+        std::cerr << "EMPTY NODE !!" << std::endl;
+        return ;
+    }
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[node]));
+
+    std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ;
+
+    for(uint32_t i=0;i<d.subdirs.size();++i)
+        std::cerr << d.subdirs[i] << " " ;
+    std::cerr << std::endl;
+
+    for(uint32_t i=0;i<d.subdirs.size();++i)
+        recursPrint(depth+1,d.subdirs[i]) ;
+
+    for(uint32_t i=0;i<d.subfiles.size();++i)
+    {
+        FileEntry& f(*static_cast<FileEntry*>(mNodes[d.subfiles[i]]));
+        std::cerr << indent << "  hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << "  " << f.file_size << "  " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl;
+    }
+}
+
+bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
+{
+    std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
+    return false ;
+}
+
+// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
+// phase. That allows to keep file information when moving them around.
+
+bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::EntryIndex dir)
+{
+    DirEntry& d(*static_cast<DirEntry*>(mNodes[dir])) ;
+
+    for(uint32_t i=0;i<d.subdirs.size();++i)
+        recursRemoveDirectory(d.subdirs[i]);
+
+    for(uint32_t i=0;i<d.subfiles.size();++i)
+    {
+        delete mNodes[d.subfiles[i]] ;
+        mNodes[d.subfiles[i]] = NULL ;
+    }
+    delete mNodes[dir] ;
+    mNodes[dir] = NULL ;
+
+    return true ;
+}
+
+bool InternalFileHierarchyStorage::save(const std::string& fname)
+{
+    NOT_IMPLEMENTED();
+}
+
+bool InternalFileHierarchyStorage::load(const std::string& fname)
+{
+    NOT_IMPLEMENTED();
+}

libretroshare/src/file_sharing/dir_hierarchy.h

@@ -0,0 +1,119 @@
+#pragma once
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "directory_storage.h"
+
+class InternalFileHierarchyStorage
+{
+public:
+    class FileStorageNode
+    {
+    public:
+        static const uint32_t TYPE_UNKNOWN = 0x0000 ;
+        static const uint32_t TYPE_FILE    = 0x0001 ;
+        static const uint32_t TYPE_DIR     = 0x0002 ;
+
+        virtual ~FileStorageNode() {}
+        virtual uint32_t type() const =0;
+
+        DirectoryStorage::EntryIndex parent_index;
+        uint32_t row ;
+    };
+    class FileEntry: public FileStorageNode
+    {
+    public:
+        FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {}
+        virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
+        virtual ~FileEntry() {}
+
+        // local stuff
+        std::string file_name ;
+        uint64_t    file_size ;
+        time_t      file_modtime;
+        RsFileHash  file_hash ;
+    };
+
+    class DirEntry: public FileStorageNode
+    {
+    public:
+        DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {}
+        virtual ~DirEntry() {}
+
+        virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
+
+        // local stuff
+        std::string dir_name ;
+        std::string dir_parent_path ;
+
+        std::vector<DirectoryStorage::EntryIndex> subdirs ;
+        std::vector<DirectoryStorage::EntryIndex> subfiles ;
+
+        time_t dir_modtime;
+        time_t most_recent_time;	// recursive most recent modification time, including files and subdirs in the entire hierarchy below.
+        time_t dir_update_time;		// last time the information was updated for that directory. Includes subdirs indexes and subfile info.
+    };
+
+    // class stuff
+    InternalFileHierarchyStorage() ;
+
+    bool load(const std::string& fname) ;
+    bool save(const std::string& fname) ;
+
+    int parentRow(DirectoryStorage::EntryIndex e);
+    bool isIndexValid(DirectoryStorage::EntryIndex e) const;
+    bool stampDirectory(const DirectoryStorage::EntryIndex& indx);
+    bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs);
+    bool removeDirectory(DirectoryStorage::EntryIndex indx)	;
+    bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const;
+    bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files);
+    bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash);
+    bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time);
+    bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector<DirectoryStorage::EntryIndex>& subdirs_array,const std::vector<DirectoryStorage::EntryIndex>& subfiles_array);
+    bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS);
+    bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS);
+
+    // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
+
+    time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index);
+
+    // file/dir access and modification
+    bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
+
+    uint32_t mRoot ;
+    std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
+
+    void compress() ;					// use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
+                                            // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
+                                            // are not referenced.
+
+    friend class DirectoryStorage ;		// only class that can use this.
+    friend class LocalDirectoryStorage ;		// only class that can use this.
+
+    // Low level stuff. Should normally not be used externally.
+
+    const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const;
+    const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const;
+    const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const;
+    uint32_t getType(DirectoryStorage::EntryIndex indx) const;
+    DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index);
+    DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index);
+    bool searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results);
+
+    bool check(std::string& error_string) const	;// checks consistency of storage.
+
+    void print() const;
+
+private:
+    void recursPrint(int depth,DirectoryStorage::EntryIndex node) const;
+    static bool nodeAccessError(const std::string& s);
+
+    // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
+    // phase. That allows to keep file information when moving them around.
+
+    bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir);
+
+    std::map<RsFileHash,DirectoryStorage::EntryIndex> mHashes ; // used for fast search access. We should try something faster than std::map. hash_map??
+};

libretroshare/src/file_sharing/directory_storage.cc

@@ -3,622 +3,11 @@
 #include "retroshare/rspeers.h"
 #include "util/rsdir.h"
 #include "util/rsstring.h"
+#include "file_sharing_defaults.h"
 #include "directory_storage.h"
+#include "dir_hierarchy.h"
 #include "filelist_io.h"
 
-#define DEBUG_DIRECTORY_STORAGE 1
-
-/******************************************************************************************************************/
-/*                                              Internal File Hierarchy Storage                                   */
-/******************************************************************************************************************/
-
-template<class T> typename std::set<T>::iterator erase_from_set(typename std::set<T>& s,const typename std::set<T>::iterator& it)
-{
-    typename std::set<T>::iterator tmp(it);
-    ++tmp;
-    s.erase(it) ;
-    return tmp;
-}
-
-// This class handles the file hierarchy
-// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed,
-// so that the hierarchy stays completely coherent between calls.
-
-class InternalFileHierarchyStorage
-{
-public:
-    class FileStorageNode
-    {
-    public:
-        static const uint32_t TYPE_UNKNOWN = 0x0000 ;
-        static const uint32_t TYPE_FILE    = 0x0001 ;
-        static const uint32_t TYPE_DIR     = 0x0002 ;
-
-        virtual ~FileStorageNode() {}
-        virtual uint32_t type() const =0;
-
-        DirectoryStorage::EntryIndex parent_index;
-        uint32_t row ;
-    };
-    class FileEntry: public FileStorageNode
-    {
-    public:
-        FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {}
-        virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
-        virtual ~FileEntry() {}
-
-        // local stuff
-        time_t      file_modtime;
-        std::string file_name ;
-        uint64_t    file_size ;
-        RsFileHash  file_hash ;
-    };
-
-    class DirEntry: public FileStorageNode
-    {
-    public:
-        DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {}
-        virtual ~DirEntry() {}
-
-        virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
-
-        // local stuff
-        std::string dir_name ;
-        std::string dir_parent_path ;
-
-        std::vector<DirectoryStorage::EntryIndex> subdirs ;
-        std::vector<DirectoryStorage::EntryIndex> subfiles ;
-
-        time_t dir_modtime;
-        time_t most_recent_time;	// recursive most recent modification time, including files and subdirs in the entire hierarchy below.
-        time_t dir_update_time;		// last time the information was updated for that directory. Includes subdirs indexes and subfile info.
-    };
-
-    // class stuff
-    InternalFileHierarchyStorage() : mRoot(0)
-    {
-        DirEntry *de = new DirEntry("") ;
-
-        de->row=0;
-        de->parent_index=0;
-        de->dir_modtime=0;
-
-        mNodes.push_back(de) ;
-    }
-
-    int parentRow(DirectoryStorage::EntryIndex e)
-    {
-        if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0)
-            return -1 ;
-
-        return mNodes[mNodes[e]->parent_index]->row;
-    }
-
-    // high level modification routines
-
-    bool isIndexValid(DirectoryStorage::EntryIndex e) const
-    {
-        return e < mNodes.size() && mNodes[e] != NULL ;
-    }
-    bool stampDirectory(const DirectoryStorage::EntryIndex& indx)
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return false;
-
-        static_cast<DirEntry*>(mNodes[indx])->dir_modtime = time(NULL) ;
-        return true;
-    }
-
-    bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return false;
-
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
-
-        std::map<std::string,time_t> should_create(subdirs);
-
-        for(uint32_t i=0;i<d.subdirs.size();)
-            if(subdirs.find(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) == subdirs.end())
-            {
-                std::cerr << "[directory storage] Removing subdirectory " << static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl;
-
-                if( !removeDirectory(d.subdirs[i]))
-                    i++ ;
-            }
-            else
-            {
-                std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl;
-
-                should_create.erase(static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_name) ;
-                ++i;
-            }
-
-        for(std::map<std::string,time_t>::const_iterator it(should_create.begin());it!=should_create.end();++it)
-        {
-            std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl;
-
-            DirEntry *de = new DirEntry(it->first) ;
-
-            de->row = mNodes.size();
-            de->parent_index = indx;
-            de->dir_modtime = it->second;
-
-            d.subdirs.push_back(mNodes.size()) ;
-            mNodes.push_back(de) ;
-        }
-
-        return true;
-    }
-    bool removeDirectory(DirectoryStorage::EntryIndex indx)	// no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!!
-    {
-        // check that it's a directory
-
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return false;
-
-        if(indx == 0)
-            return nodeAccessError("checkIndex(): Cannot remove top level directory") ;
-
-#ifdef DEBUG_DIRECTORY_STORAGE
-        std::cerr << "(--) Removing directory " << indx << std::endl;
-                print();
-#endif
-        // remove from parent
-
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
-        DirEntry& parent_dir(*static_cast<DirEntry*>(mNodes[d.parent_index]));
-
-        for(uint32_t i=0;i<parent_dir.subdirs.size();++i)
-            if(parent_dir.subdirs[i] == indx)
-            {
-                parent_dir.subdirs[i] = parent_dir.subdirs.back() ;
-                parent_dir.subdirs.pop_back();
-
-                bool res = recursRemoveDirectory(indx) ;
-#ifdef DEBUG_DIRECTORY_STORAGE
-                print();
-                std::string err ;
-                if(!check(err))
-                    std::cerr << "(EE) Error after removing subdirectory. Error string=\"" << err << "\", Hierarchy is : " << std::endl;
-#endif
-                return true ;
-            }
-        return nodeAccessError("removeDirectory(): inconsistency!!") ;
-    }
-
-    bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const
-    {
-        if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL)
-            return nodeAccessError("checkIndex(): Node does not exist") ;
-
-        if(! (mNodes[indx]->type() & type))
-            return nodeAccessError("checkIndex(): Node is of wrong type") ;
-
-        return true;
-    }
-
-    bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return false;
-
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
-        new_files = subfiles ;
-
-        // remove from new_files the ones that already exist and have a modf time that is not older.
-
-        for(uint32_t i=0;i<d.subfiles.size();)
-        {
-            FileEntry& f(*static_cast<FileEntry*>(mNodes[d.subfiles[i]])) ;
-            std::map<std::string,DirectoryStorage::FileTS>::const_iterator it = subfiles.find(f.file_name) ;
-
-            if(it == subfiles.end())				// file does not exist anymore => delete
-            {
-                std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl;
-
-                delete mNodes[d.subfiles[i]] ;
-                mNodes[d.subfiles[i]] = NULL ;
-
-                d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ;
-                d.subfiles.pop_back();
-                continue;
-            }
-
-            if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)	// file is newer and/or has different size
-            {
-                f.file_hash.clear();																// hash needs recomputing
-                f.file_modtime = it->second.modtime;
-                f.file_size = it->second.size;
-            }
-            new_files.erase(f.file_name) ;
-
-            ++i;
-        }
-
-        for(std::map<std::string,DirectoryStorage::FileTS>::const_iterator it(new_files.begin());it!=new_files.end();++it)
-        {
-            std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl;
-
-            d.subfiles.push_back(mNodes.size()) ;
-            mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime));
-            mNodes.back()->row = mNodes.size()-1;
-            mNodes.back()->parent_index = indx;
-        }
-        return true;
-    }
-    bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
-    {
-        if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
-        {
-            std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
-            return false;
-        }
-
-        std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl;
-
-        RsFileHash& old_hash (static_cast<FileEntry*>(mNodes[file_index])->file_hash) ;
-
-        old_hash = hash ;
-
-        return true;
-    }
-    bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time)
-    {
-        if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
-        {
-            std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
-            return false;
-        }
-
-        FileEntry& fe(*static_cast<FileEntry*>(mNodes[file_index])) ;
-
-        std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl;
-
-        fe.file_hash = hash;
-        fe.file_size = size;
-        fe.file_modtime = modf_time;
-        fe.file_name = fname;
-
-        return true;
-    }
-
-    bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector<DirectoryStorage::EntryIndex>& subdirs_array,const std::vector<DirectoryStorage::EntryIndex>& subfiles_array)
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-        {
-            std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
-            return false;
-        }
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
-
-        d.most_recent_time = most_recent_time;
-        d.dir_modtime      = dir_modtime;
-        d.dir_update_time  = time(NULL);
-        d.dir_name         = dir_name;
-
-        d.subfiles = subfiles_array ;
-        d.subdirs  = subdirs_array ;
-
-        // check that all subdirs already exist. If not, create.
-        for(uint32_t i=0;i<subdirs_array.size();++i)
-        {
-            if(subdirs_array[i] >= mNodes.size() )
-                mNodes.resize(subdirs_array[i]+1,NULL);
-
-            FileStorageNode *& node(mNodes[subdirs_array[i]]);
-
-            if(node != NULL && node->type() != FileStorageNode::TYPE_DIR)
-            {
-                delete node ;
-                node = NULL ;
-            }
-
-            if(node == NULL)
-                node = new DirEntry("");
-
-            ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ;
-            node->row = i ;
-            node->parent_index = indx ;
-        }
-        for(uint32_t i=0;i<subfiles_array.size();++i)
-        {
-            if(subfiles_array[i] >= mNodes.size() )
-                mNodes.resize(subfiles_array[i]+1,NULL);
-
-            FileStorageNode *& node(mNodes[subfiles_array[i]]);
-
-            if(node != NULL && node->type() != FileStorageNode::TYPE_FILE)
-            {
-                delete node ;
-                node = NULL ;
-            }
-
-            if(node == NULL)
-                node = new FileEntry("",0,0);
-
-            node->row = subdirs_array.size()+i ;
-            node->parent_index = indx ;
-        }
-
-        return true;
-    }
-
-    bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS)
-    {
-        if(!checkIndex(index,FileStorageNode::TYPE_DIR))
-        {
-            std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl;
-            return false;
-        }
-
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
-
-        recurs_max_modf_TS = d.most_recent_time ;
-        local_update_TS    = d.dir_update_time ;
-
-        return true;
-    }
-    bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS)
-    {
-        if(!checkIndex(index,FileStorageNode::TYPE_DIR))
-        {
-            std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl;
-            return false;
-        }
-
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[index])) ;
-
-        d.most_recent_time = recurs_max_modf_TS ;
-        d.dir_update_time  = local_update_TS    ;
-
-        return true;
-    }
-
-    // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
-
-    time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index)
-    {
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[dir_index])) ;
-
-        time_t largest_modf_time = d.dir_modtime ;
-
-        for(uint32_t i=0;i<d.subfiles.size();++i)
-            largest_modf_time = std::max(largest_modf_time,static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_modtime) ;
-
-        for(uint32_t i=0;i<d.subdirs.size();++i)
-            largest_modf_time = std::max(largest_modf_time,recursUpdateLastModfTime(d.subdirs[i])) ;
-
-        d.most_recent_time = largest_modf_time ;
-
-        return largest_modf_time ;
-    }
-
-    // file/dir access and modification
-    bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ;	// returns true when s is the name of a sub-directory in the given entry e
-
-    uint32_t mRoot ;
-    std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
-
-    void compress() ;					// use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
-                                            // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
-                                            // are not referenced.
-
-    friend class DirectoryStorage ;		// only class that can use this.
-    friend class LocalDirectoryStorage ;		// only class that can use this.
-
-    // Low level stuff. Should normally not be used externally.
-
-    const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const
-    {
-        if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR))
-            return mNodes[indx] ;
-        else
-            return NULL ;
-    }
-
-    const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return NULL ;
-
-        return static_cast<DirEntry*>(mNodes[indx]) ;
-    }
-    const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const
-    {
-        if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
-            return NULL ;
-
-        return static_cast<FileEntry*>(mNodes[indx]) ;
-    }
-    uint32_t getType(DirectoryStorage::EntryIndex indx) const
-    {
-        if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR))
-            return mNodes[indx]->type() ;
-        else
-            return FileStorageNode::TYPE_UNKNOWN;
-    }
-
-    DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index)
-    {
-        if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR))
-            return DirectoryStorage::NO_INDEX;
-
-        if(static_cast<DirEntry*>(mNodes[parent_index])->subfiles.size() <= file_tab_index)
-            return DirectoryStorage::NO_INDEX;
-
-        return static_cast<DirEntry*>(mNodes[parent_index])->subfiles[file_tab_index];
-    }
-    DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index)
-    {
-        if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR))
-            return DirectoryStorage::NO_INDEX;
-
-        if(static_cast<DirEntry*>(mNodes[parent_index])->subdirs.size() <= dir_tab_index)
-            return DirectoryStorage::NO_INDEX;
-
-        return static_cast<DirEntry*>(mNodes[parent_index])->subdirs[dir_tab_index];
-    }
-
-    bool searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results)
-    {
-        std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mHashes.find(hash);
-
-        if( it != mHashes.end() )
-        {
-            results.clear();
-            results.push_back(it->second) ;
-            return true ;
-        }
-        else
-           return false;
-    }
-
-    bool check(std::string& error_string) const	// checks consistency of storage.
-    {
-        // recurs go through all entries, check that all
-
-       std::vector<uint32_t> hits(mNodes.size(),0) ;	// count hits of children. Should be 1 for all in the end. Otherwise there's an error.
-       hits[0] = 1 ;	// because 0 is never the child of anyone
-
-       for(uint32_t i=0;i<mNodes.size();++i)
-           if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR)
-           {
-               // stamp the kids
-               const DirEntry& de = *static_cast<DirEntry*>(mNodes[i]) ;
-
-               for(uint32_t j=0;j<de.subdirs.size();++j)
-               {
-                   if(de.subdirs[j] >= mNodes.size())
-                   {
-                       error_string = "Node child out of tab!" ;
-                       return false ;
-                   }
-                   if(hits[de.subdirs[j]] != 0)
-                   {
-                       error_string = "Double hit on a single node" ;
-                       return false;
-                   }
-                   hits[de.subdirs[j]] = 1;
-               }
-               for(uint32_t j=0;j<de.subfiles.size();++j)
-               {
-                   if(de.subfiles[j] >= mNodes.size())
-                   {
-                       error_string = "Node child out of tab!" ;
-                       return false ;
-                   }
-                   if(hits[de.subfiles[j]] != 0)
-                   {
-                       error_string = "Double hit on a single node" ;
-                       return false;
-                   }
-                   hits[de.subfiles[j]] = 1;
-               }
-
-           }
-
-       for(uint32_t i=0;i<hits.size();++i)
-           if(hits[i] == 0 && mNodes[i] != NULL)
-           {
-               error_string = "Orphean node!" ;
-               return false;
-           }
-
-       return true;
-    }
-
-    void print() const
-    {
-        int nfiles = 0 ;
-        int ndirs = 0 ;
-        int nempty = 0 ;
-        int nunknown = 0;
-
-        for(uint32_t i=0;i<mNodes.size();++i)
-            if(mNodes[i] == NULL)
-            {
-                std::cerr << "  Node " << i << ": empty " << std::endl;
-                ++nempty ;
-            }
-            else if(mNodes[i]->type() == FileStorageNode::TYPE_DIR)
-            {
-                std::cerr << "  Node " << i << ": type=" << mNodes[i]->type() << std::endl;
-                ++ndirs;
-            }
-            else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE)
-            {
-                std::cerr << "  Node " << i << ": type=" << mNodes[i]->type() << std::endl;
-                ++nfiles;
-            }
-            else
-            {
-                ++nunknown;
-                std::cerr << "(EE) Error: unknown type node found!" << std::endl;
-            }
-
-        std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl;
-
-        recursPrint(0,DirectoryStorage::EntryIndex(0));
-    }
-private:
-    void recursPrint(int depth,DirectoryStorage::EntryIndex node) const
-    {
-        std::string indent(2*depth,' ');
-
-        if(mNodes[node] == NULL)
-        {
-            std::cerr << "EMPTY NODE !!" << std::endl;
-            return ;
-        }
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[node]));
-
-        std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ;
-
-        for(int i=0;i<d.subdirs.size();++i)
-            std::cerr << d.subdirs[i] << " " ;
-        std::cerr << std::endl;
-
-        for(int i=0;i<d.subdirs.size();++i)
-            recursPrint(depth+1,d.subdirs[i]) ;
-
-        for(int i=0;i<d.subfiles.size();++i)
-        {
-            FileEntry& f(*static_cast<FileEntry*>(mNodes[d.subfiles[i]]));
-            std::cerr << indent << "  hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << "  " << f.file_size << "  " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl;
-        }
-    }
-
-    static bool nodeAccessError(const std::string& s)
-    {
-        std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
-        return false ;
-    }
-
-    // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
-    // phase. That allows to keep file information when moving them around.
-
-    bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir)
-    {
-        DirEntry& d(*static_cast<DirEntry*>(mNodes[dir])) ;
-
-        for(uint32_t i=0;i<d.subdirs.size();++i)
-            recursRemoveDirectory(d.subdirs[i]);
-
-        for(uint32_t i=0;i<d.subfiles.size();++i)
-        {
-            delete mNodes[d.subfiles[i]] ;
-            mNodes[d.subfiles[i]] = NULL ;
-        }
-        delete mNodes[dir] ;
-        mNodes[dir] = NULL ;
-
-        return true ;
-    }
-
-    std::map<RsFileHash,DirectoryStorage::EntryIndex> mHashes ; // used for fast search access. We should try something faster than std::map. hash_map??
-};
-
 /******************************************************************************************************************/
 /*                                                      Iterators                                                 */
 /******************************************************************************************************************/
@@ -666,8 +55,8 @@ std::string DirectoryStorage::DirIterator::name()      const { const InternalFil
 /*                                                 Directory Storage                                              */
 /******************************************************************************************************************/
 
-DirectoryStorage::DirectoryStorage(const std::string& local_file_name, const RsPeerId &pid)
+DirectoryStorage::DirectoryStorage(const RsPeerId &pid)
-    : mFileName(local_file_name),mPeerId(pid), mDirStorageMtx("Directory storage "+local_file_name)
+    : mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString())
 {
     RS_STACK_MUTEX(mDirStorageMtx) ;
     mFileHierarchy = new InternalFileHierarchyStorage();
@@ -771,10 +160,20 @@ void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_
 
 void DirectoryStorage::load(const std::string& local_file_name)
 {
+    std::cerr << "DirectoryStorage::load()" << std::endl;
+
     // first load the header, than all fields.
+
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    mFileHierarchy->load(local_file_name);
 }
 void DirectoryStorage::save(const std::string& local_file_name)
 {
+    std::cerr << "DirectoryStorage::Save()" << std::endl;
+
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    mFileHierarchy->save(local_file_name);
+
     // first write the header, than all fields.
 }
 void DirectoryStorage::print()
@@ -787,8 +186,8 @@ void DirectoryStorage::print()
 bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info)
 {
     RS_STACK_MUTEX(mDirStorageMtx) ;
-#warning todo
 
+    NOT_IMPLEMENTED();
     return true;
 }
 
@@ -1142,7 +541,24 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
 /*                                           Remote Directory Storage                                              */
 /******************************************************************************************************************/
 
-bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata)
+RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname)
+    : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname)
+{
+    load(fname) ;
+}
+
+void RemoteDirectoryStorage::checkSave()
+{
+    time_t now = time(NULL);
+
+    if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
+    {
+        save(mFileName);
+        mLastSavedTime = now ;
+    }
+}
+
+bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata)
 {
     const unsigned char *section_data = (unsigned char*)bindata.bin_data ;
     uint32_t section_size = bindata.bin_len ;
@@ -1241,6 +657,7 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs
         if(!mFileHierarchy->updateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i]))
             std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl;
     }
+    mChanged = true ;
 
     return true ;
 }

libretroshare/src/file_sharing/directory_storage.h

@@ -16,10 +16,10 @@ class RsTlvBinaryData ;
 class DirectoryStorage
 {
 	public:
-        DirectoryStorage(const std::string& local_file_name,const RsPeerId& pid) ;
+        DirectoryStorage(const RsPeerId& pid) ;
         virtual ~DirectoryStorage() {}
 
-        typedef int32_t EntryIndex ;
+        typedef uint32_t EntryIndex ;
         static const EntryIndex NO_INDEX = 0xffffffff;
 
 		void save() const ;
@@ -104,10 +104,12 @@ class DirectoryStorage
         void print();
         void cleanup();
 
-    private:
+    protected:
 		void load(const std::string& local_file_name) ;
 		void save(const std::string& local_file_name) ;
 
+    private:
+
         void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ;
         void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ;
 
@@ -116,7 +118,6 @@ class DirectoryStorage
 
         // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
 
-        std::string mFileName;
         RsPeerId mPeerId;
 
     protected:
@@ -128,7 +129,7 @@ class DirectoryStorage
 class RemoteDirectoryStorage: public DirectoryStorage
 {
 public:
-    RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {}
+    RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) ;
     virtual ~RemoteDirectoryStorage() {}
 
     /*!
@@ -140,13 +141,24 @@ public:
      * \param bindata   binary data to deserialise from
      * \return 			false when the directory cannot be found.
      */
-    bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ;
+    bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ;
+
+    /*!
+     * \brief checkSave
+     * 			Checks the time of last saving, last modification time, and saves if needed.
+     */
+    void checkSave() ;
+
+private:
+    time_t mLastSavedTime ;
+    bool mChanged ;
+    std::string mFileName;
 };
 
 class LocalDirectoryStorage: public DirectoryStorage
 {
 public:
-    LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(fname,own_id) {}
+    LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {}
     virtual ~LocalDirectoryStorage() {}
 
     void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
@@ -194,6 +206,7 @@ private:
     std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const;
 
     std::map<std::string,SharedDirInfo> mLocalDirs ;	// map is better for search. it->first=it->second.filename
+    std::string mFileName;
 };
 
 

libretroshare/src/file_sharing/file_sharing_defaults.h

@@ -10,4 +10,5 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ;		// key
 static const std::string FILE_SHARING_DIR_NAME  = "file_sharing" ;			// hard-coded directory name to store friend file lists, hash cache, etc.
 static const std::string HASH_CACHE_FILE_NAME   = "hash_cache.bin" ;		// hard-coded directory name to store encrypted hash cache.
 
-static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE      = 20 ;  // never save hash cache more often than every 60 secs.
+static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE         = 20 ;  // never save hash cache more often than every 20 secs.
+static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE   = 23 ;  // never save remote directories more often than this

libretroshare/src/file_sharing/p3filelists.cc

@@ -51,6 +51,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
 
 	mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
     mLastRemoteDirSweepTS = 0 ;
+    mLastCleanupTime = 0 ;
 
     // This is for the transmission of data
 
@@ -141,13 +142,10 @@ int p3FileDatabase::tick()
 	// 	- remove/delete shared file lists for people who are not friend anymore
 	// 	- 
 
-    static time_t last_cleanup_time = 0;
+    if(mLastCleanupTime + 5 < now)
-
-#warning we should use members here, not static
-    if(last_cleanup_time + 5 < now)
     {
         cleanup();
-        last_cleanup_time = now ;
+        mLastCleanupTime = now ;
     }
 
     static time_t last_print_time = 0;
@@ -186,6 +184,7 @@ int p3FileDatabase::tick()
         mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ;
 
         for(uint32_t i=0;i<mRemoteDirectories.size();++i)
+        {
             if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end())
             {
                 std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl;
@@ -196,6 +195,9 @@ int p3FileDatabase::tick()
                 locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ;
             }
 
+            mRemoteDirectories[i]->checkSave() ;
+        }
+
         mLastRemoteDirSweepTS = now;
     }
 
@@ -440,8 +442,7 @@ void p3FileDatabase::cleanup()
 
 std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const
 {
-#warning we should use the default paths here. Ask p3config
+    return mFileSharingDir + "/" + "dirlist_"+pid.toStdString()+".bin" ;
-    return "dirlist_"+pid.toStdString()+".txt" ;
 }
 
 uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
@@ -998,7 +999,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item)
     {
         P3FILELISTS_DEBUG() << "  Item contains directory data. Deserialising/Updating." << std::endl;
 
-        if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data))
+        if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data))
             RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);						 // notify the GUI if the hierarchy has changed
         else
             std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl;

libretroshare/src/file_sharing/p3filelists.h

@@ -196,5 +196,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         mutable RsMutex mFLSMtx ;
         uint32_t mUpdateFlags ;
         std::string mFileSharingDir ;
+        time_t mLastCleanupTime;
 };
 

libretroshare/src/libretroshare.pro

@@ -46,6 +46,7 @@ file_lists {
 			file_sharing/directory_storage.h \
 			file_sharing/directory_updater.h \
 			file_sharing/rsfilelistitems.h \
+			file_sharing/dir_hierarchy.h \
 			file_sharing/file_sharing_defaults.h
 
 	SOURCES *= file_sharing/p3filelists.cc \
@@ -53,6 +54,7 @@ file_lists {
 			file_sharing/filelist_io.cc \
 			file_sharing/directory_storage.cc \
 			file_sharing/directory_updater.cc \
+			file_sharing/dir_hierarchy.cc \
 			file_sharing/rsfilelistitems.cc
 }