diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 5765dd394..0ad931077 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -285,7 +285,26 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex return true; } -bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_hash,const std::vector& subfiles_hash) +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex() +{ + int found = -1; + for(uint32_t j=0;j& subdirs_hash,const std::vector& subfiles_array) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) { @@ -301,118 +320,131 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.dir_update_time = time(NULL); d.dir_name = dir_name; + std::map existing_subdirs ; + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_hash] = d.subdirs[i] ; + d.subdirs.clear(); - d.subfiles.clear(); // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::iterator it = existing_subdirs.find(subdirs_hash[i]) ; DirectoryStorage::EntryIndex dir_index = 0; - std::map::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; - if(it == mDirHashes.end() || it->second >= mNodes.size()) + if(it != existing_subdirs.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_DIR) { - // find an epty slot - int found = -1 ; + dir_index = it->second ; - for(uint32_t j=0;jdir_parent_path = d.dir_parent_path + "/" + dir_name ; + de->dir_hash = subdirs_hash[i]; mDirHashes[subdirs_hash[i]] = dir_index ; std::cerr << " created, at new index " << dir_index << std::endl; } - else - { - dir_index = it->second; - - if(mNodes[dir_index] != NULL && mNodes[dir_index]->type() != FileStorageNode::TYPE_DIR) - { - delete mNodes[dir_index] ; - mNodes[dir_index] = NULL ; - } - std::cerr << " already exists, index=" << dir_index << "." << std::endl; - } - FileStorageNode *& node(mNodes[dir_index]); - if(!node) - node = new DirEntry(""); d.subdirs.push_back(dir_index) ; - - ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; - ((DirEntry*&)node)->dir_hash = subdirs_hash[i]; - node->row = i ; - node->parent_index = indx ; + mDirHashes[subdirs_hash[i]] = dir_index ; } - for(uint32_t i=0;i::const_iterator it = existing_subdirs.begin();it!=existing_subdirs.end();++it) { - DirectoryStorage::EntryIndex file_index = 0; - std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; - std::cerr << " subfile hash " << i << ": " << subfiles_hash[i] ; - - if(it == mFileHashes.end()) + if(!checkIndex(it->second,FileStorageNode::TYPE_DIR)) { - // find an epty slot - int found = -1; - - for(uint32_t j=0;jsecond << " because it is not a file. Inconsistency error!" << std::endl; + continue ; } - else + recursRemoveDirectory(it->second) ; + } + + // now update subfiles. This is more stricky because we need to not suppress hash duplicates + + std::map existing_subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name] = d.subfiles[i] ; + + d.subfiles.clear(); + + for(uint32_t i=0;i::iterator it = existing_subfiles.find(subfiles_array[i].file_name) ; + const FileEntry& f(subfiles_array[i]) ; + DirectoryStorage::EntryIndex file_index ; + + std::cerr << " subfile name = " << subfiles_array[i].file_name << ": " ; + + if(it != existing_subfiles.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_FILE) { file_index = it->second ; - if(mNodes[file_index] != NULL && mNodes[file_index]->type() != FileStorageNode::TYPE_FILE) - { - delete mNodes[file_index] ; - mNodes[file_index] = NULL ; - } + std::cerr << " already exists, at index " << file_index << std::endl; - file_index = it->second; + if(!updateFile(file_index,f.file_hash,f.file_name,f.file_size,f.file_modtime)) + std::cerr << "(EE) Cannot update file with index " << it->second <<" and hash " << f.file_hash << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - std::cerr << " already exists, index=" << file_index << "." << std::endl; + existing_subfiles.erase(it) ; } - FileStorageNode *& node(mNodes[file_index]); + else + { + file_index = allocateNewIndex() ; - if(!node) - node = new FileEntry("",0,0,subfiles_hash[i]); + mNodes[file_index] = new FileEntry(f.file_name,f.file_size,f.file_modtime,f.file_hash) ; + mFileHashes[f.file_hash] = file_index ; + + std::cerr << " created, at new index " << file_index << std::endl; + } d.subfiles.push_back(file_index) ; - - node->row = subdirs_hash.size()+i ; - node->parent_index = indx ; } + // remove subfiles that do not exist anymore + + for(std::map::const_iterator it = existing_subfiles.begin();it!=existing_subfiles.end();++it) + { + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; + + if(!checkIndex(it->second,FileStorageNode::TYPE_FILE)) + { + std::cerr << "(EE) Cannot delete node of index " << it->second << " because it is not a file. Inconsistency error!" << std::endl; + continue ; + } + delete mNodes[it->second] ; + mNodes[it->second] = NULL ; + } + + // now update row and parent index for all subnodes + + uint32_t n=0; + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subdirs[i]]->row = n++ ; + } + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subfiles[i]]->row = n++ ; + } + return true; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 7d7eb35af..33c5a562e 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -25,6 +25,7 @@ public: class FileEntry: public FileStorageNode { public: + FileEntry() : file_size(0), file_modtime(0) {} FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {} @@ -74,7 +75,7 @@ public: bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_hash); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_array); bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); @@ -120,14 +121,19 @@ private: static bool nodeAccessError(const std::string& s); static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; + // Allocates a new entry in mNodes, possible re-using an empty slot and returns its index. + + DirectoryStorage::EntryIndex allocateNewIndex(); + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning // phase. That allows to keep file information when moving them around. bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // Map of the hash of all files. The file hashes are the sha1sum of the file data. // is used for fast search access for FT. // Note: We should try something faster than std::map. hash_map?? + // Unlike directories, multiple files may have the same hash. So this cannot be used for anything else than FT. std::map mFileHashes ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index fae385d24..933660986 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -671,11 +671,8 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co } // deserialise directory subfiles, with info for each of them - std::vector subfiles_array ; - std::vector subfiles_name ; - std::vector subfiles_size ; - std::vector subfiles_hash ; - std::vector subfiles_modtime ; + + std::vector subfiles_array ; for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_hash)) + if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array)) { std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; return false ; } - // Then update the subfiles - for(uint32_t i=0;igetIndexFromFileHash(subfiles_hash[i],file_index)) - { - std::cerr << "(EE) Cannot obtain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; - continue; - } - std::cerr << " updating file entry " << subfiles_hash[i] << std::endl; - - if(!mFileHierarchy->updateFile(file_index,subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) - std::cerr << "(EE) Cannot update file with index " << file_index <<" and hash " << subfiles_hash[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - } mChanged = true ; return true ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 07f3cd30b..c7b659a85 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1106,6 +1106,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { +#warning needs to be improved. It's quite likely that random_bias and then e can be bruteforced from the result of this function static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; @@ -1117,11 +1118,8 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId r ^= (0x011933ff92892a94 * e + peer_id.toByteArray()[i] * 0x1001fff92ee640f9) ; r <<= 8 ; r ^= 0xf392843890321808; - - std::cerr << std::hex << "r=" << r << std::endl; } - std::cerr << "making request ID: peer_id=" << peer_id << ", e=" << e << ", random_bias=" << std::hex<< random_bias << " returning " << (r^random_bias) << std::dec << std::endl; return r ^ random_bias; }