Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-02 14:16:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed conflicts in merging upstream/master to v0.6-FTEncryption

Browse source
This commit is contained in:
mr-alice 2016-11-01 14:23:13 +01:00
commit 8c7c7647b0
173 changed files with 2701 additions and 13742 deletions
Download patch file Download diff file Expand all files Collapse all files

30
libretroshare/src/file_sharing/dir_hierarchy.cc
View file

@ -149,7 +149,7 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
return e < mNodes.size() && mNodes[e] != NULL ;
}
bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs,const RsFileHash& random_hash_seed)
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
@ -188,9 +188,9 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage
de->row = mNodes.size();
de->parent_index = indx;
de->dir_modtime = it->second;
de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ;
de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ;
de->dir_modtime = 0;// forces parsing.it->second;
de->dir_parent_path = RsDirUtil::makePath(d.dir_parent_path, d.dir_name) ;
de->dir_hash = createDirHash(de->dir_name,d.dir_hash,random_hash_seed) ;
mDirHashes[de->dir_hash] = mNodes.size() ;
@ -201,7 +201,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage
return true;
}
RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_name*/, const std::string &/*dir_parent_path*/)
RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name, const RsFileHash& dir_parent_hash, const RsFileHash& random_hash_salt)
{
// What we need here: a unique identifier
// - that cannot be bruteforced to find the real directory name and path
@ -215,7 +215,19 @@ RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_
// Option 3: just compute something random, but then we need to store it so as to not
// confuse friends when restarting.
return RsFileHash::random();
RsTemporaryMemory mem(dir_name.size() + 2*RsFileHash::SIZE_IN_BYTES) ;
memcpy( mem, random_hash_salt.toByteArray(),RsFileHash::SIZE_IN_BYTES) ;
memcpy(&mem[ RsFileHash::SIZE_IN_BYTES], dir_parent_hash.toByteArray(),RsFileHash::SIZE_IN_BYTES) ;
memcpy(&mem[2*RsFileHash::SIZE_IN_BYTES],dir_name.c_str(), dir_name.size()) ;
RsFileHash res = RsDirUtil::sha1sum( mem,mem.size() ) ;
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "Creating new dir hash for dir " << dir_name << ", parent dir hash=" << dir_parent_hash << " seed=[hidden]" << " result is " << res << std::endl;
#endif
return res ;
}
bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!!
@ -434,7 +446,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI
mNodes[dir_index] = de ;
de->dir_parent_path = d.dir_parent_path + "/" + dir_name ;
de->dir_parent_path = RsDirUtil::makePath(d.dir_parent_path, dir_name) ;
de->dir_hash = subdirs_hash[i];
mDirHashes[subdirs_hash[i]] = dir_index ;
@ -675,7 +687,7 @@ public:
inline virtual uint64_t file_size() const { return mFe.file_size ; }
inline virtual const RsFileHash& file_hash() const { return mFe.file_hash ; }
inline virtual time_t file_modtime() const { return mFe.file_modtime ; }
inline virtual std::string file_parent_path()const { return mDe.dir_parent_path + "/" + mDe.dir_name ; }
inline virtual std::string file_parent_path()const { return RsDirUtil::makePath(mDe.dir_parent_path, mDe.dir_name) ; }
inline virtual uint32_t file_popularity() const { NOT_IMPLEMENTED() ; return 0; }
private:
@ -1123,7 +1135,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname)
}
catch(read_error& e)
{
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "Error while reading: " << e.what() << std::endl;
#endif
if(buffer != NULL)
free(buffer) ;

4
libretroshare/src/file_sharing/dir_hierarchy.h
View file

@ -95,7 +95,7 @@ public:
int parentRow(DirectoryStorage::EntryIndex e);
bool isIndexValid(DirectoryStorage::EntryIndex e) const;
bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const;
bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs);
bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::map<std::string,time_t>& subdirs, const RsFileHash &random_hash_seed);
bool removeDirectory(DirectoryStorage::EntryIndex indx) ;
bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const;
bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files);
@ -153,7 +153,7 @@ public:
private:
void recursPrint(int depth,DirectoryStorage::EntryIndex node) const;
static bool nodeAccessError(const std::string& s);
static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ;
static RsFileHash createDirHash(const std::string& dir_name, const RsFileHash &dir_parent_hash, const RsFileHash &random_hash_salt) ;
// Allocates a new entry in mNodes, possible re-using an empty slot and returns its index.

13
libretroshare/src/file_sharing/directory_storage.cc
View file

@ -127,10 +127,10 @@ bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update
bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); }
bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); }
bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map<std::string,time_t>& subdirs,const RsFileHash& hash_salt)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ;
bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs,hash_salt) ;
locked_check() ;
return res ;
}
@ -168,10 +168,10 @@ bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash
return mFileHierarchy->updateHash(index,hash);
}
void DirectoryStorage::load(const std::string& local_file_name)
bool DirectoryStorage::load(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->load(local_file_name);
return mFileHierarchy->load(local_file_name);
}
void DirectoryStorage::save(const std::string& local_file_name)
{
@ -238,7 +238,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
d.min_age = now - dir_entry->dir_most_recent_time ;
d.age = now - dir_entry->dir_modtime ;
d.name = dir_entry->dir_name;
d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ;
d.path = RsDirUtil::makePath(dir_entry->dir_parent_path, dir_entry->dir_name) ;
d.parent = (void*)(intptr_t)dir_entry->parent_index ;
if(indx == 0)
@ -262,7 +262,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index);
if(parent_dir_entry != NULL)
d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ;
d.path = RsDirUtil::makePath(parent_dir_entry->dir_parent_path, parent_dir_entry->dir_name) ;
else
d.path = "" ;
}
@ -770,6 +770,7 @@ void RemoteDirectoryStorage::checkSave()
{
save(mFileName);
mLastSavedTime = now ;
mChanged = false ;
}
}

4
libretroshare/src/file_sharing/directory_storage.h
View file

@ -132,7 +132,7 @@ class DirectoryStorage
// Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with
// just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache.
//
bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::string, time_t> &subdirs) ;
bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::string, time_t> &subdirs, const RsFileHash &random_hash_salt) ;
bool updateSubFilesList(const EntryIndex& indx, const std::map<std::string, FileTS> &subfiles, std::map<std::string, FileTS> &new_files) ;
bool removeDirectory(const EntryIndex& indx) ;
@ -153,7 +153,7 @@ class DirectoryStorage
void cleanup();
protected:
void load(const std::string& local_file_name) ;
bool load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;
private:

118
libretroshare/src/file_sharing/directory_updater.cc
View file

@ -55,7 +55,7 @@ void LocalDirectoryUpdater::setEnabled(bool b)
return ;
if(b)
start("fs dir updater") ;
start("fs dir updater") ;
else
shutdown();
@ -88,17 +88,23 @@ void LocalDirectoryUpdater::forceUpdate()
void LocalDirectoryUpdater::sweepSharedDirectories()
{
if(mHashSalt.isNull())
{
std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
return ;
}
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
@ -109,7 +115,7 @@ void LocalDirectoryUpdater::sweepSharedDirectories()
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
// now for each of them, go recursively and match both files and dirs
@ -137,59 +143,71 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
librs::util::FolderIterator dirIt(cumulated_path);
// collect subdirs and subfiles
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::map<std::string,time_t> subdirs ;
for(;dirIt.isValid();dirIt.next())
time_t dir_local_mod_time ;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
return;
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
if(dirIt.dir_modtime() != dir_local_mod_time)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
// collect subdirs and subfiles
RsFileHash hash ;
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::map<std::string,time_t> subdirs ;
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash)
mSharedDirectories->updateHash(*dit,hash);
for(;dirIt.isValid();dirIt.next())
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
RsFileHash hash ;
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash)
mSharedDirectories->updateHash(*dit,hash);
}
}
#ifdef DEBUG_LOCAL_DIR_UPDATER
else
std::cerr << " directory is unchanged. Keeping existing files and subdirs list." << std::endl;
#endif
// go through the list of sub-dirs and recursively update
DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ;
for(std::map<std::string,time_t>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;

5
libretroshare/src/file_sharing/directory_updater.h
View file

@ -41,6 +41,9 @@ public:
void forceUpdate();
bool inDirectoryCheck() const ;
void setHashSalt(const RsFileHash& hash) { mHashSalt = hash; }
const RsFileHash& hashSalt() const { return mHashSalt; }
void setFileWatchPeriod(int seconds) ;
uint32_t fileWatchPeriod() const ;
@ -60,6 +63,8 @@ private:
HashStorage *mHashCache ;
LocalDirectoryStorage *mSharedDirectories ;
RsFileHash mHashSalt ;
time_t mLastSweepTime;
time_t mLastTSUpdateTime;

1
libretroshare/src/file_sharing/file_sharing_defaults.h
View file

@ -33,6 +33,7 @@ static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 se
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.

2
libretroshare/src/file_sharing/filelist_io.cc
View file

@ -190,7 +190,9 @@ bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned cha
if(!RsDirUtil::checkFile( fname,file_size,false ) )
{
#ifdef FIM_DEBUG
std::cerr << "Encrypted file " << fname << " not available." << std::endl;
#endif
return false;
}

4
libretroshare/src/file_sharing/hash_cache.cc
View file

@ -76,7 +76,7 @@ void HashStorage::data_tick()
{
FileHashJob job;
RsFileHash hash;
uint64_t size ;
uint64_t size = 0;
{
bool empty ;
@ -270,7 +270,7 @@ void HashStorage::clean()
#endif
for(std::map<std::string,HashStorageInfo>::iterator it(mFiles.begin());it!=mFiles.end();)
if(it->second.time_stamp + duration < (uint64_t)now)
if((uint64_t)(it->second.time_stamp + duration) < (uint64_t)now)
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ;

35
libretroshare/src/file_sharing/p3filelists.cc
View file

@ -224,7 +224,7 @@ int p3FileDatabase::tick()
mLastRemoteDirSweepTS = now;
// This is a hash to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up,
// This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up,
// so the first notify is ignored, and no other notify will happen next.
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
@ -326,7 +326,14 @@ cleanup = true;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
kv.key = WATCH_HASH_SALT_SS;
kv.value = mLocalDirWatcher->hashSalt().toStdString();
rskv->tlvkvs.pairs.push_back(kv);
}
/* Add KeyValue to saveList */
sList.push_back(rskv);
@ -373,6 +380,11 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
{
setWatchEnabled(kit->value == "YES") ;
}
else if(kit->key == WATCH_HASH_SALT_SS)
{
std::cerr << "Initing directory watcher with saved secret salt..." << std::endl;
mLocalDirWatcher->setHashSalt(RsFileHash(kit->value)) ;
}
delete *it ;
continue ;
}
@ -397,6 +409,14 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
delete *it ;
}
if(mLocalDirWatcher->hashSalt().isNull())
{
std::cerr << "(WW) Initialising directory watcher salt to some random value " << std::endl;
mLocalDirWatcher->setHashSalt(RsFileHash::random()) ;
IndicateConfigChanged();
}
/* set directories */
mLocalSharedDirs->setSharedDirectoryList(dirList);
@ -620,12 +640,9 @@ void p3FileDatabase::requestDirUpdate(void *ref)
}
}
bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const
bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result,
FileSearchFlags flags ) const
{
RS_STACK_MUTEX(mFLSMtx) ;
result = NULL ;
if (ref == NULL)
{
if(flags & RS_FILE_HINTS_LOCAL)
@ -668,8 +685,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe
return res;
}
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
{
RS_STACK_MUTEX(mFLSMtx) ;
@ -1130,7 +1147,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
P3FILELISTS_ERROR() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#endif
return;
}