Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity
c3c2671db4
RetroShare/libretroshare/src/file_sharing/p3filelists.cc

1628 lines
54 KiB
C++
Raw Blame History

/*
* RetroShare File lists service.
*
* file_sharing/p3filelists.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "serialiser/rsserviceids.h"
#include "file_sharing/p3filelists.h"
#include "file_sharing/directory_storage.h"
#include "file_sharing/directory_updater.h"
#include "file_sharing/rsfilelistitems.h"
#include "file_sharing/file_sharing_defaults.h"
#include "retroshare/rsids.h"
#include "retroshare/rspeers.h"
#include "rsserver/rsaccounts.h"
#include "rsserver/p3face.h"
#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : "
#define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : "
//#define DEBUG_P3FILELISTS 1
static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ;
static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ;
static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ;
static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ;
p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
: mServCtrl(mpeers), mFLSMtx("p3FileLists")
{
// make sure the base directory exists
std::string base_dir = rsAccounts->PathAccountDirectory();
if(base_dir.empty())
throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ;
mFileSharingDir = base_dir + "/" + FILE_SHARING_DIR_NAME ;
if(!RsDirUtil::checkCreateDirectory(mFileSharingDir))
throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ;
// loads existing indexes for friends. Some might be already present here.
//
mRemoteDirectories.clear() ; // we should load them!
mOwnId = mpeers->getOwnId() ;
mLocalSharedDirs = new LocalDirectoryStorage(mFileSharingDir + "/" + LOCAL_SHARED_DIRS_FILE_NAME,mOwnId);
mHashCache = new HashStorage(mFileSharingDir + "/" + HASH_CACHE_FILE_NAME) ;
mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ;
mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
mLastRemoteDirSweepTS = 0 ;
mLastCleanupTime = 0 ;
mLastDataRecvTS = 0 ;
// This is for the transmission of data
addSerialType(new RsFileListsSerialiser()) ;
}
RsSerialiser *p3FileDatabase::setupSerialiser()
{
// This one is for saveList/loadList
RsSerialiser *rss = new RsSerialiser ;
rss->addSerialType(new RsFileListsSerialiser()) ;
rss->addSerialType(new RsGeneralConfigSerialiser());
rss->addSerialType(new RsFileConfigSerialiser());
return rss ;
}
void p3FileDatabase::setSharedDirectories(const std::list<SharedDirInfo>& shared_dirs)
{
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ;
mLocalDirWatcher->forceUpdate();
}
IndicateConfigChanged();
}
void p3FileDatabase::getSharedDirectories(std::list<SharedDirInfo>& shared_dirs)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ;
}
void p3FileDatabase::updateShareFlags(const SharedDirInfo& info)
{
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->updateShareFlags(info) ;
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
IndicateConfigChanged();
}
p3FileDatabase::~p3FileDatabase()
{
RS_STACK_MUTEX(mFLSMtx) ;
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
delete mRemoteDirectories[i];
mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around.
delete mLocalSharedDirs ;
delete mLocalDirWatcher ;
delete mHashCache ;
}
const std::string FILE_DB_APP_NAME = "file_database";
const uint16_t FILE_DB_APP_MAJOR_VERSION = 1;
const uint16_t FILE_DB_APP_MINOR_VERSION = 0;
const uint16_t FILE_DB_MIN_MAJOR_VERSION = 1;
const uint16_t FILE_DB_MIN_MINOR_VERSION = 0;
RsServiceInfo p3FileDatabase::getServiceInfo()
{
return RsServiceInfo(RS_SERVICE_TYPE_FILE_DATABASE,
FILE_DB_APP_NAME,
FILE_DB_APP_MAJOR_VERSION,
FILE_DB_APP_MINOR_VERSION,
FILE_DB_MIN_MAJOR_VERSION,
FILE_DB_MIN_MINOR_VERSION);
}
int p3FileDatabase::tick()
{
// tick the input/output list of update items and process them
//
tickRecv() ;
tickSend() ;
time_t now = time(NULL) ;
// cleanup
// - remove/delete shared file lists for people who are not friend anymore
// -
if(mLastCleanupTime + 5 < now)
{
cleanup();
mLastCleanupTime = now ;
}
static time_t last_print_time = 0;
if(last_print_time + 20 < now)
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_FILE_HIERARCHY
mLocalSharedDirs->print();
#endif
last_print_time = now ;
#warning mr-alice 2016-08-19: This should be removed, but it's necessary atm for updating the GUI
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
}
if(mUpdateFlags)
{
IndicateConfigChanged();
if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED)
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED)
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
}
if(mLastRemoteDirSweepTS + 5 < now)
{
RS_STACK_MUTEX(mFLSMtx) ;
std::set<RsPeerId> online_peers ;
mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ;
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] != NULL)
{
if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end() && mRemoteDirectories[i]->lastSweepTime() + DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP < now)
{
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl;
mRemoteDirectories[i]->print();
#endif
locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root(),0) ;
mRemoteDirectories[i]->lastSweepTime() = now ;
}
mRemoteDirectories[i]->checkSave() ;
}
mLastRemoteDirSweepTS = now;
mLocalSharedDirs->checkSave() ;
// This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up,
// so the first notify is ignored, and no other notify will happen next. We only do it if no data was received in the last 5 secs, in order to
// avoid syncing the GUI at every dir sync which kills performance.
if(mLastDataRecvTS + 5 < now && mLastDataRecvTS + 20 > now)
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed
}
return 0;
}
void p3FileDatabase::startThreads()
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ;
#endif
mLocalDirWatcher->start("fs dir watcher");
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Done." << std::endl;
#endif
}
void p3FileDatabase::stopThreads()
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ;
#endif
mHashCache->fullstop();
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Done." << std::endl;
P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ;
#endif
mLocalDirWatcher->fullstop();
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Done." << std::endl;
#endif
}
bool p3FileDatabase::saveList(bool &cleanup, std::list<RsItem *>& sList)
{
cleanup = true;
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "Save list" << std::endl;
#endif
/* get list of directories */
std::list<SharedDirInfo> dirList;
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->getSharedDirectoryList(dirList);
}
{
RS_STACK_MUTEX(mFLSMtx) ;
for(std::list<SharedDirInfo>::iterator it = dirList.begin(); it != dirList.end(); ++it)
{
RsFileConfigItem *fi = new RsFileConfigItem();
fi->file.path = (*it).filename ;
fi->file.name = (*it).virtualname ;
fi->flags = (*it).shareflags.toUInt32() ;
for(std::list<RsNodeGroupId>::const_iterator it2( (*it).parent_groups.begin());it2!=(*it).parent_groups.end();++it2)
fi->parent_groups.ids.insert(*it2) ;
sList.push_back(fi);
}
}
RsConfigKeyValueSet *rskv = new RsConfigKeyValueSet();
/* basic control parameters */
{
RS_STACK_MUTEX(mFLSMtx) ;
std::string s ;
rs_sprintf(s, "%lu", mHashCache->rememberHashFilesDuration()) ;
RsTlvKeyValue kv;
kv.key = HASH_CACHE_DURATION_SS;
kv.value = s ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
std::string s ;
rs_sprintf(s, "%d", watchPeriod()) ;
RsTlvKeyValue kv;
kv.key = WATCH_FILE_DURATION_SS;
kv.value = s ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
kv.key = FOLLOW_SYMLINKS_SS;
kv.value = followSymLinks()?"YES":"NO" ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
kv.key = WATCH_FILE_ENABLED_SS;
kv.value = watchEnabled()?"YES":"NO" ;
rskv->tlvkvs.pairs.push_back(kv);
}
{
RsTlvKeyValue kv;
kv.key = WATCH_HASH_SALT_SS;
kv.value = mLocalDirWatcher->hashSalt().toStdString();
rskv->tlvkvs.pairs.push_back(kv);
}
/* Add KeyValue to saveList */
sList.push_back(rskv);
return true;
}
bool p3FileDatabase::loadList(std::list<RsItem *>& load)
{
/* for each item, check it exists ....
* - remove any that are dead (or flag?)
*/
static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_PERMISSIONS_MASK;
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "Load list" << std::endl;
#endif
std::list<SharedDirInfo> dirList;
for(std::list<RsItem *>::iterator it = load.begin(); it != load.end(); ++it)
{
RsConfigKeyValueSet *rskv ;
if (NULL != (rskv = dynamic_cast<RsConfigKeyValueSet *>(*it)))
{
/* make into map */
std::map<std::string, std::string> configMap;
std::map<std::string, std::string>::const_iterator mit ;
for(std::list<RsTlvKeyValue>::const_iterator kit = rskv->tlvkvs.pairs.begin(); kit != rskv->tlvkvs.pairs.end(); ++kit)
if (kit->key == HASH_CACHE_DURATION_SS)
{
uint32_t t=0 ;
if(sscanf(kit->value.c_str(),"%d",&t) == 1)
mHashCache->setRememberHashFilesDuration(t);
}
else if(kit->key == WATCH_FILE_DURATION_SS)
{
int t=0 ;
if(sscanf(kit->value.c_str(),"%d",&t) == 1)
setWatchPeriod(t);
}
else if(kit->key == FOLLOW_SYMLINKS_SS)
{
setFollowSymLinks(kit->value == "YES") ;
}
else if(kit->key == WATCH_FILE_ENABLED_SS)
{
setWatchEnabled(kit->value == "YES") ;
}
else if(kit->key == WATCH_HASH_SALT_SS)
{
std::cerr << "Initing directory watcher with saved secret salt..." << std::endl;
mLocalDirWatcher->setHashSalt(RsFileHash(kit->value)) ;
}
delete *it ;
continue ;
}
RsFileConfigItem *fi = dynamic_cast<RsFileConfigItem *>(*it);
if (fi)
{
/* ensure that it exists? */
SharedDirInfo info ;
info.filename = RsDirUtil::convertPathToUnix(fi->file.path);
info.virtualname = fi->file.name;
info.shareflags = FileStorageFlags(fi->flags) ;
info.shareflags &= PERMISSION_MASK ;
for(std::set<RsNodeGroupId>::const_iterator itt(fi->parent_groups.ids.begin());itt!=fi->parent_groups.ids.end();++itt)
info.parent_groups.push_back(*itt) ;
dirList.push_back(info) ;
}
delete *it ;
}
/* set directories */
mLocalSharedDirs->setSharedDirectoryList(dirList);
load.clear() ;
return true;
}
void p3FileDatabase::cleanup()
{
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl;
#endif
// look through the list of friend directories. Remove those who are not our friends anymore.
//
std::set<RsPeerId> friend_set ;
{
std::list<RsPeerId> friend_lst ;
rsPeers->getFriendList(friend_lst);
for(std::list<RsPeerId>::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it)
friend_set.insert(*it) ;
}
time_t now = time(NULL);
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] != NULL)
{
time_t recurs_mod_time ;
mRemoteDirectories[i]->getDirectoryRecursModTime(0,recurs_mod_time) ;
time_t last_contact = 0 ;
RsPeerDetails pd ;
if(rsPeers->getPeerDetails(mRemoteDirectories[i]->peerId(),pd))
last_contact = pd.lastConnect ;
// We remove directories in the following situations:
// - the peer is not a friend
// - the dir list is non empty but the peer is offline since more than 60 days
// - the dir list is empty and the peer is ffline since more than 5 days
bool should_remove = friend_set.find(mRemoteDirectories[i]->peerId()) == friend_set.end()
|| (recurs_mod_time == 0 && last_contact + DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR < now )
|| (recurs_mod_time != 0 && last_contact + DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR < now );
if(!should_remove)
continue ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl;
#endif
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
friend_set.erase(mRemoteDirectories[i]->peerId());
mFriendIndexMap.erase(mRemoteDirectories[i]->peerId());
// also remove the existing file
remove(mRemoteDirectories[i]->filename().c_str()) ;
delete mRemoteDirectories[i];
mRemoteDirectories[i] = NULL ;
// now, in order to avoid empty seats, just move the last one here, and update indexes
while(i < mRemoteDirectories.size() && mRemoteDirectories[i] == NULL)
{
mRemoteDirectories[i] = mRemoteDirectories.back();
mRemoteDirectories.pop_back();
}
if(i < mRemoteDirectories.size() && mRemoteDirectories[i] != NULL) // this test is needed in the case we have deleted the last element
mFriendIndexMap[mRemoteDirectories[i]->peerId()] = i;
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ;
}
// look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated.
//
for(std::set<RsPeerId>::const_iterator it(friend_set.begin());it!=friend_set.end();++it)
{
// Check if a remote directory exists for that friend, possibly creating the index if the file does not but the friend is online.
if(rsPeers->isOnline(*it) || RsDirUtil::fileExists(makeRemoteFileName(*it)))
locked_getFriendIndex(*it) ;
}
// cancel existing requests for which the peer is offline
std::set<RsPeerId> online_peers ;
mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ;
for(std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();)
if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl;
#endif
std::map<DirSyncRequestId,DirSyncRequestData>::iterator tmp(it);
++tmp;
mPendingSyncRequests.erase(it) ;
it = tmp;
}
else
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl;
#endif
++it ;
}
// This is needed at least here, because loadList() might never have been called, if there is no config file present.
if(mLocalDirWatcher->hashSalt().isNull())
{
std::cerr << "(WW) Initialising directory watcher salt to some random value " << std::endl;
mLocalDirWatcher->setHashSalt(RsFileHash::random()) ;
IndicateConfigChanged();
}
}
}
std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const
{
return mFileSharingDir + "/" + "dirlist_"+pid.toStdString()+".bin" ;
}
uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
{
std::map<RsPeerId,uint32_t>::const_iterator it = mFriendIndexMap.find(pid) ;
if(it == mFriendIndexMap.end())
{
// allocate a new index for that friend, and tell that we should save.
uint32_t found = 0 ;
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] == NULL)
{
found = i ;
break;
}
if(!found)
{
found = mRemoteDirectories.size();
mRemoteDirectories.push_back(new RemoteDirectoryStorage(pid,makeRemoteFileName(pid)));
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ;
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << pid << ", with index " << it->second << std::endl;
#endif
}
if(mRemoteDirectories.size() >= (1 << NB_FRIEND_INDEX_BITS) )
{
std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<<NB_FRIEND_INDEX_BITS) << " friends??" << std::endl;
return 1 << NB_FRIEND_INDEX_BITS ;
}
mFriendIndexMap[pid] = found;
IndicateConfigChanged();
return found ;
}
else
{
if(mRemoteDirectories.size() <= it->second)
{
mRemoteDirectories.resize(it->second+1,NULL) ;
mRemoteDirectories[it->second] = new RemoteDirectoryStorage(pid,makeRemoteFileName(pid));
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ;
mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << pid << ", with index " << it->second << std::endl;
#endif
}
return it->second;
}
}
bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index)
{
// trust me, I can do this ;-)
e = EntryIndex( *reinterpret_cast<uint32_t*>(&p) & ENTRY_INDEX_BIT_MASK ) ;
friend_index = (*reinterpret_cast<uint32_t*>(&p)) >> NB_ENTRY_INDEX_BITS ;
if(friend_index == 0)
{
std::cerr << "(EE) Cannot find friend index in pointer. Encoded value is zero!" << std::endl;
return false;
}
friend_index--;
return true;
}
bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi, void *& p)
{
// the pointer is formed the following way:
//
// [ 10 bits | 22 bits ]
//
// This means that the whoel software has the following build-in limitation:
// * 1023 friends
// * 4M shared files.
uint32_t fe = (uint32_t)e ;
if(fi+1 >= (1<<NB_FRIEND_INDEX_BITS) || fe >= (1<< NB_ENTRY_INDEX_BITS))
{
P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl;
return false ;
}
p = reinterpret_cast<void*>( ( (1+fi) << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ;
return true;
}
void p3FileDatabase::requestDirUpdate(void *ref)
{
uint32_t fi;
DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi);
if(fi == 0)
return ; // not updating current directory (should we?)
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl;
#endif
RS_STACK_MUTEX(mFLSMtx) ;
if(locked_generateAndSendSyncRequest(mRemoteDirectories[fi-1],e))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Succeed." << std::endl;
#endif
}
}
bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result,
FileSearchFlags flags ) const
{
if (ref == NULL)
{
if(flags & RS_FILE_HINTS_LOCAL)
{
if(row != 0)
return false ;
convertEntryIndexToPointer(0,0,result);
return true ;
}
else if((uint32_t)row < mRemoteDirectories.size())
{
convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result);
return true;
}
else
return false;
}
uint32_t fi;
DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi);
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL)))
{
P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
return false ;
}
DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
// Case where the index is the top of a single person. Can be us, or a friend.
EntryIndex c = 0;
bool res = storage->getChildIndex(e,row,c);
convertEntryIndexToPointer(c,fi,result) ;
return res;
}
// This function returns statistics about the entire directory
int p3FileDatabase::getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& stats)
{
RS_STACK_MUTEX(mFLSMtx) ;
if(pid == mOwnId)
{
mLocalSharedDirs->getStatistics(stats) ;
return true ;
}
else
{
uint32_t fi = locked_getFriendIndex(pid);
mRemoteDirectories[fi]->getStatistics(stats) ;
return true ;
}
}
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
{
RS_STACK_MUTEX(mFLSMtx) ;
d.children.clear();
// Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
// or at the top of our own list of shared directories, depending on the flags.
// Friend index is used as follows:
// 0 : own id
// 1...n : other friends
//
// entry_index: starts at 0.
//
// The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
// So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
//
// We should also check the consistency between flags and the content of ref.
if (ref == NULL)
{
d.ref = NULL ;
d.type = DIR_TYPE_ROOT;
d.count = 1;
d.parent = NULL;
d.prow = -1;
d.ref = NULL;
d.name = "root";
d.hash.clear() ;
d.path = "";
d.mtime = 0;
d.flags.clear() ;
d.max_mtime = 0 ;
if(flags & RS_FILE_HINTS_LOCAL)
{
void *p;
convertEntryIndexToPointer(0,0,p);
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = mServCtrl->getOwnId().toStdString();
stub.ref = p;
d.children.push_back(stub);
}
else for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] != NULL)
{
void *p;
convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p);
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = mRemoteDirectories[i]->peerId().toStdString();
stub.ref = p;
d.children.push_back(stub);
}
d.count = d.children.size();
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;
#endif
return true ;
}
uint32_t fi;
DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi);
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL)))
{
P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
return false ;
}
DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
// Case where the index is the top of a single person. Can be us, or a friend.
if(storage==NULL || !storage->extractData(e,d))
{
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
#endif
return false ;
}
// update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
// a pointer into an int.
convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ;
for(uint32_t i=0;i<d.children.size();++i)
convertEntryIndexToPointer((intptr_t)d.children[i].ref,fi,d.children[i].ref);
if(e == 0) // root
{
d.prow = 0;//fi-1 ;
d.parent = NULL ;
}
else
{
if(d.parent == 0) // child of root node
d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1);
else
d.prow = storage->parentRow(e) ;
convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ;
}
d.id = storage->peerId();
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;
P3FILELISTS_DEBUG() << d << std::endl;
#endif
return true;
}
int p3FileDatabase::RequestDirDetails(const RsPeerId &/*uid*/, const std::string &/*path*/, DirDetails &/*details*/) const
{
NOT_IMPLEMENTED();
return 0;
}
//int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const
//{
// NOT_IMPLEMENTED();
// return 0;
//}
uint32_t p3FileDatabase::getType(void *ref) const
{
RS_STACK_MUTEX(mFLSMtx) ;
EntryIndex e ;
uint32_t fi;
if(ref == NULL)
return DIR_TYPE_ROOT ;
convertPointerToEntryIndex(ref,e,fi);
if(e == 0)
return DIR_TYPE_PERSON ;
if(fi == 0 && mLocalSharedDirs != NULL)
return mLocalSharedDirs->getEntryType(e) ;
else if(fi-1 < mRemoteDirectories.size() && mRemoteDirectories[fi-1]!=NULL)
return mRemoteDirectories[fi-1]->getEntryType(e) ;
else
return DIR_TYPE_ROOT ;// some failure case. Should not happen
}
void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed
{
mLocalDirWatcher->forceUpdate();
}
bool p3FileDatabase::inDirectoryCheck()
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->inDirectoryCheck();
}
void p3FileDatabase::setFollowSymLinks(bool b)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalDirWatcher->setFollowSymLinks(b) ;
IndicateConfigChanged();
}
bool p3FileDatabase::followSymLinks() const
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->followSymLinks() ;
}
void p3FileDatabase::setWatchEnabled(bool b)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalDirWatcher->setEnabled(b) ;
IndicateConfigChanged();
}
bool p3FileDatabase::watchEnabled()
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->isEnabled() ;
}
void p3FileDatabase::setWatchPeriod(uint32_t seconds)
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalDirWatcher->setFileWatchPeriod(seconds);
IndicateConfigChanged();
}
uint32_t p3FileDatabase::watchPeriod()
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalDirWatcher->fileWatchPeriod();
}
int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
{
if(flags & RS_FILE_HINTS_LOCAL)
{
std::list<EntryIndex> firesults;
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->searchTerms(keywords,firesults) ;
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer(*it,0,p);
*it = (intptr_t)p ;
}
}
filterResults(firesults,results,flags,client_peer_id) ;
}
if(flags & RS_FILE_HINTS_REMOTE)
{
std::list<EntryIndex> firesults;
{
RS_STACK_MUTEX(mFLSMtx) ;
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] != NULL)
{
std::list<EntryIndex> local_results;
mRemoteDirectories[i]->searchTerms(keywords,local_results) ;
for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer(*it,i+1,p);
firesults.push_back((intptr_t)p) ;
}
}
}
for(std::list<EntryIndex>::const_iterator rit ( firesults.begin()); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!RequestDirDetails ((void*)(intptr_t)*rit,dd,RS_FILE_HINTS_REMOTE))
continue ;
results.push_back(dd);
}
}
return !results.empty() ;
}
int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const
{
if(flags & RS_FILE_HINTS_LOCAL)
{
std::list<EntryIndex> firesults;
{
RS_STACK_MUTEX(mFLSMtx) ;
mLocalSharedDirs->searchBoolExp(exp,firesults) ;
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer(*it,0,p);
*it = (intptr_t)p ;
}
}
filterResults(firesults,results,flags,client_peer_id) ;
}
if(flags & RS_FILE_HINTS_REMOTE)
{
std::list<EntryIndex> firesults;
{
RS_STACK_MUTEX(mFLSMtx) ;
for(uint32_t i=0;i<mRemoteDirectories.size();++i)
if(mRemoteDirectories[i] != NULL)
{
std::list<EntryIndex> local_results;
mRemoteDirectories[i]->searchBoolExp(exp,local_results) ;
for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer(*it,i+1,p);
firesults.push_back((intptr_t)p) ;
}
}
}
for(std::list<EntryIndex>::const_iterator rit ( firesults.begin()); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!RequestDirDetails ((void*)(intptr_t)*rit,dd,RS_FILE_HINTS_REMOTE))
continue ;
results.push_back(dd);
}
}
return !results.empty() ;
}
bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const
{
RS_STACK_MUTEX(mFLSMtx) ;
if(hintflags & RS_FILE_HINTS_LOCAL)
{
RsFileHash real_hash ;
EntryIndex indx;
if(!mLocalSharedDirs->searchHash(hash,real_hash,indx))
return false;
mLocalSharedDirs->getFileInfo(indx,info) ;
if(!real_hash.isNull())
{
info.hash = real_hash ;
info.transfer_info_flags |= RS_FILE_REQ_ENCRYPTED ;
}
return true;
}
if(hintflags & RS_FILE_HINTS_REMOTE)
{
NOT_IMPLEMENTED();
return false;
}
return false;
}
int p3FileDatabase::filterResults(const std::list<EntryIndex>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
{
results.clear();
flags &= RS_FILE_HINTS_PERMISSION_MASK; // remove other flags.
/* translate/filter results */
for(std::list<EntryIndex>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
{
DirDetails cdetails ;
if(!RequestDirDetails ((void*)(intptr_t)*rit,cdetails,RS_FILE_HINTS_LOCAL))
{
P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl;
continue ;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
#endif
if(!peer_id.isNull())
{
FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
cdetails.id.clear() ;
results.push_back(cdetails);
#ifdef DEBUG_P3FILELISTS
std::cerr << ": kept" << std::endl ;
#endif
}
#ifdef DEBUG_P3FILELISTS
else
std::cerr << ": discarded" << std::endl ;
#endif
}
else
results.push_back(cdetails);
}
return !results.empty() ;
}
bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath)
{
RS_STACK_MUTEX(mFLSMtx) ;
return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ;
}
//==============================================================================================================================//
// Update of remote directories //
//==============================================================================================================================//
// Algorithm:
//
// Local dirs store the last modif time of the file, in local time
// - the max time is computed upward until the root of the hierarchy
// - because the hash is performed late, the last modf time upward is updated only when the hash is obtained.
//
// Remote dirs store
// 1 - recursive max modif time of the files/dir in the friend's time
// 2 - modif time of the files/dir in the friend's time
// 3 - update TS in the local time
//
// The update algorithm is the following:
//
// [Client side]
// - every 30 sec, send a sync packet for the root of the hierarchy containing the remote known max recurs TS.
// - crawl through all directories and if the update TS is 0, ask for sync too.
// - when a sync respnse is received:
// - if response is ACK, set the update time to now. Nothing changed.
// - if response is an update
// * locally update the subfiles
// * for all subdirs, set the local update time to 0 (so as to force a sync)
//
// [Server side]
// - when sync req is received, compare to local recurs TS, and send ACK or full update accordingly
//
// Directories are designated by their hash, instead of their index. This allows to hide the non shared directories
// behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between
// friends is concerned (We obviously could not do that for GUI display, which has a small and constant cost).
void p3FileDatabase::tickRecv()
{
RsItem *item ;
while( NULL != (item = recvItem()) )
{
switch(item->PacketSubType())
{
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast<RsFileListsSyncRequestItem*>(item) ) ;
break ;
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: handleDirSyncResponse( dynamic_cast<RsFileListsSyncResponseItem*>(item) ) ;
break ;
default:
P3FILELISTS_ERROR() << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl;
}
delete item ;
}
}
void p3FileDatabase::tickSend()
{
// go through the list of out requests and send them to the corresponding friends, if they are online.
}
void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
{
RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem;
// look at item TS. If local is newer, send the full directory content.
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
#endif
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#endif
return;
}
uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ;
ritem->PeerId(item->PeerId()) ;
ritem->request_id = item->request_id;
ritem->entry_hash = item->entry_hash ;
std::list<RsNodeGroupId> node_groups;
FileStorageFlags node_flags;
if(entry_type != DIR_TYPE_DIR)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else
{
time_t local_recurs_max_time ;
mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
// We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
}
else
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
ritem->last_known_recurs_modf_TS = local_recurs_max_time ;
}
}
}
// sends the response.
splitAndSendItem(ritem) ;
}
void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem)
{
ritem->checksum = RsDirUtil::sha1sum((uint8_t*)ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len);
while(ritem->directory_content_data.bin_len > MAX_DIR_SYNC_RESPONSE_DATA_SIZE)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl;
#endif
ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL ;
RsFileListsSyncResponseItem *subitem = new RsFileListsSyncResponseItem() ;
subitem->entry_hash = ritem->entry_hash;
subitem->flags = ritem->flags ;
subitem->last_known_recurs_modf_TS = ritem->last_known_recurs_modf_TS;
subitem->request_id = ritem->request_id;
subitem->checksum = ritem->checksum ;
// copy a subpart of the data
subitem->directory_content_data.tlvtype = ritem->directory_content_data.tlvtype ;
subitem->directory_content_data.setBinData(ritem->directory_content_data.bin_data, MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ;
// update ritem to chop off the data that was sent.
memmove(ritem->directory_content_data.bin_data, &((unsigned char*)ritem->directory_content_data.bin_data)[MAX_DIR_SYNC_RESPONSE_DATA_SIZE], ritem->directory_content_data.bin_len - MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ;
ritem->directory_content_data.bin_len -= MAX_DIR_SYNC_RESPONSE_DATA_SIZE ;
// send
subitem->PeerId(ritem->PeerId()) ;
sendItem(subitem) ;
// fix up last chunk
if(ritem->directory_content_data.bin_len <= MAX_DIR_SYNC_RESPONSE_DATA_SIZE)
ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL_END ;
}
sendItem(ritem) ;
}
// This function should not take memory ownership of ritem, so it makes copies.
RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncResponseItem *ritem)
{
if(!(ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL ))
return ritem ;
// item is a partial item. Look first for a starting entry
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Item from peer " << ritem->PeerId() << " is partial. Size = " << ritem->directory_content_data.bin_len << std::endl;
#endif
RS_STACK_MUTEX(mFLSMtx) ;
bool is_ending = (ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL_END);
std::map<DirSyncRequestId,RsFileListsSyncResponseItem*>::iterator it = mPartialResponseItems.find(ritem->request_id) ;
if(it == mPartialResponseItems.end())
{
if(is_ending)
{
P3FILELISTS_ERROR() << "Impossible situation: partial item ended right away. Dropping..." << std::endl;
return NULL;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Creating new item buffer" << std::endl;
#endif
mPartialResponseItems[ritem->request_id] = new RsFileListsSyncResponseItem(*ritem) ;
return NULL ;
}
else if(it->second->checksum != ritem->checksum)
{
P3FILELISTS_ERROR() << "Impossible situation: partial items with different checksums. Dropping..." << std::endl;
mPartialResponseItems.erase(it);
return NULL;
}
// collapse the item at the end of the existing partial item. Dont delete ritem as it will be by the caller.
uint32_t old_len = it->second->directory_content_data.bin_len ;
uint32_t added = ritem->directory_content_data.bin_len;
it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,old_len + added) ;
memcpy(&((unsigned char*)it->second->directory_content_data.bin_data)[old_len],ritem->directory_content_data.bin_data,added);
it->second->directory_content_data.bin_len = old_len + added ;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Added new chunk of length " << added << ". New size is " << old_len + added << std::endl;
#endif
// if finished, return the item
if(is_ending)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Item is complete. Returning it" << std::endl;
#endif
RsFileListsSyncResponseItem *ret = it->second ;
mPartialResponseItems.erase(it) ;
ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL_END ;
ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL ;
return ret ;
}
else
return NULL ;
}
void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem)
{
RsFileListsSyncResponseItem *item = recvAndRebuildItem(sitem) ;
if(!item)
return ;
time_t now = time(NULL);
// check the hash. If anything goes wrong (in the chunking for instance) the hash will not match
if(RsDirUtil::sha1sum((uint8_t*)item->directory_content_data.bin_data,item->directory_content_data.bin_len) != item->checksum)
{
P3FILELISTS_ERROR() << "Checksum error in response item " << std::hex << item->request_id << std::dec << " . This is unexpected, and might be due to connection problems." << std::endl;
return ;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl;
#endif
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
// find the correct friend entry
uint32_t fi = 0 ;
{
RS_STACK_MUTEX(mFLSMtx) ;
fi = locked_getFriendIndex(item->PeerId());
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " friend index is " << fi ;
#endif
// make sure we have a remote directory for that friend.
if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index))
{
std::cerr << std::endl;
P3FILELISTS_ERROR() << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl;
return ;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " entry index is " << entry_index << " " ;
#endif
}
if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED)
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
std::cerr << " removing directory with index " << entry_index << " because it does not exist." << std::endl;
#endif
mRemoteDirectories[fi]->removeDirectory(entry_index);
mRemoteDirectories[fi]->print();
}
else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE)
{
RS_STACK_MUTEX(mFLSMtx) ;
#ifdef DEBUG_P3FILELISTS
std::cerr << " Directory is up to date. Setting local TS." << std::endl;
#endif
mRemoteDirectories[fi]->setDirectoryUpdateTime(entry_index,now) ;
}
else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl;
#endif
RS_STACK_MUTEX(mFLSMtx) ;
if(mLastDataRecvTS + 1 < now) // avoid notifying the GUI too often as it kills performance.
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed
mLastDataRecvTS = now;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Performing update of directory index " << std::hex << entry_index << std::dec << " from friend " << item->PeerId() << std::endl;
#endif
if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data))
mRemoteDirectories[fi]->lastSweepTime() = now - DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP + 10 ; // force re-sweep in 10 secs, so as to fasten updated
else
P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " new content after update: " << std::endl;
mRemoteDirectories[fi]->print();
#endif
}
// remove the original request from pending list in the end. doing this here avoids that a new request is added while the previous response is not treated.
{
RS_STACK_MUTEX(mFLSMtx) ;
std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.find(item->request_id) ;
if(it == mPendingSyncRequests.end())
{
P3FILELISTS_ERROR() << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl;
return ;
}
mPendingSyncRequests.erase(it) ;
}
}
void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e,int depth)
{
time_t now = time(NULL) ;
std::string indent(2*depth,' ') ;
// if not up to date, request update, and return (content is not certified, so no need to recurs yet).
// if up to date, return, because TS is about the last modif TS below, so no need to recurs either.
// get the info for this entry
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl;
#endif
time_t local_update_TS;
if(!rds->getDirectoryUpdateTime(e,local_update_TS))
{
P3FILELISTS_ERROR() << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl;
return;
}
// compare TS
if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time.
if(locked_generateAndSendSyncRequest(rds,e))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl;
#endif
}
for(DirectoryStorage::DirIterator it(rds,e);it;++it)
locked_recursSweepRemoteDirectory(rds,*it,depth+1);
}
p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,const RsFileHash& hash)
{
static uint64_t random_bias = RSRandom::random_u64();
uint8_t mem[RsPeerId::SIZE_IN_BYTES + RsFileHash::SIZE_IN_BYTES];
memcpy(mem,peer_id.toByteArray(),RsPeerId::SIZE_IN_BYTES) ;
memcpy(&mem[RsPeerId::SIZE_IN_BYTES],hash.toByteArray(),RsFileHash::SIZE_IN_BYTES) ;
RsFileHash tmp = RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES + RsFileHash::SIZE_IN_BYTES) ;
// This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,entry index) pair, in a way
// that cannot be brute-forced or reverse-engineered, which explains the random bias and the usage of the hash, that is itself random.
uint64_t r = random_bias ^ *((uint64_t*)tmp.toByteArray()) ;
#ifdef DEBUG_P3FILELISTS
std::cerr << "Creating ID " << std::hex << r << std::dec << " from peer id " << peer_id << " and hash " << hash << std::endl;
#endif
return r ;
}
bool p3FileDatabase::locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e)
{
RsFileHash entry_hash ;
time_t now = time(NULL) ;
time_t max_known_recurs_modf_time ;
if(!rds->getDirectoryRecursModTime(e,max_known_recurs_modf_time))
{
P3FILELISTS_ERROR() << " (EE) cannot find recurs mod time for entry index " << e << ". This is very unexpected." << std::endl;
return false;
}
if(!rds->getDirHashFromIndex(e,entry_hash) )
{
P3FILELISTS_ERROR() << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl;
return false;
}
// check if a request already exists and is not too old either: no need to re-ask.
DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),entry_hash) ;
std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.find(sync_req_id) ;
if(it != mPendingSyncRequests.end())
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl;
#endif
return false ;
}
RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
item->entry_hash = entry_hash ;
item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ;
item->request_id = sync_req_id ;
item->last_known_recurs_modf_TS = max_known_recurs_modf_time ;
item->PeerId(rds->peerId()) ;
DirSyncRequestData data ;
data.request_TS = now ;
data.peer_id = item->PeerId();
data.flags = item->flags;
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Pushing req " << std::hex << sync_req_id << std::dec << " in pending list with peer id " << data.peer_id << std::endl;
#endif
mPendingSyncRequests[sync_req_id] = data ;
sendItem(item) ; // at end! Because item is destroyed by the process.
return true;
}
Reference in New Issue View Git Blame Copy Permalink