Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

added update of last modif LS, and rsitems to update directories between friends

Browse Source
This commit is contained in:
mr-alice 2016-08-22 07:49:45 +02:00
parent edc602f68f
commit 76831348e6
9 changed files with 129 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -53,7 +53,7 @@ class InternalFileHierarchyStorage
class DirEntry: public FileStorageNode
{
public:
DirEntry(const std::string& name) : dir_name(name) {}
DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {}
virtual ~DirEntry() {}
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
@ -65,7 +65,8 @@ class InternalFileHierarchyStorage
std::vector<DirectoryStorage::EntryIndex> subdirs ;
std::vector<DirectoryStorage::EntryIndex> subfiles ;
time_t most_recent_time;
time_t dir_modtime ; // this accounts for deleted files, etc.
time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below.
};
// class stuff
@ -221,7 +222,10 @@ class InternalFileHierarchyStorage
std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl;
static_cast<FileEntry*>(mNodes[file_index])->file_hash = hash ;
RsFileHash& old_hash (static_cast<FileEntry*>(mNodes[file_index])->file_hash) ;
old_hash = hash ;
return true;
}
bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time)
@ -243,6 +247,25 @@ class InternalFileHierarchyStorage
return true;
}
// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index)
{
DirEntry& d(*static_cast<DirEntry*>(mNodes[dir_index])) ;
time_t largest_modf_time = d.dir_modtime ;
for(uint32_t i=0;i<d.subfiles.size();++i)
largest_modf_time = std::max(largest_modf_time,static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_modtime) ;
for(uint32_t i=0;i<d.subdirs.size();++i)
largest_modf_time = std::max(largest_modf_time,recursUpdateLastModfTime(d.subdirs[i])) ;
d.most_recent_time = largest_modf_time ;
return largest_modf_time ;
}
// file/dir access and modification
bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
@ -367,7 +390,7 @@ private:
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[node]));
std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ;
std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ;
for(int i=0;i<d.subdirs.size();++i)
std::cerr << d.subdirs[i] << " " ;
std::cerr << std::endl;
@ -714,6 +737,15 @@ bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::
return true;
}
void LocalDirectoryStorage::updateTimeStamps()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ;
std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl;
}
std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const
{
/**** MUST ALREADY BE LOCKED ****/

1
libretroshare/src/file_sharing/directory_storage.h
View File

@ -137,6 +137,7 @@ public:
void updateShareFlags(const SharedDirInfo& info) ;
bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ;
void updateTimeStamps();
/*!
* \brief getFileInfo Converts an index info a full file info structure.
* \param i index in the directory structure

31
libretroshare/src/file_sharing/directory_updater.cc
View File

@ -10,13 +10,15 @@
// Local Directory Updater //
//=============================================================================================================//
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!!
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!!
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!!
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!!
static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!!
LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds)
: mHashCache(hc),mSharedDirectories(lds)
{
mLastSweepTime = 0;
mLastTSUpdateTime = 0;
}
void LocalDirectoryUpdater::data_tick()
@ -28,8 +30,13 @@ void LocalDirectoryUpdater::data_tick()
sweepSharedDirectories() ;
mLastSweepTime = now;
}
else
usleep(10*1000*1000);
if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime)
{
mSharedDirectories->updateTimeStamps() ;
mLastTSUpdateTime = now ;
}
usleep(10*1000*1000);
}
void LocalDirectoryUpdater::forceUpdate()
@ -93,13 +100,13 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
subfiles[dirIt.file_name()].size = dirIt.file_size();
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ;
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ;
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
@ -139,3 +146,9 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash))
std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl;
}

1
libretroshare/src/file_sharing/directory_updater.h
View File

@ -35,5 +35,6 @@ private:
LocalDirectoryStorage *mSharedDirectories ;
time_t mLastSweepTime;
time_t mLastTSUpdateTime;
};

45
libretroshare/src/file_sharing/p3filelists.cc
View File

@ -17,9 +17,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ;
static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ;
static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ;
static const uint32_t NB_FRIEND_INDEX_BITS = 10 ;
static const uint32_t NB_ENTRY_INDEX_BITS = 22 ;
static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer.
static const uint32_t NB_FRIEND_INDEX_BITS = 10 ;
static const uint32_t NB_ENTRY_INDEX_BITS = 22 ;
static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer.
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so.
p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
: mServCtrl(mpeers), mFLSMtx("p3FileLists")
@ -608,6 +609,17 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string&
// Update of remote directories //
//==============================================================================================================================//
// Algorithm:
//
// Local dirs store the last modif time of the file, in local time
// - the max time is computed upward until the root of the hierarchy
// - because the hash is performed late, the last modf time upward is updated only when the hash is obtained.
//
// Remote dirs store the last modif time of the files/dir in the friend's time
// - local node sends the last known modf time to friends,
// - friends respond with either a full directory content, or an acknowledge that the time is right
//
void p3FileDatabase::tickRecv()
{
RsItem *item ;
@ -637,15 +649,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item)
{
// look at item TS. If local is newer, send the full directory content.
time_t recurs_max_modf_TS;
time_t recurs_max_modf_TS, last_update_TS;
if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS))
{
std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl;
return ;
}
P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
if(item->known_recurs_last_modf_TS < recurs_last_modf_TS)
if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS))
{
std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl;
return ;
}
if(item->last_known_recurs_modf_TS < recurs_max_modf_TS)
{
// send full update of directory content
}
@ -655,7 +669,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item)
}
}
void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item)
void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item)
{
// update the directory content for the specified friend.
@ -677,9 +691,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
time_t recurs_max_modf_TS,last_update_TS ;
if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS))
if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS))
{
std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl;
std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl;
return ;
}
@ -705,9 +719,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
mPendingSyncRequests[sync_req_id] = data ;
RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ;
item->entry_index = e ;
item->known_TS = recurs_max_modf_TS ;
item->last_known_recurs_modf_TS = recurs_max_modf_TS ;
item->PeerId(rds->peerId()) ;
sendItem(item) ;

12
libretroshare/src/file_sharing/p3filelists.h
View File

@ -173,8 +173,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
std::map<RsPeerId,uint32_t> mFriendIndexMap ;
std::vector<RsPeerId> mFriendIndexTab;
// TS for friend list update
time_t mLastRemoteDirSweepTS ;
// Directory synchronization
//
struct DirSyncRequestData
{
time_t request_TS ;
uint32_t flags ;
};
time_t mLastRemoteDirSweepTS ; // TS for friend list update
std::map<DirSyncRequestId,DirSyncRequestData> mPendingSyncRequests ; // pending requests, waiting for an answer
void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e);

40
libretroshare/src/file_sharing/rsfilelistitems.cc
View File

@ -1,3 +1,5 @@
#include "serialiser/rsbaseserial.h"
#include "file_sharing/rsfilelistitems.h"
RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
@ -8,8 +10,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
/* get the type and size */
uint32_t rstype = getRsItemId(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE))
return NULL; /* wrong type */
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE))
return NULL; /* wrong type */
switch(getRsItemSubType(rstype))
{
@ -28,26 +30,26 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
uint32_t RsFileListsSerialiser::size(RsItem *item)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serial_size() ;
if(flst_item != NULL)
return flst_item->serial_size() ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serialise(data,*size) ;
if(flst_item != NULL)
return flst_item->serialise(data,*size) ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
@ -185,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void
return item;
}
RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size)
RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size)
{
bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
uint32_t offset = 8;
RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem();
RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem();
uint32_t entry_index ; // index of the directory to sync
uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
@ -234,7 +236,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype)))
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype)))
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl;
@ -265,11 +267,6 @@ uint32_t RsFileListsSyncReqItem::serial_size()const
uint32_t s = 8; //header size
entry_index ;
flags;
last_known_recurs_modf_TS;
request_id;
s += 4; // entry index
s += 4; // flags
s += 4; // last_known_recurs_modf_TS
@ -278,16 +275,11 @@ uint32_t RsFileListsSyncReqItem::serial_size()const
return s;
}
uint32_t RsFileListsSyncReqItem::serial_size()const
uint32_t RsFileListsSyncDirItem::serial_size()const
{
uint32_t s = 8; //header size
entry_index ;
flags;
last_known_recurs_modf_TS;
request_id;
s += 4; // entry index
s += 4; // flags
s += 4; // last_known_recurs_modf_TS

22
libretroshare/src/file_sharing/rsfilelistitems.h
View File

@ -75,7 +75,7 @@ class RsFileListsSyncReqItem : public RsFileListsItem
{
public:
RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {}
RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
@ -93,20 +93,20 @@ class RsFileListsSyncDirItem : public RsFileListsItem
{
public:
RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {}
RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
uint32_t entry_index ; // advises whether to use sync hash
uint32_t flags; // is it a partial/final item (used for large items only)
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
uint32_t entry_index ; // advises whether to use sync hash
uint32_t flags; // is it a partial/final item (used for large items only)
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
};
class RsFileListsSerialiser : public RsSerialType

4
libretroshare/src/libretroshare.pro
View File

@ -41,14 +41,14 @@ file_lists {
file_sharing/filelist_io.h \
file_sharing/directory_storage.h \
file_sharing/directory_updater.h \
file_sharing/filelistitems.h
file_sharing/rsfilelistitems.h
SOURCES *= file_sharing/p3filelists.cc \
file_sharing/hash_cache.cc \
file_sharing/filelist_io.cc \
file_sharing/directory_storage.cc \
file_sharing/directory_updater.cc \
file_sharing/filelistitems.cc
file_sharing/rsfilelistitems.cc
}
dsdv {