added infrastructure for synchronisation of file lists

libretroshare/src/file_sharing/directory_storage.cc

@@ -303,6 +303,20 @@ class InternalFileHierarchyStorage
         return static_cast<DirEntry*>(mNodes[parent_index])->subdirs[dir_tab_index];
     }
 
+    bool searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results)
+    {
+        std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it = mHashes.find(hash);
+
+        if( it != mHashes.end() )
+        {
+            results.clear();
+            results.push_back(it->second) ;
+            return true ;
+        }
+        else
+           return false;
+    }
+
     bool check()	// checks consistency of storage.
     {
         return true;
@@ -390,7 +404,7 @@ private:
         return true ;
     }
 
-    std::map<RsFileHash,DirectoryStorage::EntryIndex> mHashes ; // used for fast search access
+    std::map<RsFileHash,DirectoryStorage::EntryIndex> mHashes ; // used for fast search access. We should try something faster than std::map. hash_map??
 };
 
 /******************************************************************************************************************/
@@ -498,6 +512,12 @@ bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash
     return mFileHierarchy->updateHash(index,hash);
 }
 
+int DirectoryStorage::searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const
+{
+    RS_STACK_MUTEX(mDirStorageMtx) ;
+    return mFileHierarchy->searchHash(hash,results);
+}
+
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH         =  0x01 ;
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME         =  0x02 ;
 // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE         =  0x03 ;

libretroshare/src/file_sharing/directory_storage.h

@@ -23,10 +23,10 @@ class DirectoryStorage
 		void save() const ;
 
         virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const { NOT_IMPLEMENTED() ; return 0;}
-        virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const { NOT_IMPLEMENTED() ; return 0; }
+        virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const ;
         virtual int searchBoolExp(Expression * exp, std::list<EntryIndex> &results) const { NOT_IMPLEMENTED() ; return 0; }
 
-		void getFileDetails(EntryIndex i) ;
+        bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ;
         uint32_t getEntryType(const EntryIndex& indx) ;	                     // returns DIR_TYPE_*, not the internal directory storage stuff.
         virtual bool extractData(const EntryIndex& indx,DirDetails& d);
 
@@ -47,6 +47,8 @@ class DirectoryStorage
                 // info about the directory that is pointed by the iterator
 
                 std::string name() const ;
+                time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified.
+                time_t last_update_time() const ; // last time this directory was updated
             private:
                 EntryIndex mParentIndex ;		// index of the parent dir.
                 uint32_t mDirTabIndex ;				// index in the vector of subdirs.

libretroshare/src/file_sharing/directory_updater.cc

@@ -6,12 +6,12 @@
 
 #define DEBUG_LOCAL_DIR_UPDATER 1
 
-static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!!
+//=============================================================================================================//
+//                                           Local Directory Updater                                           //
+//=============================================================================================================//
 
-void RemoteDirectoryUpdater::tick()
+static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES         = 100 ; // 10 seconds for testing. Should be much more!!
-{
+static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!!
-	// use the stored iterator
-}
 
 LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds)
     : mHashCache(hc),mSharedDirectories(lds)
@@ -139,4 +139,3 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
     if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash))
         std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl;
 }
-

libretroshare/src/file_sharing/directory_updater.h

@@ -13,6 +13,8 @@ class DirectoryUpdater
         virtual ~DirectoryUpdater(){}
 };
 
+#warning: simplify, if we don't keep the remote directory updater
+
 class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread
 {
 public:
@@ -35,11 +37,3 @@ private:
     time_t mLastSweepTime;
 };
 
-class RemoteDirectoryUpdater: public DirectoryUpdater
-{
-    public:
-            RemoteDirectoryUpdater() {}
-        virtual ~RemoteDirectoryUpdater() {}
-
-        virtual void tick() ;
-};

libretroshare/src/file_sharing/p3filelists.cc

@@ -3,6 +3,7 @@
 #include "file_sharing/p3filelists.h"
 #include "file_sharing/directory_storage.h"
 #include "file_sharing/directory_updater.h"
+#include "file_sharing/rsfilelistitems.h"
 
 #include "retroshare/rsids.h"
 #include "retroshare/rspeers.h"
@@ -25,17 +26,16 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
 {
 	// loads existing indexes for friends. Some might be already present here.
 	// 
-    mDirectories.clear() ;	// we should load them!
+    mRemoteDirectories.clear() ;	// we should load them!
-
+    mOwnId = mpeers->getOwnId() ;
-    mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ;
-    mDirectories.push_back(mLocalSharedDirs) ;
 
+    mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId);
     mHashCache = new HashStorage("hash_cache.bin") ;
 
     mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ;
-    mRemoteDirWatcher = NULL;	// not used yet
 
 	mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
+    mLastRemoteDirSweepTS = 0 ;
 }
 
 void p3FileDatabase::setSharedDirectories(const std::list<SharedDirInfo>& shared_dirs)
@@ -60,14 +60,13 @@ p3FileDatabase::~p3FileDatabase()
 {
     RS_STACK_MUTEX(mFLSMtx) ;
 
-    for(uint32_t i=0;i<mDirectories.size();++i)
+    for(uint32_t i=0;i<mRemoteDirectories.size();++i)
-        delete mDirectories[i];
+        delete mRemoteDirectories[i];
 
-    mDirectories.clear(); // just a precaution, not to leave deleted pointers around.
+    mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around.
 
     delete mLocalSharedDirs ;
     delete mLocalDirWatcher ;
-    delete mRemoteDirWatcher ;
     delete mHashCache ;
 }
 
@@ -128,6 +127,17 @@ int p3FileDatabase::tick()
 
         mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
     }
+
+    if(mLastRemoteDirSweepTS + 5 < now)
+    {
+        RS_STACK_MUTEX(mFLSMtx) ;
+
+        for(uint32_t i=0;i<mRemoteDirectories.size();++i)
+           locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ;
+
+        mLastRemoteDirSweepTS = now;
+    }
+
     return 0;
 }
 
@@ -153,19 +163,12 @@ void p3FileDatabase::stopThreads()
 void p3FileDatabase::tickWatchers()
 {
 }
-void p3FileDatabase::tickRecv()
-{
-}
-void p3FileDatabase::tickSend()
-{
-	// go through the list of out requests and send them to the corresponding friends, if they are online.
-}
-
 bool p3FileDatabase::loadList(std::list<RsItem *>& items)
 {
 	// This loads
 	//
 	// 	- list of locally shared directories, and the permissions that go with them
+
     NOT_IMPLEMENTED();
 
     return true ;
@@ -194,19 +197,19 @@ void p3FileDatabase::cleanup()
                 friend_set.insert(*it) ;
         }
 
-        for(uint32_t i=1;i<mDirectories.size();++i)	// start at 1, so that we don't cleanup our own.
+        for(uint32_t i=0;i<mRemoteDirectories.size();++i)
-            if(friend_set.find(mDirectories[i]->peerId()) == friend_set.end())
+            if(friend_set.find(mRemoteDirectories[i]->peerId()) == friend_set.end())
             {
-                P3FILELISTS_DEBUG() << "  removing file list of non friend " << mDirectories[i]->peerId() << std::endl;
+                P3FILELISTS_DEBUG() << "  removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl;
 
-                delete mDirectories[i];
+                delete mRemoteDirectories[i];
-                mDirectories[i] = NULL ;
+                mRemoteDirectories[i] = NULL ;
 
                 mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
 
-                friend_set.erase(mDirectories[i]->peerId());
+                friend_set.erase(mRemoteDirectories[i]->peerId());
 
-                mFriendIndexMap.erase(mDirectories[i]->peerId());
+                mFriendIndexMap.erase(mRemoteDirectories[i]->peerId());
                 mFriendIndexTab[i].clear();
             }
 
@@ -218,12 +221,12 @@ void p3FileDatabase::cleanup()
 
             uint32_t friend_index = locked_getFriendIndex(*it) ;
 
-            if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL)
+            if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL)
                 continue ;
 
             P3FILELISTS_DEBUG() << "  adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl;
 
-            mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it));
+            mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it));
 
             mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
         }
@@ -244,7 +247,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
     {
         // allocate a new index for that friend, and tell that we should save.
         uint32_t found = 0 ;
-        for(uint32_t i=1;i<mFriendIndexTab.size();++i)
+        for(uint32_t i=0;i<mFriendIndexTab.size();++i)
             if(mFriendIndexTab[i].isNull())
             {
                 found = i ;
@@ -266,8 +269,8 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
         mFriendIndexTab[found] = pid ;
         mFriendIndexMap[pid] = found;
 
-        if(mDirectories.size() <= found)
+        if(mRemoteDirectories.size() <= found)
-            mDirectories.resize(found+1,NULL) ;
+            mRemoteDirectories.resize(found,NULL) ;
 
         IndicateConfigChanged();
 
@@ -275,8 +278,8 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
     }
     else
     {
-        if(mDirectories.size() <= it->second)
+        if(mRemoteDirectories.size() <= it->second)
-            mDirectories.resize(it->second+1,NULL) ;
+            mRemoteDirectories.resize(it->second,NULL) ;
 
         return it->second;
     }
@@ -372,14 +375,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
             stub.ref  = p;
             d.children.push_back(stub);
         }
-        else for(uint32_t i=1;i<mDirectories.size();++i)
+        else for(uint32_t i=0;i<mRemoteDirectories.size();++i)
         {
             void *p;
-            convertEntryIndexToPointer(mDirectories[i]->root(),i,p);
+            convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i,p);
 
             DirStub stub;
             stub.type = DIR_TYPE_PERSON;
-            stub.name = mDirectories[i]->peerId().toStdString();
+            stub.name = mRemoteDirectories[i]->peerId().toStdString();
             stub.ref  = p;
             d.children.push_back(stub);
         }
@@ -396,7 +399,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
     // Case where the index is the top of a single person. Can be us, or a friend.
 
-    bool res = mDirectories[fi]->extractData(e,d) ;
+    bool res =  (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ;
 
     // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
     // a pointer into an int.
@@ -414,11 +417,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     }
     else
     {
-        d.prow = mDirectories[fi]->parentRow(e) ;
+        d.prow = mRemoteDirectories[fi]->parentRow(e) ;
         convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ;
     }
 
-    d.id   = mDirectories[fi]->peerId();
+    if(flags & RS_FILE_HINTS_LOCAL)
+        d.id = mOwnId ;
+    else
+        d.id = mRemoteDirectories[fi]->peerId();
 
     std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;
     std::cerr << d << std::endl;
@@ -451,7 +457,7 @@ uint32_t p3FileDatabase::getType(void *ref) const
     if(e == 0)
         return DIR_TYPE_PERSON ;
 
-    return mDirectories[fi]->getEntryType(e) ;
+    return mRemoteDirectories[fi]->getEntryType(e) ;
 }
 
 void p3FileDatabase::forceDirectoryCheck()              // Force re-sweep the directories and see what's changed
@@ -536,13 +542,14 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F
         if(res.empty())
             return false;
 
-        EntryIndex indx = *res.begin() ; // no need to report dupicates
+        EntryIndex indx = *res.begin() ; // no need to report duplicates
 
         mLocalSharedDirs->getFileInfo(indx,info) ;
 
         return true;
     }
-    else
+
+    if(hintflags & RS_FILE_HINTS_REMOTE)
     {
         NOT_IMPLEMENTED();
         return false;
@@ -596,3 +603,114 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string&
     RS_STACK_MUTEX(mFLSMtx) ;
     return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ;
 }
+
+//==============================================================================================================================//
+//                                               Update of remote directories                                                   //
+//==============================================================================================================================//
+
+void p3FileDatabase::tickRecv()
+{
+   RsItem *item ;
+
+   while( NULL != (item = recvItem()) )
+   {
+      switch(item->PacketSubType())
+      {
+      case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast<RsFileListsSyncReqItem*>(item) ) ;
+         break ;
+      case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast<RsFileListsSyncDirItem*>(item) ) ;
+         break ;
+      default:
+         std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl;
+      }
+
+      delete item ;
+   }
+}
+
+void p3FileDatabase::tickSend()
+{
+    // go through the list of out requests and send them to the corresponding friends, if they are online.
+}
+
+void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item)
+{
+    // look at item TS. If local is newer, send the full directory content.
+
+   time_t recurs_max_modf_TS;
+
+   if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS))
+   {
+      std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl;
+      return ;
+   }
+
+    if(item->known_recurs_last_modf_TS < recurs_last_modf_TS)
+    {
+        // send full update of directory content
+    }
+    else
+    {
+        // send last recurs update TS.
+    }
+}
+
+void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item)
+{
+    // update the directory content for the specified friend.
+
+    // set the update TS, and the remote modif TS accordingly
+
+    // notify the GUI if the hierarchy has changed
+}
+
+void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e)
+{
+   time_t now = time(NULL) ;
+
+   // get the info for this entry
+
+   // compare TS
+
+   // if not up to date, request update, and return (content is not certified, so no need to recurs yet).
+   // if up to date, return, because TS is about the last modif TS below, so no need to recurs either.
+
+   time_t recurs_max_modf_TS,last_update_TS ;
+
+   if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS))
+   {
+      std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl;
+      return ;
+   }
+
+   if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ)
+   {
+        // check if a request already exists and is not too old either: no need to re-ask.
+
+        DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ;
+
+        std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.find(sync_req_id) ;
+
+        if(it != mPendingSyncRequests.end())
+        {
+            std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl;
+            return ;
+        }
+
+        std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl;
+
+        DirSyncRequestData data ;
+
+        data.request_TS = now ;
+
+        mPendingSyncRequests[sync_req_id] = data ;
+
+        RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
+        item->entry_index =  e ;
+        item->known_TS =  recurs_max_modf_TS ;
+
+        sendItem(item) ;
+
+        return ;
+   }
+}

libretroshare/src/file_sharing/p3filelists.h

@@ -37,6 +37,9 @@ class LocalDirectoryUpdater ;
 class RemoteDirectoryStorage ;
 class LocalDirectoryStorage ;
 
+class RsFileListsSyncReqItem ;
+class RsFileListsSyncDirItem ;
+
 class HashStorage ;
 
 class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService
@@ -135,22 +138,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
 
     private:
         p3ServiceControl *mServCtrl ;
+        RsPeerId mOwnId ;
+
+        typedef uint64_t DirSyncRequestId ;
+
+        static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ;
 
 		// File sync request queues. The fast one is used for online browsing when friends are connected.
 		// The slow one is used for background update of file lists.
 		//
-		std::list<RsFileListSyncRequest> mFastRequestQueue ;
+        std::map<DirSyncRequestId,RsFileListSyncRequest> mFastRequestQueue ;
-		std::list<RsFileListSyncRequest> mSlowRequestQueue ;
+        std::map<DirSyncRequestId,RsFileListSyncRequest> mSlowRequestQueue ;
 
 		// Directory storage hierarchies
 		//
         // The remote one is the reference for the PeerId index below:
         //     RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid)
 
-        std::vector<DirectoryStorage *> mDirectories ;	// mDirectories[0]=mLocalSharedDirs
+        std::vector<RemoteDirectoryStorage *> mRemoteDirectories ;
         LocalDirectoryStorage *mLocalSharedDirs ;
 
-        RemoteDirectoryUpdater *mRemoteDirWatcher ;	// not used yet.
         LocalDirectoryUpdater *mLocalDirWatcher ;
 
         // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc
@@ -160,9 +167,17 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         uint32_t locked_getFriendIndex(const RsPeerId& pid);
         const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const;
 
+        void handleDirSyncRequest(RsFileListsSyncReqItem *) ;
+        void handleDirSyncContent(RsFileListsSyncDirItem *) ;
+
         std::map<RsPeerId,uint32_t> mFriendIndexMap ;
         std::vector<RsPeerId> mFriendIndexTab;
 
+        // TS for friend list update
+        time_t mLastRemoteDirSweepTS ;
+
+        void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e);
+
         // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name.
 		//
         HashStorage *mHashCache ;

libretroshare/src/file_sharing/rsfilelistitems.cc

@@ -0,0 +1,336 @@
+#include "file_sharing/rsfilelistitems.h"
+
+RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
+{
+#ifdef RSSERIAL_DEBUG
+        std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl;
+#endif
+        /* get the type and size */
+        uint32_t rstype = getRsItemId(data);
+
+		if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE))
+			return NULL; /* wrong type */
+
+        switch(getRsItemSubType(rstype))
+        {
+        case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM:   return deserialFileListsSyncReqItem(data, size);
+        case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM:   return deserialFileListsSyncDirItem(data, size);
+        case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM:     return deserialFileListsConfigItem (data, size);
+
+        default:
+            {
+                std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl;
+                return NULL;
+
+            }
+        }
+}
+
+uint32_t RsFileListsSerialiser::size(RsItem *item)
+{
+	RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
+
+	if(flst_item != NULL)
+		return flst_item->serial_size() ;
+	else
+	{
+		std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!"  << std::endl;
+		return 0;
+	}
+}
+
+bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size)
+{
+	RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
+
+	if(flst_item != NULL)
+		return flst_item->serialise(data,*size) ;
+	else
+	{
+		std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!"  << std::endl;
+		return 0;
+	}
+}
+
+bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const
+{
+	tlvsize = serial_size() ;
+	offset = 0;
+
+	if (pktsize < tlvsize)
+		return false; /* not enough space */
+
+	pktsize = tlvsize;
+
+	if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize))
+	{
+		std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl;
+		return false ;
+	}
+#ifdef RSSERIAL_DEBUG
+	std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl;
+#endif
+	offset += 8;
+
+	return true ;
+}
+
+bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const
+{
+    uint32_t tlvsize,offset=0;
+    bool ok = true;
+
+    if(!serialise_header(data,size,tlvsize,offset))
+        return false ;
+
+#ifdef RSSERIAL_DEBUG
+    std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
+#endif
+
+    /* RsFileListsSyncMsgItem */
+
+    ok &= setRawUInt32(data, size, &offset, entry_index);
+    ok &= setRawUInt32(data, size, &offset, flags      );
+    ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
+    ok &= setRawUInt64(data, size, &offset, request_id);
+
+    if(offset != tlvsize){
+#ifdef RSSERIAL_DEBUG
+        std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
+#endif
+        ok = false;
+    }
+
+#ifdef RSSERIAL_DEBUG
+    if (!ok)
+    {
+        std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
+    }
+#endif
+
+    return ok;
+}
+
+bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const
+{
+    uint32_t tlvsize,offset=0;
+    bool ok = true;
+
+    if(!serialise_header(data,size,tlvsize,offset))
+        return false ;
+
+#ifdef RSSERIAL_DEBUG
+    std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
+#endif
+
+    /* RsFileListsSyncMsgItem */
+
+    ok &= setRawUInt32(data, size, &offset, entry_index);
+    ok &= setRawUInt32(data, size, &offset, flags      );
+    ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
+    ok &= setRawUInt64(data, size, &offset, request_id);
+    ok &= directory_content_data.SetTlv(data,size,&offset) ;
+
+    if(offset != tlvsize){
+#ifdef RSSERIAL_DEBUG
+        std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
+#endif
+        ok = false;
+    }
+
+#ifdef RSSERIAL_DEBUG
+    if (!ok)
+    {
+        std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
+    }
+#endif
+
+    return ok;
+}
+
+//============================================================================================================================//
+//                                                     Deserialisation                                                        //
+//============================================================================================================================//
+
+RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size)
+{
+    bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
+    uint32_t offset = 8;
+
+    RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem();
+
+    ok &= getRawUInt32(data, *size, &offset, &item->entry_index);
+    ok &= getRawUInt32(data, *size, &offset, &item->flags);
+    ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
+    ok &= getRawUInt64(data, *size, &offset, &item->request_id);
+
+    if (offset != *size)
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
+#endif
+            /* error */
+            delete item;
+            return NULL;
+    }
+
+    if (!ok)
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
+#endif
+            delete item;
+            return NULL;
+    }
+
+    return item;
+}
+RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size)
+{
+    bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
+    uint32_t offset = 8;
+
+    RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem();
+
+    uint32_t entry_index ;              // index of the directory to sync
+    uint32_t flags;                     // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
+    uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
+    uint64_t request_id;                // use to determine if changes that have occured since last hash
+
+    ok &= getRawUInt32(data, *size, &offset, &item->entry_index);
+    ok &= getRawUInt32(data, *size, &offset, &item->flags);
+    ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
+    ok &= getRawUInt64(data, *size, &offset, &item->request_id);
+
+    ok &= item->directory_content_data.GetTlv(data,*size,&offset) ;
+
+    if (offset != *size)
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
+#endif
+            /* error */
+            delete item;
+            return NULL;
+    }
+
+    if (!ok)
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
+#endif
+            delete item;
+            return NULL;
+    }
+
+    return item;
+}
+bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type)
+{
+#ifdef RSSERIAL_DEBUG
+    std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl;
+#endif
+    /* get the type and size */
+    uint32_t rstype = getRsItemId(data);
+    uint32_t rssize = getRsItemSize(data);
+
+    if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype)))
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl;
+#endif
+            return false; /* wrong type */
+    }
+
+    if (*size < rssize)    /* check size */
+    {
+#ifdef RSSERIAL_DEBUG
+            std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl;
+#endif
+            return false; /* not enough data */
+    }
+
+    /* set the packet length */
+    *size = rssize;
+    
+    return true ;
+}
+
+//============================================================================================================================//
+//                                                         Sizes                                                              //
+//============================================================================================================================//
+
+uint32_t RsFileListsSyncReqItem::serial_size()const
+{
+
+    uint32_t s = 8; //header size
+
+    entry_index ;
+    flags;
+    last_known_recurs_modf_TS;
+    request_id;
+
+    s += 4; // entry index
+    s += 4; // flags
+    s += 4; // last_known_recurs_modf_TS
+    s += 8; // request_id
+
+    return s;
+}
+
+uint32_t RsFileListsSyncReqItem::serial_size()const
+{
+
+    uint32_t s = 8; //header size
+
+    entry_index ;
+    flags;
+    last_known_recurs_modf_TS;
+    request_id;
+
+    s += 4; // entry index
+    s += 4; // flags
+    s += 4; // last_known_recurs_modf_TS
+    s += 8; // request_id
+    s += directory_content_data.TlvSize();
+
+    return s;
+}
+
+void RsFileListsSyncReqItem::clear()
+{
+}
+void RsFileListsSyncDirItem::clear()
+{
+    directory_content_data.TlvClear();
+}
+std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent)
+{
+    printRsItemBase(out, "RsFileListsSyncReqItem", indent);
+    uint16_t int_Indent = indent + 2;
+
+    printIndent(out , int_Indent); out << "Entry index:  " << entry_index << std::endl;
+    printIndent(out , int_Indent); out << "Flags:        " << (uint32_t) flags << std::endl;
+    printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
+    printIndent(out , int_Indent); out << "request id:   " << std::hex << request_id  << std::dec << std::endl;
+
+    printRsItemEnd(out ,"RsFileListsSyncReqItem", indent);
+
+    return out;
+}
+
+std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent)
+{
+    printRsItemBase(out, "RsFileListsSyncDirItem", indent);
+    uint16_t int_Indent = indent + 2;
+
+    printIndent(out , int_Indent); out << "Entry index:  " << entry_index << std::endl;
+    printIndent(out , int_Indent); out << "Flags:        " << (uint32_t) flags << std::endl;
+    printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
+    printIndent(out , int_Indent); out << "request id:   " << std::hex << request_id  << std::dec << std::endl;
+    printIndent(out , int_Indent); out << "Data size:    " << directory_content_data.bin_len << std::endl;
+
+    printRsItemEnd(out ,"RsFileListsSyncDirItem", indent);
+
+    return out;
+}

libretroshare/src/file_sharing/rsfilelistitems.h

@@ -0,0 +1,132 @@
+#pragma once
+
+/*
+ * libretroshare/src/serialiser: rsnxssitems.h
+ *
+ * RetroShare Serialiser.
+ *
+ * Copyright 2012 Christopher Evi-Parker, Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <map>
+#include <openssl/ssl.h>
+
+#include "serialiser/rsserviceids.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvbase.h"
+#include "serialiser/rstlvitem.h"
+#include "serialiser/rstlvkeys.h"
+#include "gxs/rsgxsdata.h"
+
+// These items have "flag type" numbers, but this is not used.
+
+const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM    = 0x01;
+const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM    = 0x02;
+const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM      = 0x03;
+
+/*!
+ * Base class for filelist sync items
+ */
+class RsFileListsItem : public RsItem
+{
+public:
+    RsFileListsItem(uint8_t subtype)
+        : RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype)
+    {
+        setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST);	// this is the default. Someitems may be faster, on demand.
+        return;
+    }
+    virtual ~RsFileListsItem(){}
+   
+    virtual bool serialise(void *data,uint32_t& size) const = 0 ;	
+    virtual uint32_t serial_size() const = 0 ; 			
+    virtual void clear() = 0;
+    virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0;
+
+    bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const;
+
+    static const uint32_t FLAGS_SYNC_REQUEST      = 0x0001 ;
+    static const uint32_t FLAGS_SYNC_RESPONSE     = 0x0002 ;
+    static const uint32_t FLAGS_ENTRY_UP_TO_DATE  = 0x0004 ;
+    static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ;
+};
+
+/*!
+ * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date.
+ */
+class RsFileListsSyncReqItem : public RsFileListsItem
+{
+public:
+
+	RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {}
+
+	virtual void clear();
+	virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+	virtual bool serialise(void *data,uint32_t& size) const;	
+	virtual uint32_t serial_size() const ; 			
+        
+    uint32_t entry_index ;              // index of the directory to sync
+    uint32_t flags;                     // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
+    uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
+    uint64_t request_id;                // use to determine if changes that have occured since last hash
+};
+
+class RsFileListsSyncDirItem : public RsFileListsItem
+{
+public:
+
+	RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {}
+
+	virtual void clear();
+	virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+	virtual bool serialise(void *data,uint32_t& size) const;
+	virtual uint32_t serial_size() const ;
+
+	uint32_t entry_index ;              // advises whether to use sync hash
+	uint32_t flags;                     // is it a partial/final item (used for large items only)
+	uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
+	uint64_t request_id;                // use to determine if changes that have occured since last hash
+
+	RsTlvBinaryData directory_content_data ;	// encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
+};
+
+class RsFileListsSerialiser : public RsSerialType
+{
+public:
+
+    RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {}
+
+    virtual ~RsFileListsSerialiser() {}
+
+    virtual uint32_t size(RsItem *item);
+    virtual bool serialise(RsItem *item, void *data, uint32_t *size);
+    virtual RsItem* deserialise(void *data, uint32_t *size);
+
+private:
+    RsFileListsSyncReqItem  *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
+    RsFileListsSyncDirItem  *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
+    RsFileListsSyncDirItem  *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
+
+    bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type);
+};
+
+

libretroshare/src/libretroshare.pro

@@ -37,16 +37,18 @@ CONFIG += file_lists
 
 file_lists {
 	HEADERS *= file_sharing/p3filelists.h \
-				file_sharing/hash_cache.h \
+			file_sharing/hash_cache.h \
-                                file_sharing/filelist_io.h \
+			file_sharing/filelist_io.h \
-                                file_sharing/directory_storage.h \
+			file_sharing/directory_storage.h \
-				file_sharing/directory_updater.h
+			file_sharing/directory_updater.h \
+			file_sharing/filelistitems.h
 
 	SOURCES *= file_sharing/p3filelists.cc \
-				file_sharing/hash_cache.cc \
+			file_sharing/hash_cache.cc \
-                                file_sharing/filelist_io.cc \
+			file_sharing/filelist_io.cc \
-                                file_sharing/directory_storage.cc \
+			file_sharing/directory_storage.cc \
-				file_sharing/directory_updater.cc
+			file_sharing/directory_updater.cc \
+			file_sharing/filelistitems.cc
 }
 
 dsdv {

libretroshare/src/serialiser/itempriorities.h

@@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST       		= 2 ; // same priority.
 const uint8_t QOS_PRIORITY_RS_DISC_SERVICES    			= 2 ;
 const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT    			= 1 ;
 
+// File database
+//
+const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST         = 7 ;
+const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST         = 3 ;
+
 // Heartbeat.
 //
 const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE 			= 8 ;