fixed a few bugs in serialisation of shared dir content

libretroshare/src/file_sharing/directory_storage.cc

@@ -391,10 +391,8 @@ public:
     }
     uint32_t getType(DirectoryStorage::EntryIndex indx) const
     {
-        if(checkIndex(indx,FileStorageNode::TYPE_FILE))
+        if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR))
-            return FileStorageNode::TYPE_FILE;
+            return mNodes[indx]->type() ;
-        else if(checkIndex(indx,FileStorageNode::TYPE_DIR))
-            return FileStorageNode::TYPE_DIR;
         else
             return FileStorageNode::TYPE_UNKNOWN;
     }
@@ -751,17 +749,14 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
     d.children.clear() ;
     time_t now = time(NULL) ;
 
-    std::cerr << "LocalDirectoryStorage::extractData(). Index=" << indx << std::endl;
+    uint32_t type = mFileHierarchy->getType(indx) ;
-
-    const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ;
 
     d.ref = (void*)(intptr_t)indx ;
 
-    if (dir_entry != NULL) /* has children --- fill */
+    if (type == InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR) /* has children --- fill */
     {
-#ifdef DEBUG_DIRECTORY_STORAGE
+        const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ;
-        std::cerr << "FileIndex::extractData() ref=dir" << std::endl;
+
-#endif
         /* extract all the entries */
 
         for(DirectoryStorage::DirIterator it(this,indx);it;++it)
@@ -798,12 +793,10 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
             d.name = mPeerId.toStdString();
         }
     }
-    else
+    else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)
     {
         const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ;
-#ifdef DEBUG_DIRECTORY_STORAGE
+
-        std::cerr << "FileIndexStore::extractData() ref=file" << std::endl;
-#endif
         d.type    = DIR_TYPE_FILE;
         d.count   = file_entry->file_size;
         d.min_age = now - file_entry->file_modtime ;
@@ -819,6 +812,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
         else
             d.path = "" ;
     }
+    else
+        return false;
 
     d.flags.clear() ;
 

libretroshare/src/file_sharing/filelist_io.h

@@ -24,6 +24,8 @@ static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER         =  0x0a ;
 static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX        =  0x0b ;
 static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY  =  0x0c ;
 
+static const uint32_t SECTION_HEADER_MAX_SIZE            =  6 ;   // section tag (1 byte) + size (max = 5 bytes)
+
 class FileListIO
 {
 public:
@@ -68,9 +70,9 @@ private:
 
     static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S)
     {
-        if(offset + S > buff_size)
+        if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size)
         {
-            buff = (unsigned char *)realloc(buff,offset + S) ;
+            buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ;
             buff_size = offset + S ;
 
             if(!buff)

libretroshare/src/file_sharing/p3filelists.cc

@@ -21,6 +21,7 @@ static const uint32_t NB_FRIEND_INDEX_BITS                    = 10 ;
 static const uint32_t NB_ENTRY_INDEX_BITS                     = 22 ;
 static const uint32_t ENTRY_INDEX_BIT_MASK                    = 0x003fffff ;	// used for storing (EntryIndex,Friend) couples into a 32bits pointer.
 static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; 			// every minute, for debugging. Should be evey 10 minutes or so.
+static const uint32_t DELAY_BEFORE_DROP_REQUEST               = 55 ; 			// every 55 secs, for debugging. Should be evey 10 minutes or so.
 
 p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
     : mServCtrl(mpeers), mFLSMtx("p3FileLists")
@@ -37,8 +38,18 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
 
 	mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ;
     mLastRemoteDirSweepTS = 0 ;
+
+    addSerialType(new RsFileListsSerialiser()) ;
 }
 
+RsSerialiser *p3FileDatabase::setupSerialiser()
+{
+    RsSerialiser *rss = new RsSerialiser ;
+    rss->addSerialType(new RsFileListsSerialiser()) ;
+    rss->addSerialType(new RsGeneralConfigSerialiser());
+
+    return rss ;
+}
 void p3FileDatabase::setSharedDirectories(const std::list<SharedDirInfo>& shared_dirs)
 {
     RS_STACK_MUTEX(mFLSMtx) ;
@@ -97,13 +108,22 @@ int p3FileDatabase::tick()
 	tickRecv() ;
 	tickSend() ;
 
+    time_t now = time(NULL) ;
+
     // cleanup
 	// 	- remove/delete shared file lists for people who are not friend anymore
 	// 	- 
+
+    static time_t last_cleanup_time = 0;
+
+#warning we should use members here, not static
+    if(last_cleanup_time + 5 < now)
+    {
         cleanup();
+        last_cleanup_time = now ;
+    }
 
     static time_t last_print_time = 0;
-    time_t now = time(NULL) ;
 
     if(last_print_time + 20 < now)
     {
@@ -133,7 +153,11 @@ int p3FileDatabase::tick()
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
+        std::set<RsPeerId> online_peers ;
+        mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ;
+
         for(uint32_t i=0;i<mRemoteDirectories.size();++i)
+            if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end())
                 locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ;
 
         mLastRemoteDirSweepTS = now;
@@ -186,6 +210,8 @@ void p3FileDatabase::cleanup()
     {
         RS_STACK_MUTEX(mFLSMtx) ;
 
+        std::cerr << "p3FileDatabase::cleanup()" << std::endl;
+
         // look through the list of friend directories. Remove those who are not our friends anymore.
         //
         std::set<RsPeerId> friend_set ;
@@ -234,6 +260,29 @@ void p3FileDatabase::cleanup()
 
             mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ;
         }
+
+        // cancel existing requests for which the peer is offline
+
+        std::set<RsPeerId> online_peers ;
+        mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ;
+
+        time_t now = time(NULL);
+
+        for(std::map<DirSyncRequestId,DirSyncRequestData>::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();)
+            if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now)
+            {
+                std::cerr << "  removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl;
+
+                std::map<DirSyncRequestId,DirSyncRequestData>::iterator tmp(it);
+                ++tmp;
+                mPendingSyncRequests.erase(it) ;
+                it = tmp;
+            }
+            else
+            {
+                std::cerr << "  keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl;
+                ++it ;
+            }
     }
 }
 
@@ -405,8 +454,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
         d.count = d.children.size();
 
+#ifdef DEBUG_FILE_HIERARCHY
         std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;
         std::cerr << d << std::endl;
+#endif
 
         return true ;
     }
@@ -453,8 +504,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
     d.id = storage->peerId();
 
+#ifdef DEBUG_FILE_HIERARCHY
     std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;
     std::cerr << d << std::endl;
+#endif
 
     return true;
 }
@@ -801,13 +854,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
             return ;
         }
 
-        P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl;
+        P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl;
-
-        DirSyncRequestData data ;
-
-        data.request_TS = now ;
-
-        mPendingSyncRequests[sync_req_id] = data ;
 
         RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ;
         item->entry_index =  e ;
@@ -816,7 +863,17 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r
         item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ;
         item->PeerId(rds->peerId()) ;
 
-        sendItem(item) ;
+        DirSyncRequestData data ;
+
+        data.request_TS = now ;
+        data.peer_id = item->PeerId();
+        data.flags = item->flags;
+
+        std::cerr << "Pushing req in pending list with peer id " << data.peer_id << std::endl;
+
+        mPendingSyncRequests[sync_req_id] = data ;
+
+        sendItem(item) ;	// at end! Because item is destroyed by the process.
 
         // Dont recurs into sub-directories, since we dont know yet were to go.
 

libretroshare/src/file_sharing/p3filelists.h

@@ -129,7 +129,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         //
         virtual bool loadList(std::list<RsItem *>& items);
         virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
-        virtual RsSerialiser *setupSerialiser() { return NULL;}
+        virtual RsSerialiser *setupSerialiser() ;
 
         void cleanup();
         void tickRecv();
@@ -177,6 +177,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         //
         struct DirSyncRequestData
         {
+            RsPeerId peer_id ;
             time_t request_TS ;
             uint32_t flags ;
         };

libretroshare/src/file_sharing/rsfilelistitems.cc

@@ -196,7 +196,7 @@ RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestI
 }
 RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size)
 {
-    bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
+    bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM);
     uint32_t offset = 8;
 
     RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem();

libretroshare/src/file_sharing/rsfilelistitems.h

@@ -114,7 +114,7 @@ class RsFileListsSerialiser : public RsSerialType
 {
 public:
 
-    RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {}
+    RsFileListsSerialiser() : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {}
 
     virtual ~RsFileListsSerialiser() {}