diff --git a/libretroshare/src/ft/ftchunkmap.cc b/libretroshare/src/ft/ftchunkmap.cc index 2af63480f..8348854ee 100644 --- a/libretroshare/src/ft/ftchunkmap.cc +++ b/libretroshare/src/ft/ftchunkmap.cc @@ -67,6 +67,35 @@ ChunkMap::ChunkMap(uint64_t s) #endif } +ChunkMap::ChunkMap(uint64_t file_size, + const std::vector& map, + uint32_t chunk_size, + uint32_t chunk_number, + FileChunksInfo::ChunkStrategy strategy) + + :_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy) +{ +#ifdef DEBUG_FTCHUNK + std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ; +#endif + + _map.clear() ; + _map.resize(chunk_number) ; + _total_downloaded = 0 ; + + for(uint32_t i=0;i<_map.size();++i) + { + uint32_t j = i & 31 ; // i%32 + uint32_t k = i >> 5 ; // i/32 + + _map[i] = ( (map[k] & (1< 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ; + + if(_map[i] == FileChunksInfo::CHUNK_DONE) + _total_downloaded += _chunk_size ; + } +} + + void ChunkMap::dataReceived(const ftChunk::ChunkId& cid) { // 1 - find which chunk contains the received data. @@ -247,5 +276,31 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const info.chunks = _map ; } +void ChunkMap::buildAvailabilityMap(std::vector& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const +{ + chunk_size = _chunk_size ; + chunk_number = _map.size() ; + strategy = _strategy ; + + map.clear() ; + map.reserve((chunk_number >> 5)+1) ; + + uint32_t r=0 ; + for(uint32_t i=0;i<_map.size();++i) + { + uint32_t j = i & 31 ; + r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<file.hash << " in mDownloads list !" << std::endl ; + } + else + (fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ; + + delete rsft ; + mPendingChunkMaps.erase(it) ; + } + } } @@ -1308,8 +1344,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h std::list ids; ids.push_back(id); - FileRequest(hash, hash, size, path, - RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids); + FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids); return true; } @@ -1422,6 +1457,7 @@ std::list ftController::saveList(bool &cleanup) //rft->flags = fit->second.mFlags; fit->second.mTransfer->getFileSources(rft->allPeerIds.ids); + fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ; saveData.push_back(rft); } @@ -1461,17 +1497,27 @@ bool ftController::loadList(std::list load) } else if (NULL != (rsft = dynamic_cast(*it))) { -// csoler: I'm suppressing this lock since there is a double lock below -// in FileRequest, line 382. -// RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ -// - /* This will get stored on a waiting list - until the * config files are fully loaded */ - FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, - rsft->file.path, 0, rsft->allPeerIds.ids); + FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, rsft->file.path, 0, rsft->allPeerIds.ids); + { + RsStackMutex mtx(ctrlMutex) ; + + std::map::iterator fit = mDownloads.find(rsft->file.hash); + + if((fit==mDownloads.end() || (fit->second).mCreator == NULL)) + { + std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ; + std::cerr << "Storing the map in a wait list." << std::endl ; + + mPendingChunkMaps[rsft->file.hash] = rsft ; + continue ; // i.e. don't delete the item! + } + else + (fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ; + } } /* cleanup */ diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index f597bed15..92da29bad 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -189,6 +189,7 @@ bool handleAPendingRequest(); bool setPeerState(ftTransferModule *tm, std::string id, uint32_t maxrate, bool online); + time_t last_save_time ; /* pointers to other components */ ftSearch *mSearch; @@ -224,6 +225,7 @@ bool setPeerState(ftTransferModule *tm, std::string id, bool mFtActive; bool mFtPendingDone; std::list mPendingRequests; + std::map mPendingChunkMaps ; /* share incoming directory */ bool mShareDownloadDir; diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index d7237b715..af84e2a9b 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -403,4 +403,19 @@ bool ftFileCreator::locked_printChunkMap() return true; } +void ftFileCreator::loadAvailabilityMap(const std::vector& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy) +{ + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + + chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ; +} +void ftFileCreator::storeAvailabilityMap(std::vector& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy) +{ + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + + FileChunksInfo::ChunkStrategy strat ; + chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ; + strategy = (uint32_t)strat ; +} + diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index 91b14c19b..5b686fd44 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -59,6 +59,9 @@ virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data); bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size); bool addFileData(uint64_t offset, uint32_t chunk_size, void *data); + void loadAvailabilityMap(const std::vector& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ; + void storeAvailabilityMap(std::vector& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ; + protected: virtual int initializeFileAttrs(); diff --git a/libretroshare/src/serialiser/rsconfigitems.cc b/libretroshare/src/serialiser/rsconfigitems.cc index e60072998..c3bf0c7ea 100644 --- a/libretroshare/src/serialiser/rsconfigitems.cc +++ b/libretroshare/src/serialiser/rsconfigitems.cc @@ -31,8 +31,6 @@ #define RSSERIAL_DEBUG 1 ***/ -#define RSSERIAL_DEBUG 1 - #include /*************************************************************************/ @@ -198,6 +196,11 @@ uint32_t RsFileConfigSerialiser::sizeTransfer(RsFileTransfer *item) s += 4; /* trate */ s += 4; /* lrate */ s += 4; /* ltransfer */ + s += 4; // chunk_size + s += 4; // chunk_number + s += 4; // chunk_strategy + s += 4; // chunk map size + s += 4*item->chunk_map.size(); // chunk_map return s; } @@ -240,6 +243,14 @@ bool RsFileConfigSerialiser::serialiseTransfer(RsFileTransfer *item, void *d ok &= setRawUInt32(data, tlvsize, &offset, item->lrate); ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer); + ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_size); + ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number); + ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy); + ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map.size()); + + for(uint32_t i=0;ichunk_map.size();++i) + ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map[i]); + if (offset != tlvsize) { ok = false; @@ -299,6 +310,16 @@ RsFileTransfer *RsFileConfigSerialiser::deserialiseTransfer(void *data, uint32_t ok &= getRawUInt32(data, rssize, &offset, &(item->lrate)); ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer)); + ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_size)); + ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number)); + ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy)); + uint32_t map_size = 0 ; + ok &= getRawUInt32(data, rssize, &offset, &map_size); + + item->chunk_map.resize(map_size) ; + for(uint32_t i=0;ichunk_map[i])); + if (offset != rssize) { /* error */ diff --git a/libretroshare/src/serialiser/rsconfigitems.h b/libretroshare/src/serialiser/rsconfigitems.h index 230de4f5f..c079d8279 100644 --- a/libretroshare/src/serialiser/rsconfigitems.h +++ b/libretroshare/src/serialiser/rsconfigitems.h @@ -27,6 +27,7 @@ */ #include +#include #include "serialiser/rsserial.h" #include "serialiser/rstlvbase.h" @@ -179,31 +180,34 @@ virtual RsItem * deserialise(void *data, uint32_t *size); class RsFileTransfer: public RsItem { public: - RsFileTransfer() - :RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, - RS_PKT_TYPE_FILE_CONFIG, - RS_PKT_SUBTYPE_FILE_TRANSFER) - { return; } -virtual ~RsFileTransfer(); -virtual void clear(); -std::ostream &print(std::ostream &out, uint16_t indent = 0); + RsFileTransfer() :RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, RS_PKT_TYPE_FILE_CONFIG, RS_PKT_SUBTYPE_FILE_TRANSFER) + { + return; + } + virtual ~RsFileTransfer(); + virtual void clear(); + std::ostream &print(std::ostream &out, uint16_t indent = 0); - RsTlvFileItem file; - RsTlvPeerIdSet allPeerIds; + RsTlvFileItem file; + RsTlvPeerIdSet allPeerIds; - std::string cPeerId; + std::string cPeerId; - uint16_t state; - uint16_t in; + uint16_t state; + uint16_t in; - uint64_t transferred; - uint32_t crate; - uint32_t trate; + uint64_t transferred; + uint32_t crate; + uint32_t trate; + uint32_t lrate; + uint32_t ltransfer; - uint32_t lrate; - uint32_t ltransfer; - + // chunk information + uint32_t chunk_size ; // common size of chunks + uint32_t chunk_number ; // total number of chunks (this is not redondant, cause chunks are compressed) + uint32_t chunk_strategy ; // strategy flags for chunks + std::vector chunk_map ; // chunk availability (bitwise) }; /**************************************************************************/