mirror of
https://github.com/RetroShare/RetroShare.git
synced 2024-12-15 02:44:20 -05:00
Implemented load/save of chunk maps for current downloads.
The side effect is that even when stopped, downloads show the correct downloaded size (up to the size of one chunk). git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1865 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler
parent
4d190963cf
commit
b6c048a5fd
@ -67,6 +67,35 @@ ChunkMap::ChunkMap(uint64_t s)
|
||||
#endif
|
||||
}
|
||||
|
||||
ChunkMap::ChunkMap(uint64_t file_size,
|
||||
const std::vector<uint32_t>& map,
|
||||
uint32_t chunk_size,
|
||||
uint32_t chunk_number,
|
||||
FileChunksInfo::ChunkStrategy strategy)
|
||||
|
||||
:_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
|
||||
{
|
||||
#ifdef DEBUG_FTCHUNK
|
||||
std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
|
||||
#endif
|
||||
|
||||
_map.clear() ;
|
||||
_map.resize(chunk_number) ;
|
||||
_total_downloaded = 0 ;
|
||||
|
||||
for(uint32_t i=0;i<_map.size();++i)
|
||||
{
|
||||
uint32_t j = i & 31 ; // i%32
|
||||
uint32_t k = i >> 5 ; // i/32
|
||||
|
||||
_map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
|
||||
|
||||
if(_map[i] == FileChunksInfo::CHUNK_DONE)
|
||||
_total_downloaded += _chunk_size ;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
|
||||
{
|
||||
// 1 - find which chunk contains the received data.
|
||||
@ -247,5 +276,31 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const
|
||||
info.chunks = _map ;
|
||||
}
|
||||
|
||||
void ChunkMap::buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const
|
||||
{
|
||||
chunk_size = _chunk_size ;
|
||||
chunk_number = _map.size() ;
|
||||
strategy = _strategy ;
|
||||
|
||||
map.clear() ;
|
||||
map.reserve((chunk_number >> 5)+1) ;
|
||||
|
||||
uint32_t r=0 ;
|
||||
for(uint32_t i=0;i<_map.size();++i)
|
||||
{
|
||||
uint32_t j = i & 31 ;
|
||||
r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<<j):0 ;
|
||||
|
||||
if(j==31 || i==_map.size()-1)
|
||||
{
|
||||
map.push_back(r);
|
||||
r=0 ;
|
||||
}
|
||||
}
|
||||
#ifdef DEBUG_FTCHUNK
|
||||
std::cerr << "ChunkMap:: built availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << ", strategy=" << _strategy << std::endl ;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -80,6 +80,11 @@ class ChunkMap
|
||||
// Constructor. Decides what will be the size of chunks and how many there will be.
|
||||
|
||||
ChunkMap(uint64_t file_size) ;
|
||||
|
||||
// constructor from saved map info
|
||||
ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
|
||||
|
||||
// destructor
|
||||
virtual ~ChunkMap() {}
|
||||
|
||||
// Returns an slice of data to be asked to the peer within a chunk.
|
||||
@ -106,13 +111,14 @@ class ChunkMap
|
||||
|
||||
void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; }
|
||||
|
||||
#ifdef TO_DO
|
||||
// Properly fills an vector of fixed size chunks with availability or download state.
|
||||
// chunks is given with the proper number of chunks and we have to adapt to it. This can be used
|
||||
// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
|
||||
|
||||
void linearize(FileChunksInfo& info) const ;
|
||||
void buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& s) const ;
|
||||
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
|
||||
|
||||
#ifdef TO_DO
|
||||
// Updates the peer's availablility map
|
||||
//
|
||||
void setPeerAvailabilityMap(const std::string& peer_id,const std::vector<uint32_t>& peer_map) ;
|
||||
@ -131,17 +137,17 @@ class ChunkMap
|
||||
uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id) const ;
|
||||
|
||||
private:
|
||||
const uint64_t _file_size ; // total size of the file in bytes.
|
||||
const uint32_t _chunk_size ; // Size of chunks. Common to all chunks.
|
||||
FileChunksInfo::ChunkStrategy _strategy ; // how do we allocate new chunks
|
||||
std::map<std::string,Chunk> _active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
|
||||
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size)
|
||||
uint64_t _file_size ; // total size of the file in bytes.
|
||||
uint32_t _chunk_size ; // Size of chunks. Common to all chunks.
|
||||
FileChunksInfo::ChunkStrategy _strategy ; // how do we allocate new chunks
|
||||
std::map<std::string,Chunk> _active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
|
||||
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size)
|
||||
|
||||
std::vector<FileChunksInfo::ChunkState> _map ; // vector of chunk state over the whole file
|
||||
std::vector<FileChunksInfo::ChunkState> _map ; // vector of chunk state over the whole file
|
||||
|
||||
std::map<std::string,std::vector<uint32_t> > _peers_chunks_availability ; // what does each source peer have.
|
||||
|
||||
uint64_t _total_downloaded ;
|
||||
uint64_t _total_downloaded ;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -57,6 +57,8 @@
|
||||
* #define CONTROL_DEBUG 1
|
||||
*****/
|
||||
|
||||
static const uint32_t SAVE_TRANSFERS_DELAY = 30 ; // save transfer progress every 30 seconds.
|
||||
|
||||
ftFileControl::ftFileControl()
|
||||
:mTransfer(NULL), mCreator(NULL),
|
||||
mState(0), mSize(0), mFlags(0)
|
||||
@ -79,7 +81,7 @@ ftFileControl::ftFileControl(std::string fname,
|
||||
|
||||
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir)
|
||||
:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL), mDataplex(dm), mFtActive(false),
|
||||
mTurtle(NULL), mShareDownloadDir(true)
|
||||
mTurtle(NULL), mShareDownloadDir(true),last_save_time(0)
|
||||
{
|
||||
/* TODO */
|
||||
}
|
||||
@ -182,6 +184,13 @@ void ftController::run()
|
||||
doPending = (mFtActive) && (!mFtPendingDone);
|
||||
}
|
||||
|
||||
time_t now = time(NULL) ;
|
||||
if(now - last_save_time > SAVE_TRANSFERS_DELAY)
|
||||
{
|
||||
IndicateConfigChanged() ;
|
||||
last_save_time = now ;
|
||||
}
|
||||
|
||||
if (doPending)
|
||||
{
|
||||
if (!handleAPendingRequest())
|
||||
@ -546,17 +555,44 @@ bool ftController::isActiveAndNoPending()
|
||||
bool ftController::handleAPendingRequest()
|
||||
{
|
||||
ftPendingRequest req;
|
||||
{ RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
|
||||
{
|
||||
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
|
||||
|
||||
if (mPendingRequests.size() < 1)
|
||||
{
|
||||
return false;
|
||||
if (mPendingRequests.size() < 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
req = mPendingRequests.front();
|
||||
mPendingRequests.pop_front();
|
||||
}
|
||||
req = mPendingRequests.front();
|
||||
mPendingRequests.pop_front();
|
||||
}
|
||||
|
||||
FileRequest(req.mName, req.mHash, req.mSize, req.mDest, req.mFlags, req.mSrcIds);
|
||||
return true;
|
||||
|
||||
{
|
||||
// See whether there is a pendign chunk map recorded for this hash.
|
||||
//
|
||||
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
|
||||
|
||||
std::map<std::string,RsFileTransfer*>::iterator it(mPendingChunkMaps.find(req.mHash)) ;
|
||||
|
||||
if(it != mPendingChunkMaps.end())
|
||||
{
|
||||
RsFileTransfer *rsft = it->second ;
|
||||
std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
|
||||
|
||||
if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
|
||||
{
|
||||
// This should never happen, because the last call to FileRequest must have created the fileCreator!!
|
||||
//
|
||||
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
|
||||
}
|
||||
else
|
||||
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
|
||||
|
||||
delete rsft ;
|
||||
mPendingChunkMaps.erase(it) ;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1308,8 +1344,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h
|
||||
std::list<std::string> ids;
|
||||
ids.push_back(id);
|
||||
|
||||
FileRequest(hash, hash, size, path,
|
||||
RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
|
||||
FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -1422,6 +1457,7 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
|
||||
//rft->flags = fit->second.mFlags;
|
||||
|
||||
fit->second.mTransfer->getFileSources(rft->allPeerIds.ids);
|
||||
fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ;
|
||||
|
||||
saveData.push_back(rft);
|
||||
}
|
||||
@ -1461,17 +1497,27 @@ bool ftController::loadList(std::list<RsItem *> load)
|
||||
}
|
||||
else if (NULL != (rsft = dynamic_cast<RsFileTransfer *>(*it)))
|
||||
{
|
||||
// csoler: I'm suppressing this lock since there is a double lock below
|
||||
// in FileRequest, line 382.
|
||||
// RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
|
||||
//
|
||||
|
||||
/* This will get stored on a waiting list - until the
|
||||
* config files are fully loaded
|
||||
*/
|
||||
FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize,
|
||||
rsft->file.path, 0, rsft->allPeerIds.ids);
|
||||
FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, rsft->file.path, 0, rsft->allPeerIds.ids);
|
||||
|
||||
{
|
||||
RsStackMutex mtx(ctrlMutex) ;
|
||||
|
||||
std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
|
||||
|
||||
if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
|
||||
{
|
||||
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
|
||||
std::cerr << "Storing the map in a wait list." << std::endl ;
|
||||
|
||||
mPendingChunkMaps[rsft->file.hash] = rsft ;
|
||||
continue ; // i.e. don't delete the item!
|
||||
}
|
||||
else
|
||||
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
|
||||
}
|
||||
}
|
||||
|
||||
/* cleanup */
|
||||
|
||||
@ -189,6 +189,7 @@ bool handleAPendingRequest();
|
||||
bool setPeerState(ftTransferModule *tm, std::string id,
|
||||
uint32_t maxrate, bool online);
|
||||
|
||||
time_t last_save_time ;
|
||||
/* pointers to other components */
|
||||
|
||||
ftSearch *mSearch;
|
||||
@ -224,6 +225,7 @@ bool setPeerState(ftTransferModule *tm, std::string id,
|
||||
bool mFtActive;
|
||||
bool mFtPendingDone;
|
||||
std::list<ftPendingRequest> mPendingRequests;
|
||||
std::map<std::string,RsFileTransfer*> mPendingChunkMaps ;
|
||||
|
||||
/* share incoming directory */
|
||||
bool mShareDownloadDir;
|
||||
|
||||
@ -403,4 +403,19 @@ bool ftFileCreator::locked_printChunkMap()
|
||||
return true;
|
||||
}
|
||||
|
||||
void ftFileCreator::loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy)
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ;
|
||||
}
|
||||
void ftFileCreator::storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy)
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
FileChunksInfo::ChunkStrategy strat ;
|
||||
chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ;
|
||||
strategy = (uint32_t)strat ;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -59,6 +59,9 @@ virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
|
||||
bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size);
|
||||
bool addFileData(uint64_t offset, uint32_t chunk_size, void *data);
|
||||
|
||||
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ;
|
||||
void storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ;
|
||||
|
||||
protected:
|
||||
|
||||
virtual int initializeFileAttrs();
|
||||
|
||||
@ -31,8 +31,6 @@
|
||||
#define RSSERIAL_DEBUG 1
|
||||
***/
|
||||
|
||||
#define RSSERIAL_DEBUG 1
|
||||
|
||||
#include <iostream>
|
||||
|
||||
/*************************************************************************/
|
||||
@ -198,6 +196,11 @@ uint32_t RsFileConfigSerialiser::sizeTransfer(RsFileTransfer *item)
|
||||
s += 4; /* trate */
|
||||
s += 4; /* lrate */
|
||||
s += 4; /* ltransfer */
|
||||
s += 4; // chunk_size
|
||||
s += 4; // chunk_number
|
||||
s += 4; // chunk_strategy
|
||||
s += 4; // chunk map size
|
||||
s += 4*item->chunk_map.size(); // chunk_map
|
||||
|
||||
return s;
|
||||
}
|
||||
@ -240,6 +243,14 @@ bool RsFileConfigSerialiser::serialiseTransfer(RsFileTransfer *item, void *d
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->lrate);
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer);
|
||||
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_size);
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number);
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy);
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map.size());
|
||||
|
||||
for(uint32_t i=0;i<item->chunk_map.size();++i)
|
||||
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map[i]);
|
||||
|
||||
if (offset != tlvsize)
|
||||
{
|
||||
ok = false;
|
||||
@ -299,6 +310,16 @@ RsFileTransfer *RsFileConfigSerialiser::deserialiseTransfer(void *data, uint32_t
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->lrate));
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer));
|
||||
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_size));
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy));
|
||||
uint32_t map_size = 0 ;
|
||||
ok &= getRawUInt32(data, rssize, &offset, &map_size);
|
||||
|
||||
item->chunk_map.resize(map_size) ;
|
||||
for(uint32_t i=0;i<map_size;++i)
|
||||
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_map[i]));
|
||||
|
||||
if (offset != rssize)
|
||||
{
|
||||
/* error */
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
*/
|
||||
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
#include "serialiser/rsserial.h"
|
||||
#include "serialiser/rstlvbase.h"
|
||||
@ -179,31 +180,34 @@ virtual RsItem * deserialise(void *data, uint32_t *size);
|
||||
class RsFileTransfer: public RsItem
|
||||
{
|
||||
public:
|
||||
RsFileTransfer()
|
||||
:RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG,
|
||||
RS_PKT_TYPE_FILE_CONFIG,
|
||||
RS_PKT_SUBTYPE_FILE_TRANSFER)
|
||||
{ return; }
|
||||
virtual ~RsFileTransfer();
|
||||
virtual void clear();
|
||||
std::ostream &print(std::ostream &out, uint16_t indent = 0);
|
||||
RsFileTransfer() :RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, RS_PKT_TYPE_FILE_CONFIG, RS_PKT_SUBTYPE_FILE_TRANSFER)
|
||||
{
|
||||
return;
|
||||
}
|
||||
virtual ~RsFileTransfer();
|
||||
virtual void clear();
|
||||
std::ostream &print(std::ostream &out, uint16_t indent = 0);
|
||||
|
||||
RsTlvFileItem file;
|
||||
RsTlvPeerIdSet allPeerIds;
|
||||
RsTlvFileItem file;
|
||||
RsTlvPeerIdSet allPeerIds;
|
||||
|
||||
std::string cPeerId;
|
||||
std::string cPeerId;
|
||||
|
||||
uint16_t state;
|
||||
uint16_t in;
|
||||
uint16_t state;
|
||||
uint16_t in;
|
||||
|
||||
uint64_t transferred;
|
||||
uint32_t crate;
|
||||
uint32_t trate;
|
||||
uint64_t transferred;
|
||||
uint32_t crate;
|
||||
uint32_t trate;
|
||||
|
||||
uint32_t lrate;
|
||||
uint32_t ltransfer;
|
||||
|
||||
uint32_t lrate;
|
||||
uint32_t ltransfer;
|
||||
|
||||
// chunk information
|
||||
uint32_t chunk_size ; // common size of chunks
|
||||
uint32_t chunk_number ; // total number of chunks (this is not redondant, cause chunks are compressed)
|
||||
uint32_t chunk_strategy ; // strategy flags for chunks
|
||||
std::vector<uint32_t> chunk_map ; // chunk availability (bitwise)
|
||||
};
|
||||
|
||||
/**************************************************************************/
|
||||
|
||||
Loading…
Reference in New Issue
Block a user