Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-15 02:44:20 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implemented load/save of chunk maps for current downloads.

Browse Source 
The side effect is that even when stopped, downloads show the correct downloaded size (up to the size of one chunk).



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1865 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2009-12-10 22:55:27 +00:00
parent 4d190963cf
commit b6c048a5fd
8 changed files with 200 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
libretroshare/src/ft/ftchunkmap.cc
View File

@ -67,6 +67,35 @@ ChunkMap::ChunkMap(uint64_t s)
#endif
}
ChunkMap::ChunkMap(uint64_t file_size,
const std::vector<uint32_t>& map,
uint32_t chunk_size,
uint32_t chunk_number,
FileChunksInfo::ChunkStrategy strategy)
:_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
{
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
#endif
_map.clear() ;
_map.resize(chunk_number) ;
_total_downloaded = 0 ;
for(uint32_t i=0;i<_map.size();++i)
{
uint32_t j = i & 31 ; // i%32
uint32_t k = i >> 5 ; // i/32
_map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
if(_map[i] == FileChunksInfo::CHUNK_DONE)
_total_downloaded += _chunk_size ;
}
}
void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
{
// 1 - find which chunk contains the received data.
@ -247,5 +276,31 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const
info.chunks = _map ;
}
void ChunkMap::buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const
{
chunk_size = _chunk_size ;
chunk_number = _map.size() ;
strategy = _strategy ;
map.clear() ;
map.reserve((chunk_number >> 5)+1) ;
uint32_t r=0 ;
for(uint32_t i=0;i<_map.size();++i)
{
uint32_t j = i & 31 ;
r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<<j):0 ;
if(j==31 || i==_map.size()-1)
{
map.push_back(r);
r=0 ;
}
}
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: built availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << ", strategy=" << _strategy << std::endl ;
#endif
}

14
libretroshare/src/ft/ftchunkmap.h
View File

@ -80,6 +80,11 @@ class ChunkMap
// Constructor. Decides what will be the size of chunks and how many there will be.
ChunkMap(uint64_t file_size) ;
// constructor from saved map info
ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
// destructor
virtual ~ChunkMap() {}
// Returns an slice of data to be asked to the peer within a chunk.
@ -106,13 +111,14 @@ class ChunkMap
void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; }
#ifdef TO_DO
// Properly fills an vector of fixed size chunks with availability or download state.
// chunks is given with the proper number of chunks and we have to adapt to it. This can be used
// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
void linearize(FileChunksInfo& info) const ;
void buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& s) const ;
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
#ifdef TO_DO
// Updates the peer's availablility map
//
void setPeerAvailabilityMap(const std::string& peer_id,const std::vector<uint32_t>& peer_map) ;
@ -131,8 +137,8 @@ class ChunkMap
uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id) const ;
private:
const uint64_t _file_size ; // total size of the file in bytes.
const uint32_t _chunk_size ; // Size of chunks. Common to all chunks.
uint64_t _file_size ; // total size of the file in bytes.
uint32_t _chunk_size ; // Size of chunks. Common to all chunks.
FileChunksInfo::ChunkStrategy _strategy ; // how do we allocate new chunks
std::map<std::string,Chunk> _active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size)

70
libretroshare/src/ft/ftcontroller.cc
View File

@ -57,6 +57,8 @@
* #define CONTROL_DEBUG 1
*****/
static const uint32_t SAVE_TRANSFERS_DELAY = 30 ; // save transfer progress every 30 seconds.
ftFileControl::ftFileControl()
:mTransfer(NULL), mCreator(NULL),
mState(0), mSize(0), mFlags(0)
@ -79,7 +81,7 @@ ftFileControl::ftFileControl(std::string fname,
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir)
:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL), mDataplex(dm), mFtActive(false),
mTurtle(NULL), mShareDownloadDir(true)
mTurtle(NULL), mShareDownloadDir(true),last_save_time(0)
{
/* TODO */
}
@ -182,6 +184,13 @@ void ftController::run()
doPending = (mFtActive) && (!mFtPendingDone);
}
time_t now = time(NULL) ;
if(now - last_save_time > SAVE_TRANSFERS_DELAY)
{
IndicateConfigChanged() ;
last_save_time = now ;
}
if (doPending)
{
if (!handleAPendingRequest())
@ -546,7 +555,8 @@ bool ftController::isActiveAndNoPending()
bool ftController::handleAPendingRequest()
{
ftPendingRequest req;
{ RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
if (mPendingRequests.size() < 1)
{
@ -555,8 +565,34 @@ bool ftController::handleAPendingRequest()
req = mPendingRequests.front();
mPendingRequests.pop_front();
}
FileRequest(req.mName, req.mHash, req.mSize, req.mDest, req.mFlags, req.mSrcIds);
return true;
{
// See whether there is a pendign chunk map recorded for this hash.
//
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
std::map<std::string,RsFileTransfer*>::iterator it(mPendingChunkMaps.find(req.mHash)) ;
if(it != mPendingChunkMaps.end())
{
RsFileTransfer *rsft = it->second ;
std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
{
// This should never happen, because the last call to FileRequest must have created the fileCreator!!
//
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
}
else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
delete rsft ;
mPendingChunkMaps.erase(it) ;
}
}
}
@ -1308,8 +1344,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h
std::list<std::string> ids;
ids.push_back(id);
FileRequest(hash, hash, size, path,
RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
return true;
}
@ -1422,6 +1457,7 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
//rft->flags = fit->second.mFlags;
fit->second.mTransfer->getFileSources(rft->allPeerIds.ids);
fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ;
saveData.push_back(rft);
}
@ -1461,17 +1497,27 @@ bool ftController::loadList(std::list<RsItem *> load)
}
else if (NULL != (rsft = dynamic_cast<RsFileTransfer *>(*it)))
{
// csoler: I'm suppressing this lock since there is a double lock below
// in FileRequest, line 382.
// RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
//
/* This will get stored on a waiting list - until the
* config files are fully loaded
*/
FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize,
rsft->file.path, 0, rsft->allPeerIds.ids);
FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, rsft->file.path, 0, rsft->allPeerIds.ids);
{
RsStackMutex mtx(ctrlMutex) ;
std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
{
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
std::cerr << "Storing the map in a wait list." << std::endl ;
mPendingChunkMaps[rsft->file.hash] = rsft ;
continue ; // i.e. don't delete the item!
}
else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
}
}
/* cleanup */

2
libretroshare/src/ft/ftcontroller.h
View File

@ -189,6 +189,7 @@ bool handleAPendingRequest();
bool setPeerState(ftTransferModule *tm, std::string id,
uint32_t maxrate, bool online);
time_t last_save_time ;
/* pointers to other components */
ftSearch *mSearch;
@ -224,6 +225,7 @@ bool setPeerState(ftTransferModule *tm, std::string id,
bool mFtActive;
bool mFtPendingDone;
std::list<ftPendingRequest> mPendingRequests;
std::map<std::string,RsFileTransfer*> mPendingChunkMaps ;
/* share incoming directory */
bool mShareDownloadDir;

15
libretroshare/src/ft/ftfilecreator.cc
View File

@ -403,4 +403,19 @@ bool ftFileCreator::locked_printChunkMap()
return true;
}
void ftFileCreator::loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ;
}
void ftFileCreator::storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
FileChunksInfo::ChunkStrategy strat ;
chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ;
strategy = (uint32_t)strat ;
}

3
libretroshare/src/ft/ftfilecreator.h
View File

@ -59,6 +59,9 @@ virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size);
bool addFileData(uint64_t offset, uint32_t chunk_size, void *data);
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ;
void storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ;
protected:
virtual int initializeFileAttrs();

25
libretroshare/src/serialiser/rsconfigitems.cc
View File

@ -31,8 +31,6 @@
#define RSSERIAL_DEBUG 1
***/
#define RSSERIAL_DEBUG 1
#include <iostream>
/*************************************************************************/
@ -198,6 +196,11 @@ uint32_t RsFileConfigSerialiser::sizeTransfer(RsFileTransfer *item)
s += 4; /* trate */
s += 4; /* lrate */
s += 4; /* ltransfer */
s += 4; // chunk_size
s += 4; // chunk_number
s += 4; // chunk_strategy
s += 4; // chunk map size
s += 4*item->chunk_map.size(); // chunk_map
return s;
}
@ -240,6 +243,14 @@ bool RsFileConfigSerialiser::serialiseTransfer(RsFileTransfer *item, void *d
ok &= setRawUInt32(data, tlvsize, &offset, item->lrate);
ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_size);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map.size());
for(uint32_t i=0;i<item->chunk_map.size();++i)
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map[i]);
if (offset != tlvsize)
{
ok = false;
@ -299,6 +310,16 @@ RsFileTransfer *RsFileConfigSerialiser::deserialiseTransfer(void *data, uint32_t
ok &= getRawUInt32(data, rssize, &offset, &(item->lrate));
ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_size));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy));
uint32_t map_size = 0 ;
ok &= getRawUInt32(data, rssize, &offset, &map_size);
item->chunk_map.resize(map_size) ;
for(uint32_t i=0;i<map_size;++i)
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_map[i]));
if (offset != rssize)
{
/* error */

22
libretroshare/src/serialiser/rsconfigitems.h
View File

@ -27,6 +27,7 @@
*/
#include <map>
#include <vector>
#include "serialiser/rsserial.h"
#include "serialiser/rstlvbase.h"
@ -179,14 +180,13 @@ virtual RsItem * deserialise(void *data, uint32_t *size);
class RsFileTransfer: public RsItem
{
public:
RsFileTransfer()
:RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG,
RS_PKT_TYPE_FILE_CONFIG,
RS_PKT_SUBTYPE_FILE_TRANSFER)
{ return; }
virtual ~RsFileTransfer();
virtual void clear();
std::ostream &print(std::ostream &out, uint16_t indent = 0);
RsFileTransfer() :RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, RS_PKT_TYPE_FILE_CONFIG, RS_PKT_SUBTYPE_FILE_TRANSFER)
{
return;
}
virtual ~RsFileTransfer();
virtual void clear();
std::ostream &print(std::ostream &out, uint16_t indent = 0);
RsTlvFileItem file;
RsTlvPeerIdSet allPeerIds;
@ -200,10 +200,14 @@ std::ostream &print(std::ostream &out, uint16_t indent = 0);
uint32_t crate;
uint32_t trate;
uint32_t lrate;
uint32_t ltransfer;
// chunk information
uint32_t chunk_size ; // common size of chunks
uint32_t chunk_number ; // total number of chunks (this is not redondant, cause chunks are compressed)
uint32_t chunk_strategy ; // strategy flags for chunks
std::vector<uint32_t> chunk_map ; // chunk availability (bitwise)
};
/**************************************************************************/