mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-05-17 05:22:31 -04:00
First implementation of per-chunk CRC32 check. This is triggered
- by the right-click+Force Check on files.
- when a global hash on a downloaded file does not match the announced hash.
When a CRC map check is ordered, the CRC map is requested to one of the sources for the current file download.
When received, all downloaded chunks are checked w.r.t the reference CRC and marked as not done if the CRCs do not match.
The exchange of CRC32 map and requests has been tested, as well as CRC map checking during download (force check).
To be implemented soon:
- caching of CRC32 maps (although these are fast to compute)
- CRC32 map packets for normal downloads. For now these work only for turtle tunnels.
- handling of errors if the CRC never comes. For now, the download will stay stuck in "Checking..." mode.
So, don't play too much with the force check feature for now...
git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@3310 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler
parent
db034b026f
commit
8bfc74485a
27 changed files with 1012 additions and 118 deletions
|
|
@ -419,17 +419,28 @@ void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
|
|||
#endif
|
||||
}
|
||||
|
||||
uint32_t ChunkMap::getNumberOfChunks(uint64_t size)
|
||||
{
|
||||
uint64_t n = size/(uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE ;
|
||||
|
||||
if(size % (uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE != 0)
|
||||
++n ;
|
||||
|
||||
uint32_t value = n & 0xffffffffull ;
|
||||
|
||||
if((uint64_t)value != n)
|
||||
{
|
||||
std::cerr << "ERROR: number of chunks is a totally absurd value. File size is " << size << ", chunks are " << n << "!!" << std::endl ;
|
||||
return 0 ;
|
||||
}
|
||||
return value ;
|
||||
}
|
||||
|
||||
void ChunkMap::buildPlainMap(uint64_t size, CompressedChunkMap& map)
|
||||
{
|
||||
uint32_t chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE) ; // 1MB chunks
|
||||
uint64_t n = size/(uint64_t)chunk_size ;
|
||||
|
||||
if(size % (uint64_t)chunk_size != 0)
|
||||
++n ;
|
||||
uint32_t n = getNumberOfChunks(size) ;
|
||||
|
||||
map = CompressedChunkMap(n,~uint32_t(0)) ;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -144,6 +144,9 @@ class ChunkMap
|
|||
//
|
||||
static void buildPlainMap(uint64_t size,CompressedChunkMap& map) ;
|
||||
|
||||
/// Computes the number of chunks for the given file size.
|
||||
static uint32_t getNumberOfChunks(uint64_t size) ;
|
||||
|
||||
/// This function is used by the parent ftFileProvider to know whether the chunk can be sent or not.
|
||||
bool isChunkAvailable(uint64_t offset, uint32_t chunk_size) const ;
|
||||
|
||||
|
|
@ -162,6 +165,7 @@ class ChunkMap
|
|||
bool isComplete() const { return _file_is_complete ; }
|
||||
|
||||
void getChunksInfo(FileChunksInfo& info) const ;
|
||||
|
||||
protected:
|
||||
/// handles what size the last chunk has.
|
||||
uint32_t sizeOfChunk(uint32_t chunk_number) const ;
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@
|
|||
* #define CONTROL_DEBUG 1
|
||||
* #define DEBUG_DWLQUEUE 1
|
||||
*****/
|
||||
//#define CONTROL_DEBUG 1
|
||||
|
||||
static const uint32_t SAVE_TRANSFERS_DELAY = 61 ; // save transfer progress every 61 seconds.
|
||||
static const uint32_t INACTIVE_CHUNKS_CHECK_DELAY = 60 ; // time after which an inactive chunk is released
|
||||
|
|
@ -980,10 +981,12 @@ bool ftController::alreadyHaveFile(const std::string& hash)
|
|||
return false ;
|
||||
}
|
||||
|
||||
bool ftController::FileRequest(std::string fname, std::string hash,
|
||||
uint64_t size, std::string dest, uint32_t flags,
|
||||
std::list<std::string> &srcIds)
|
||||
bool ftController::FileRequest(const std::string& fname, const std::string& hash,
|
||||
uint64_t size, const std::string& dest, uint32_t flags,
|
||||
const std::list<std::string> &_srcIds)
|
||||
{
|
||||
std::list<std::string> srcIds(_srcIds) ;
|
||||
|
||||
/* check if we have the file */
|
||||
|
||||
if(alreadyHaveFile(hash))
|
||||
|
|
@ -1047,8 +1050,8 @@ bool ftController::FileRequest(std::string fname, std::string hash,
|
|||
}
|
||||
|
||||
FileInfo info;
|
||||
std::list<std::string>::iterator it;
|
||||
std::list<TransferInfo>::iterator pit;
|
||||
std::list<std::string>::const_iterator it;
|
||||
std::list<TransferInfo>::const_iterator pit;
|
||||
|
||||
#ifdef CONTROL_DEBUG
|
||||
std::cerr << "ftController::FileRequest(" << fname << ",";
|
||||
|
|
@ -1077,7 +1080,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
|
|||
{
|
||||
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
|
||||
|
||||
std::map<std::string, ftFileControl*>::iterator dit = mDownloads.find(hash);
|
||||
std::map<std::string, ftFileControl*>::const_iterator dit = mDownloads.find(hash);
|
||||
|
||||
if (dit != mDownloads.end())
|
||||
{
|
||||
|
|
@ -1312,7 +1315,7 @@ bool ftController::setChunkStrategy(const std::string& hash,FileChunksInfo::Chun
|
|||
return true ;
|
||||
}
|
||||
|
||||
bool ftController::FileCancel(std::string hash)
|
||||
bool ftController::FileCancel(const std::string& hash)
|
||||
{
|
||||
rsTurtle->stopMonitoringFileTunnels(hash) ;
|
||||
|
||||
|
|
@ -1418,12 +1421,20 @@ bool ftController::FileControl(const std::string& hash, uint32_t flags)
|
|||
{
|
||||
case RS_FILE_CTRL_PAUSE:
|
||||
mit->second->mState = ftFileControl::PAUSED ;
|
||||
std::cerr << "setting state to " << ftFileControl::PAUSED << std::endl ;
|
||||
std::cerr << "setting state to " << ftFileControl::PAUSED << std::endl ;
|
||||
break;
|
||||
|
||||
case RS_FILE_CTRL_START:
|
||||
mit->second->mState = ftFileControl::DOWNLOADING ;
|
||||
std::cerr << "setting state to " << ftFileControl::DOWNLOADING << std::endl ;
|
||||
std::cerr << "setting state to " << ftFileControl::DOWNLOADING << std::endl ;
|
||||
break;
|
||||
|
||||
case RS_FILE_CTRL_FORCE_CHECK:
|
||||
// mit->second->mState = ftFileControl::CHECKING_HASH ;
|
||||
mit->second->mTransfer->forceCheck();
|
||||
std::cerr << "setting state to " << ftFileControl::CHECKING_HASH << std::endl ;
|
||||
break;
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,9 +99,9 @@ class ftFileControl
|
|||
class ftPendingRequest
|
||||
{
|
||||
public:
|
||||
ftPendingRequest(std::string fname, std::string hash,
|
||||
uint64_t size, std::string dest, uint32_t flags,
|
||||
std::list<std::string> &srcIds)
|
||||
ftPendingRequest(const std::string& fname, const std::string& hash,
|
||||
uint64_t size, const std::string& dest, uint32_t flags,
|
||||
const std::list<std::string> &srcIds)
|
||||
: mName(fname), mHash(hash), mSize(size),
|
||||
mDest(dest), mFlags(flags),mSrcIds(srcIds) { return; }
|
||||
|
||||
|
|
@ -137,9 +137,9 @@ class ftController: public CacheTransfer, public RsThread, public pqiMonitor, pu
|
|||
/********************** Controller Access **********************/
|
||||
/***************************************************************/
|
||||
|
||||
bool FileRequest(std::string fname, std::string hash,
|
||||
uint64_t size, std::string dest, uint32_t flags,
|
||||
std::list<std::string> &sourceIds);
|
||||
bool FileRequest(const std::string& fname, const std::string& hash,
|
||||
uint64_t size, const std::string& dest, uint32_t flags,
|
||||
const std::list<std::string> &sourceIds);
|
||||
|
||||
/// Do we already have this file, either in download or in file lists ?
|
||||
bool alreadyHaveFile(const std::string& hash) ;
|
||||
|
|
@ -150,7 +150,7 @@ class ftController: public CacheTransfer, public RsThread, public pqiMonitor, pu
|
|||
uint32_t freeDiskSpaceLimit() const ;
|
||||
void setFreeDiskSpaceLimit(uint32_t size_in_mb) ;
|
||||
|
||||
bool FileCancel(std::string hash);
|
||||
bool FileCancel(const std::string& hash);
|
||||
bool FileControl(const std::string& hash, uint32_t flags);
|
||||
bool FileClearCompleted();
|
||||
bool FlagFileComplete(std::string hash);
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@
|
|||
/*************** SEND INTERFACE *******************/
|
||||
|
||||
class CompressedChunkMap ;
|
||||
class CRC32Map ;
|
||||
|
||||
class ftDataSend
|
||||
{
|
||||
|
|
@ -57,6 +58,10 @@ class ftDataSend
|
|||
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) = 0;
|
||||
/// Send a chunk map
|
||||
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) = 0;
|
||||
/// Send a request for a chunk crc map
|
||||
virtual bool sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0;
|
||||
/// Send a chunk crc map
|
||||
virtual bool sendCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) = 0;
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
|
|||
const uint32_t FT_DATA = 0x0001; // data cuhnk to be stored
|
||||
const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
|
||||
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
|
||||
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map reuqest to be treated by server
|
||||
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map request to be treated by server
|
||||
const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server
|
||||
|
||||
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
|
||||
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
|
||||
|
|
@ -250,7 +251,19 @@ bool ftDataMultiplex::recvChunkMapRequest(const std::string& peerId, const std::
|
|||
return true;
|
||||
}
|
||||
|
||||
bool ftDataMultiplex::recvCRCMapRequest(const std::string& peerId, const std::string& hash/*,const CompressedChunkMap& map*/)
|
||||
{
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
/* Store in Queue */
|
||||
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
|
||||
|
||||
mRequestQueue.push_back(ftRequest(FT_CRC32MAP_REQ,peerId,hash,0,0,0,NULL));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/*********** BACKGROUND THREAD OPERATIONS ***********/
|
||||
bool ftDataMultiplex::workQueued()
|
||||
|
|
@ -326,6 +339,14 @@ bool ftDataMultiplex::doWork()
|
|||
handleRecvServerChunkMapRequest(req.mPeerId,req.mHash) ;
|
||||
break ;
|
||||
|
||||
case FT_CRC32MAP_REQ:
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CRC32_MAP_REQ";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
handleRecvCRC32MapRequest(req.mPeerId,req.mHash,CompressedChunkMap()) ;
|
||||
break ;
|
||||
|
||||
default:
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
|
||||
|
|
@ -362,6 +383,27 @@ bool ftDataMultiplex::doWork()
|
|||
return true;
|
||||
}
|
||||
|
||||
bool ftDataMultiplex::recvCRCMap(const std::string& peerId, const std::string& hash,const CRC32Map& crc_map)
|
||||
{
|
||||
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
|
||||
|
||||
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
|
||||
|
||||
if(it == mClients.end())
|
||||
{
|
||||
std::cerr << "ftDataMultiplex::recvCRCMap() ERROR: No matching Client for CRC32map. This is an error. " << hash << " !" << std::endl;
|
||||
/* error */
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::recvCRCMap() Passing crc map of file " << hash << ", to FT Module" << std::endl;
|
||||
#endif
|
||||
|
||||
(it->second).mModule->addCRC32Map(crc_map);
|
||||
return true ;
|
||||
}
|
||||
|
||||
// A chunk map has arrived. It can be two different situations:
|
||||
// - an uploader has sent his chunk map, so we need to store it in the corresponding ftFileProvider
|
||||
// - a source for a download has sent his chunk map, so we need to send it to the corresponding ftFileCreator.
|
||||
|
|
@ -412,6 +454,53 @@ bool ftDataMultiplex::recvChunkMap(const std::string& peerId, const std::string&
|
|||
return false;
|
||||
}
|
||||
|
||||
bool ftDataMultiplex::handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash,const CompressedChunkMap&)
|
||||
{
|
||||
std::map<std::string, ftFileProvider *>::iterator it ;
|
||||
bool found = true ;
|
||||
|
||||
{
|
||||
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
|
||||
|
||||
it = mServers.find(hash) ;
|
||||
|
||||
if(it == mServers.end())
|
||||
found = false ;
|
||||
}
|
||||
|
||||
if(!found)
|
||||
{
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
if(!handleSearchRequest(peerId,hash))
|
||||
return false ;
|
||||
|
||||
#ifdef MPLEX_DEBUG
|
||||
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
CRC32Map cmap ;
|
||||
{
|
||||
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
|
||||
|
||||
it = mServers.find(hash) ;
|
||||
|
||||
if(it == mServers.end()) // handleSearchRequest should have filled mServers[hash], but we have been off-mutex since,
|
||||
return false ; // so it's safer to check again.
|
||||
else if(!it->second->getCRC32Map(cmap))
|
||||
return false ;
|
||||
}
|
||||
|
||||
std::cerr << "File CRC32 map was successfully computed. Sending it." << std::endl ;
|
||||
|
||||
mDataSend->sendCRC32Map(peerId,hash,cmap);
|
||||
return true ;
|
||||
}
|
||||
|
||||
bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash)
|
||||
{
|
||||
CompressedChunkMap cmap ;
|
||||
|
|
@ -639,7 +728,10 @@ bool ftDataMultiplex::sendChunkMapRequest(const std::string& peer_id,const std::
|
|||
{
|
||||
return mDataSend->sendChunkMapRequest(peer_id,hash);
|
||||
}
|
||||
|
||||
bool ftDataMultiplex::sendCRCMapRequest(const std::string& peer_id,const std::string& hash,const CompressedChunkMap&)
|
||||
{
|
||||
return mDataSend->sendCRC32MapRequest(peer_id,hash);
|
||||
}
|
||||
void ftDataMultiplex::deleteUnusedServers()
|
||||
{
|
||||
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
|
||||
|
|
|
|||
|
|
@ -110,19 +110,25 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
|
|||
/* Server/client Send */
|
||||
bool sendChunkMapRequest(const std::string& peerId, const std::string& hash) ;
|
||||
|
||||
/* Client Send */
|
||||
bool sendCRCMapRequest(const std::string& peerId, const std::string& hash,const CompressedChunkMap& chnks) ;
|
||||
|
||||
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
|
||||
|
||||
/* Client Recv */
|
||||
virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
|
||||
/* Server Recv */
|
||||
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
|
||||
|
||||
/// Receive a request for a chunk map
|
||||
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ;
|
||||
/// Receive a chunk map
|
||||
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
|
||||
|
||||
/* Server Recv */
|
||||
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
|
||||
|
||||
/// Receive a CRC map
|
||||
virtual bool recvCRCMap(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) ;
|
||||
/// Receive a CRC map request
|
||||
virtual bool recvCRCMapRequest(const std::string& peer_id,const std::string& hash) ;
|
||||
|
||||
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
|
||||
// map is too old. This supposes that the caller will ask again in a few seconds.
|
||||
//
|
||||
|
|
@ -142,6 +148,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
|
|||
bool handleSearchRequest(const std::string& peerId, const std::string& hash);
|
||||
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
|
||||
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
|
||||
bool handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash, const CompressedChunkMap& map) ;
|
||||
|
||||
/* We end up doing the actual server job here */
|
||||
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
|
||||
|
|
|
|||
|
|
@ -472,13 +472,55 @@ bool ftFileCreator::hashReceivedData(std::string& hash)
|
|||
// long time. Therefore, we must pay attention not to call this function
|
||||
// at a time file_name nor hash can be modified, which is easy.
|
||||
//
|
||||
if(!finished())
|
||||
return false ;
|
||||
|
||||
return RsDirUtil::hashFile(file_name,hash) ;
|
||||
}
|
||||
|
||||
#include <stdexcept>
|
||||
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref)
|
||||
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks)
|
||||
{
|
||||
// still to be implemented.
|
||||
throw std::runtime_error(std::string("Unimplemented function ") + __PRETTY_FUNCTION__) ;
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
CompressedChunkMap map ;
|
||||
chunkMap.getAvailabilityMap(map) ;
|
||||
uint32_t nb_chunks = ref.size() ;
|
||||
static const uint32_t chunk_size = ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE ;
|
||||
|
||||
std::cerr << "ftFileCreator::crossCheckChunkMap(): comparing chunks..." << std::endl;
|
||||
|
||||
if(!locked_initializeFileAttrs() )
|
||||
return false ;
|
||||
|
||||
unsigned char *buff = new unsigned char[chunk_size] ;
|
||||
incomplete_chunks = 0 ;
|
||||
bad_chunks = 0 ;
|
||||
uint32_t len = 0 ;
|
||||
|
||||
for(uint32_t i=0;i<nb_chunks;++i)
|
||||
if(map[i])
|
||||
if(fseek(fd,(uint64_t)i * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
|
||||
{
|
||||
if( RsDirUtil::rs_CRC32(buff,len) != ref[i])
|
||||
{
|
||||
++bad_chunks ;
|
||||
++incomplete_chunks ;
|
||||
map.reset(i) ;
|
||||
}
|
||||
}
|
||||
else
|
||||
return false ;
|
||||
else
|
||||
++incomplete_chunks ;
|
||||
|
||||
delete[] buff ;
|
||||
|
||||
if(bad_chunks > 0)
|
||||
chunkMap.setAvailabilityMap(map) ;
|
||||
}
|
||||
closeFile() ;
|
||||
return true ;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -64,13 +64,11 @@ class ftFileCreator: public ftFileProvider
|
|||
|
||||
bool hashReceivedData(std::string& hash) ;
|
||||
|
||||
// Computes a CRC32 map of all chunks, for comparison with reference, and re-starting invalid chunks.
|
||||
//
|
||||
bool CRC32ReceivedData(CRC32Map& crc_map) ;
|
||||
|
||||
// Checks the CRC32 of all chunks against the given CRC32 map. Re-flag the bad chunks as being void.
|
||||
// bad_chunks: count of achieved chunks that don't match the CRC
|
||||
// incomplete_chunks: count of any bad or not yet downloaded chunk
|
||||
//
|
||||
bool crossCheckChunkMap(const CRC32Map& crc_map) ;
|
||||
bool crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks) ;
|
||||
/*
|
||||
* creation functions for FileCreator
|
||||
*/
|
||||
|
|
@ -107,6 +105,12 @@ class ftFileCreator: public ftFileProvider
|
|||
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
|
||||
void setAvailabilityMap(const CompressedChunkMap& cmap) ;
|
||||
|
||||
// Provides a complete per-chunk CRC32 map to client who want to check their data.
|
||||
// This is overloads ftFileProvider, but returns false, because we can't ensure that unchecked chunks
|
||||
// will provide a CRC32 that is faithful to the original hash.
|
||||
//
|
||||
virtual bool getCRC32Map(CRC32Map& crc_map) { return false ; }
|
||||
|
||||
// This is called when receiving the availability map from a source peer, for the file being handled.
|
||||
//
|
||||
void setSourceMap(const std::string& peer_id,const CompressedChunkMap& map) ;
|
||||
|
|
|
|||
|
|
@ -292,3 +292,16 @@ int ftFileProvider::initializeFileAttrs()
|
|||
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool ftFileProvider::getCRC32Map(CRC32Map& crc_map)
|
||||
{
|
||||
if(!initializeFileAttrs())
|
||||
{
|
||||
std::cerr << "ftFileProvider::getCRC32Map(...): ERROR: can't initialize file !" << std::endl ;
|
||||
return false ;
|
||||
}
|
||||
|
||||
std::cerr << "ftFileProvider::getClientMap(): computing CRC32 map for file " << file_name << " (" << hash << ")" << std::endl ;
|
||||
return RsDirUtil::crc32File(fd,mSize,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,crc_map) ;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,12 @@ class ftFileProvider
|
|||
//
|
||||
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
|
||||
|
||||
// Provides a complete per-chunk CRC32 map to client who want to check their data.
|
||||
// This is derived in ftFileCreator, but returns false, because we can't ensure that unchecked chunks
|
||||
// will provide a CRC32 that is faithful to the original hash.
|
||||
//
|
||||
virtual bool getCRC32Map(CRC32Map& crc_map) ;
|
||||
|
||||
// a ftFileProvider feeds a distant peer. To display what the peers already has, we need to store/read this info.
|
||||
void getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old) ;
|
||||
void setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap) ;
|
||||
|
|
|
|||
|
|
@ -281,7 +281,7 @@ bool ftServer::checkHash(const std::string& hash,std::string& error_string)
|
|||
return true ;
|
||||
}
|
||||
|
||||
bool ftServer::FileRequest(std::string fname, std::string hash, uint64_t size, std::string dest, uint32_t flags, std::list<std::string> srcIds)
|
||||
bool ftServer::FileRequest(const std::string& fname, const std::string& hash, uint64_t size, const std::string& dest, uint32_t flags, const std::list<std::string>& srcIds)
|
||||
{
|
||||
std::string error_string ;
|
||||
|
||||
|
|
@ -319,8 +319,7 @@ FileChunksInfo::ChunkStrategy ftServer::defaultChunkStrategy()
|
|||
{
|
||||
return mFtController->defaultChunkStrategy() ;
|
||||
}
|
||||
|
||||
bool ftServer::FileCancel(std::string hash)
|
||||
bool ftServer::FileCancel(const std::string& hash)
|
||||
{
|
||||
// Remove from both queue and ftController, by default.
|
||||
//
|
||||
|
|
@ -329,7 +328,7 @@ bool ftServer::FileCancel(std::string hash)
|
|||
return true ;
|
||||
}
|
||||
|
||||
bool ftServer::FileControl(std::string hash, uint32_t flags)
|
||||
bool ftServer::FileControl(const std::string& hash, uint32_t flags)
|
||||
{
|
||||
return mFtController->FileControl(hash, flags);
|
||||
}
|
||||
|
|
@ -743,6 +742,8 @@ bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string&
|
|||
{
|
||||
if(mTurtleRouter->isTurtlePeer(peerId))
|
||||
mTurtleRouter->sendChunkMapRequest(peerId,hash) ;
|
||||
else
|
||||
std::cerr << "ftServer: Warning: not sending chunk map request to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
|
||||
|
||||
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
|
||||
// direct friends, so I'll see later whether I code it or not.
|
||||
|
|
@ -753,12 +754,36 @@ bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,co
|
|||
{
|
||||
if(mTurtleRouter->isTurtlePeer(peerId))
|
||||
mTurtleRouter->sendChunkMap(peerId,hash,map) ;
|
||||
else
|
||||
std::cerr << "ftServer: Warning: not sending chunk map to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
|
||||
|
||||
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
|
||||
// direct friends, so I'll see later whether I code it or not.
|
||||
return true ;
|
||||
}
|
||||
bool ftServer::sendCRC32MapRequest(const std::string& peerId,const std::string& hash)
|
||||
{
|
||||
if(mTurtleRouter->isTurtlePeer(peerId))
|
||||
mTurtleRouter->sendCRC32MapRequest(peerId,hash) ;
|
||||
else
|
||||
std::cerr << "ftServer: Warning: not sending CRC map request to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
|
||||
|
||||
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
|
||||
// direct friends, so I'll see later whether I code it or not.
|
||||
return true ;
|
||||
}
|
||||
|
||||
bool ftServer::sendCRC32Map(const std::string& peerId,const std::string& hash,const CRC32Map& map)
|
||||
{
|
||||
if(mTurtleRouter->isTurtlePeer(peerId))
|
||||
mTurtleRouter->sendCRC32Map(peerId,hash,map) ;
|
||||
else
|
||||
std::cerr << "ftServer: Warning: not sending CRC map to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
|
||||
|
||||
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
|
||||
// direct friends, so I'll see later whether I code it or not.
|
||||
return true ;
|
||||
}
|
||||
|
||||
//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */
|
||||
//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */
|
||||
|
|
|
|||
|
|
@ -120,10 +120,9 @@ ftController *getController() const { return mFtController ; }
|
|||
/***
|
||||
* Control of Downloads
|
||||
***/
|
||||
virtual bool FileRequest(std::string fname, std::string hash, uint64_t size,
|
||||
std::string dest, uint32_t flags, std::list<std::string> srcIds);
|
||||
virtual bool FileCancel(std::string hash);
|
||||
virtual bool FileControl(std::string hash, uint32_t flags);
|
||||
virtual bool FileRequest(const std::string& fname, const std::string& hash, uint64_t size, const std::string& dest, uint32_t flags, const std::list<std::string>& srcIds);
|
||||
virtual bool FileCancel(const std::string& hash);
|
||||
virtual bool FileControl(const std::string& hash, uint32_t flags);
|
||||
virtual bool FileClearCompleted();
|
||||
virtual bool setChunkStrategy(const std::string& hash,FileChunksInfo::ChunkStrategy s) ;
|
||||
virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ;
|
||||
|
|
@ -213,6 +212,9 @@ virtual bool sendData(const std::string& peerId, const std::string& hash, uint64
|
|||
virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
|
||||
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) ;
|
||||
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) ;
|
||||
virtual bool sendCRC32MapRequest(const std::string&, const std::string&) ;
|
||||
virtual bool sendCRC32Map(const std::string&, const std::string&, const CRC32Map&) ;
|
||||
|
||||
|
||||
|
||||
/*************** Internal Transfer Fns *************************/
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
*****/
|
||||
|
||||
//#define FT_DEBUG 1
|
||||
#include <rsiface/rsturtle.h>
|
||||
#include "fttransfermodule.h"
|
||||
|
||||
/*************************************************************************
|
||||
|
|
@ -52,6 +53,7 @@ const uint32_t FT_TM_MAX_RESETS = 5;
|
|||
const uint32_t FT_TM_MINIMUM_CHUNK = 128; /* ie 1/8Kb / sec */
|
||||
const uint32_t FT_TM_RESTART_DOWNLOAD = 20; /* 20 seconds */
|
||||
const uint32_t FT_TM_DOWNLOAD_TIMEOUT = 10; /* 10 seconds */
|
||||
const uint32_t FT_TM_CRC_MAP_MAX_WAIT_PER_GIG = 10; /* 10 seconds */
|
||||
|
||||
const double FT_TM_MAX_INCREASE = 1.00;
|
||||
const double FT_TM_MIN_INCREASE = -0.10;
|
||||
|
|
@ -59,6 +61,11 @@ const int32_t FT_TM_FAST_RTT = 1.0;
|
|||
const int32_t FT_TM_STD_RTT = 5.0;
|
||||
const int32_t FT_TM_SLOW_RTT = 9.0;
|
||||
|
||||
const uint32_t FT_TM_CRC_MAP_STATE_DONT_HAVE = 0 ;
|
||||
const uint32_t FT_TM_CRC_MAP_STATE_NOCHECK = 1 ;
|
||||
const uint32_t FT_TM_CRC_MAP_STATE_ASKED = 2 ;
|
||||
const uint32_t FT_TM_CRC_MAP_STATE_HAVE = 3 ;
|
||||
|
||||
#define FT_TM_FLAG_DOWNLOADING 0
|
||||
#define FT_TM_FLAG_CANCELED 1
|
||||
#define FT_TM_FLAG_COMPLETE 2
|
||||
|
|
@ -505,7 +512,7 @@ bool ftTransferModule::isCheckingHash()
|
|||
#ifdef FT_DEBUG
|
||||
std::cerr << "isCheckingHash(): mFlag=" << mFlag << std::endl;
|
||||
#endif
|
||||
return mFlag == FT_TM_FLAG_CHECKING ;
|
||||
return mFlag == FT_TM_FLAG_CHECKING || mFlag == FT_TM_FLAG_CHUNK_CRC;
|
||||
}
|
||||
|
||||
class HashThread: public RsThread
|
||||
|
|
@ -580,9 +587,11 @@ bool ftTransferModule::checkFile()
|
|||
}
|
||||
|
||||
delete _hash_thread ;
|
||||
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map. But for now, cancel file transfer.
|
||||
mFileStatus.stat = ftFileStatus::PQIFILE_FAIL_CANCEL ;
|
||||
|
||||
}
|
||||
forceCheck() ;
|
||||
return true ;
|
||||
|
||||
cancelFileTransferUpward() ;
|
||||
std::cerr << "(EE) ftTransferModule::checkFile(): File verification failed for hash " << mHash << "! Asking for CRC map. mFlag=4. For now: cancelling file transfer." << std::endl ;
|
||||
//askForCRCMap() ;
|
||||
|
|
@ -590,28 +599,172 @@ bool ftTransferModule::checkFile()
|
|||
return false ;
|
||||
}
|
||||
|
||||
void ftTransferModule::forceCheck()
|
||||
{
|
||||
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::forceCheck(): setting flags to force check." << std::endl ;
|
||||
#endif
|
||||
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map. But for now, cancel file transfer.
|
||||
|
||||
// setup flags for CRC state machine to work properly
|
||||
_crcmap_state = FT_TM_CRC_MAP_STATE_DONT_HAVE ;
|
||||
_crcmap_last_asked_time = 0 ;
|
||||
_crcmap_last_source_id = -1 ;
|
||||
}
|
||||
|
||||
bool ftTransferModule::checkCRC()
|
||||
{
|
||||
// We have a finite state machine here.
|
||||
//
|
||||
// The states are, for each chunk, and what should be done:
|
||||
// DONT_HAVE
|
||||
// -> ask for the chunk CRC
|
||||
// ASKED
|
||||
// -> do nothing
|
||||
// RECEIVED
|
||||
// -> check the chunk
|
||||
// CHECKED
|
||||
// -> do nothing
|
||||
//
|
||||
// CRCs are asked by group of CRC_REQUEST_MAX_SIZE chunks at a time. The response may contain a different
|
||||
// number of CRCs, depending on the availability.
|
||||
//
|
||||
// Server side:
|
||||
// - Only complete files can compute CRCs, as the CRCs of downloaded chunks in an unchecked file are un-verified.
|
||||
// - CRCs of files are cached, so that they don't get computed twice.
|
||||
//
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ;
|
||||
#endif
|
||||
|
||||
if(_crc32map.empty())
|
||||
return false ;
|
||||
// Loop over chunks. Collect the ones to ask for, and hash the ones received.
|
||||
// If we have
|
||||
//
|
||||
time_t now = time(NULL) ;
|
||||
|
||||
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
|
||||
|
||||
switch(_crcmap_state)
|
||||
{
|
||||
case FT_TM_CRC_MAP_STATE_NOCHECK:
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ;
|
||||
std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ;
|
||||
#endif
|
||||
break ;
|
||||
|
||||
case FT_TM_CRC_MAP_STATE_ASKED:
|
||||
std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: last time is " << _crcmap_last_asked_time << std::endl ;
|
||||
std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: now is " << now << std::endl ;
|
||||
std::cerr << "Limit is " << (uint64_t)_crcmap_last_asked_time + (uint64_t)(FT_TM_CRC_MAP_MAX_WAIT_PER_GIG * (1+mSize/float(1024ull*1024ull*1024ull))) << std::endl ;
|
||||
if( (uint64_t)_crcmap_last_asked_time + (uint64_t)(FT_TM_CRC_MAP_MAX_WAIT_PER_GIG * (1+mSize/float(1024ull*1024ull*1024ull))) > (uint64_t)now)
|
||||
{
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ;
|
||||
#endif
|
||||
break ;
|
||||
}
|
||||
#ifdef FT_DEBUG
|
||||
else
|
||||
std::cerr << "ftTransferModule::checkCRC(): state is ASKED, but time is too long. Asking again." << std::endl ;
|
||||
#endif
|
||||
|
||||
mFileCreator->crossCheckChunkMap(_crc32map) ;
|
||||
mFlag = FT_TM_FLAG_DOWNLOADING ;
|
||||
case FT_TM_CRC_MAP_STATE_DONT_HAVE:
|
||||
{
|
||||
// Ask the ones we should ask for. We use a very coarse strategy now: we
|
||||
// send the request to a random source. We'll make this more sensible
|
||||
// later.
|
||||
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): Done. Restarting download." << std::endl ;
|
||||
std::cerr << "ftTransferModule::checkCRC(): state is DONT_HAVE. Selecting a source for asking a CRC map." << std::endl ;
|
||||
#endif
|
||||
// if(_crcmap_last_source_id < 0)
|
||||
// _crcmap_last_source_id=lrand48() ;
|
||||
|
||||
// _crcmap_last_source_id = (_crcmap_last_source_id+1)%mFileSources.size() ;
|
||||
|
||||
int n=0 ;
|
||||
bool found = false ;
|
||||
std::map<std::string,peerInfo>::const_iterator mit ;
|
||||
for(mit = mFileSources.begin();mit != mFileSources.end();++mit)
|
||||
if(rsTurtle->isTurtlePeer(mit->first))
|
||||
{
|
||||
found=true ;
|
||||
break ;
|
||||
}
|
||||
|
||||
if(found)
|
||||
{
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): sending CRC map request to source " << mit->first << std::endl ;
|
||||
#endif
|
||||
_crcmap_last_asked_time = now ;
|
||||
mMultiplexor->sendCRCMapRequest(mit->first,mHash,CompressedChunkMap());
|
||||
|
||||
_crcmap_state = FT_TM_CRC_MAP_STATE_ASKED ;
|
||||
}
|
||||
else
|
||||
std::cerr << "ERROR: No file source to ask a chunkmap to!" << std::endl ;
|
||||
}
|
||||
break ;
|
||||
|
||||
case FT_TM_CRC_MAP_STATE_HAVE:
|
||||
{
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ;
|
||||
#endif
|
||||
// The CRCmap is complete. Let's cross check it with existing chunks in ftFileCreator !
|
||||
//
|
||||
uint32_t bad_chunks ;
|
||||
uint32_t incomplete_chunks ;
|
||||
|
||||
if(!mFileCreator->crossCheckChunkMap(_crcmap,bad_chunks,incomplete_chunks))
|
||||
{
|
||||
std::cerr << "ftTransferModule::checkCRC(): could not check CRC chunks!" << std::endl ;
|
||||
return false;
|
||||
}
|
||||
|
||||
// go back to download stage, if not all chunks are correct.
|
||||
//
|
||||
if(bad_chunks > 0)
|
||||
{
|
||||
mFlag = FT_TM_FLAG_DOWNLOADING ;
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): Done. " << bad_chunks << " bad chunks found. Restarting download for these chunks only." << std::endl ;
|
||||
#endif
|
||||
}
|
||||
else if(incomplete_chunks > 0)
|
||||
{
|
||||
mFlag = FT_TM_FLAG_DOWNLOADING ;
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): Done. all chunks ok. Continuing download for remaining chunks." << std::endl ;
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
mFlag = FT_TM_FLAG_COMPLETE ;
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::checkCRC(): Done. CRC check is ok, file is complete." << std::endl ;
|
||||
#endif
|
||||
}
|
||||
|
||||
_crcmap_state = FT_TM_CRC_MAP_STATE_NOCHECK ;
|
||||
}
|
||||
}
|
||||
|
||||
return true ;
|
||||
}
|
||||
|
||||
void ftTransferModule::addCRC32Map(const CRC32Map& crc_map)
|
||||
{
|
||||
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
|
||||
|
||||
// Note: for now, we only send complete CRC32 maps, so the crc_map is always complete. When we
|
||||
// send partial CRC maps, we will have to check them for completness.
|
||||
//
|
||||
_crcmap_state = FT_TM_CRC_MAP_STATE_HAVE ;
|
||||
_crcmap = crc_map ;
|
||||
}
|
||||
|
||||
void ftTransferModule::adjustSpeed()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -144,6 +144,8 @@ public:
|
|||
bool cancelFileTransferUpward();
|
||||
bool completeFileTransfer();
|
||||
bool isCheckingHash() ;
|
||||
void forceCheck() ;
|
||||
void addCRC32Map(const CRC32Map& map) ;
|
||||
|
||||
//interface to multiplex module
|
||||
bool recvFileData(std::string peerId, uint64_t offset,
|
||||
|
|
@ -189,7 +191,10 @@ private:
|
|||
double desiredRate;
|
||||
double actualRate;
|
||||
|
||||
CRC32Map _crc32map ;
|
||||
CRC32Map _crcmap ;
|
||||
uint32_t _crcmap_state ;
|
||||
time_t _crcmap_last_asked_time ;
|
||||
int _crcmap_last_source_id ;
|
||||
|
||||
ftFileStatus mFileStatus; //used for pause/resume file transfer
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue