Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-25 01:01:40 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added deferred check of chunks during file transfer. Chunk sha1 sums are requested to the sources and checked for downloaded data.

Browse source 
Validated chunks are shared to other peers. Force check is now very simple since it just turns all chunks into "needs checking" mode
and sums are asked to sources. Sources maintain a temporary cache of chunks. Since sums are requested sparsely, this should not 
affect the sources in terms of performance. We can still imagine precomputing and saving sha1 of chunks while hashing them.

For backward compatibility reasons, the following has been setup *temporarily* in this version:
- unvalidated chunks are still considered as already obtained, and are shared and saved
- force check has been disabled
- final file check is maintained
- in case of file fail, the old checking mode will be used.

All changes for next version are kept in the define 'USE_NEW_CHUNK_CHECKING_CODE' that will be made the default in a few weeks.
At start, I expect most chunk to stya yellow during download, until most sources are able to provide chunk hashs.




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@5019 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2012-03-15 19:55:43 +00:00
parent 7ab5b54266
commit 889a2b2433
31 changed files with 1540 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

100
libretroshare/src/ft/ftchunkmap.cc
View file

@ -151,9 +151,91 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
#ifdef DEBUG_FTCHUNK
std::cerr << "*** ChunkMap::dataReceived: Chunk is complete. Removing it." << std::endl ;
#endif
_map[n] = FileChunksInfo::CHUNK_DONE ;
_map[n] = FileChunksInfo::CHUNK_CHECKING ;
_chunks_checking_queue.push_back(n) ;
_slices_to_download.erase(itc) ;
updateTotalDownloaded() ;
}
}
void ChunkMap::updateTotalDownloaded()
{
_total_downloaded = 0 ;
_file_is_complete = true ;
// First, round over chunk map to get the raw info.
//
for(uint32_t i=0;i<_map.size();++i)
switch(_map[i])
{
#ifdef USE_NEW_CHUNK_CHECKING_CODE
case FileChunksInfo::CHUNK_CHECKING: _file_is_complete = false ;
#else
case FileChunksInfo::CHUNK_CHECKING: //_file_is_complete = false ;********WARNING******** Re-enable this when users have massively switched to chunk checking code
#endif
case FileChunksInfo::CHUNK_DONE: _total_downloaded += sizeOfChunk(i) ;
break ;
default:
_file_is_complete = false ;
}
// Then go through active chunks.
//
for(std::map<ChunkNumber,ChunkDownloadInfo>::const_iterator itc(_slices_to_download.begin());itc!=_slices_to_download.end();++itc)
{
if(_map[itc->first] == FileChunksInfo::CHUNK_CHECKING)
_total_downloaded -= sizeOfChunk(itc->first) ;
_total_downloaded += sizeOfChunk(itc->first) - itc->second._remains ;
if(_file_is_complete)
std::cerr << "ChunkMap::updateTotalDownloaded(): ERROR: file still has pending slices but all chunks are marked as DONE !!" << std::endl;
}
}
void ChunkMap::getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_crc_to_ask)
{
chunks_crc_to_ask.clear() ;
for(uint32_t i=0;i<_chunks_checking_queue.size();)
{
std::list<std::string> peers ;
for(std::map<std::string,SourceChunksInfo>::const_iterator it2(_peers_chunks_availability.begin());it2!=_peers_chunks_availability.end();++it2)
if(it2->second.cmap[_chunks_checking_queue[i]])
peers.push_back(it2->first) ;
if(peers.empty()) // no peers => can't ask!
{
++i ;
continue ;
}
chunks_crc_to_ask.push_back(std::pair<uint32_t,std::list<std::string> >(_chunks_checking_queue[i],peers)) ;
// remove that chunk from the queue
_chunks_checking_queue[i] = _chunks_checking_queue.back() ;
_chunks_checking_queue.pop_back() ;
}
}
void ChunkMap::setChunkCheckingResult(uint32_t chunk_number,bool check_succeeded)
{
// Find the chunk is the waiting queue. Remove it, and mark it as done.
//
if(_map[chunk_number] != FileChunksInfo::CHUNK_CHECKING)
{
std::cerr << "(EE) ChunkMap: asked to set checking result of chunk " << chunk_number<< " that is not marked as being checked!!" << std::endl;
return ;
}
if(check_succeeded)
{
_map[chunk_number] = FileChunksInfo::CHUNK_DONE ;
// We also check whether the file is complete or not.
_file_is_complete = true ;
@ -165,6 +247,11 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
break ;
}
}
else
{
_total_downloaded -= sizeOfChunk(chunk_number) ; // restore completion.
_map[chunk_number] = FileChunksInfo::CHUNK_OUTSTANDING ;
}
}
// Warning: a chunk may be empty, but still being downloaded, so asking new slices from it
@ -495,6 +582,17 @@ void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
#endif
}
void ChunkMap::forceCheck()
{
for(uint32_t i=0;i<_map.size();++i)
{
_map[i] = FileChunksInfo::CHUNK_CHECKING ;
_chunks_checking_queue.push_back(i) ;
}
updateTotalDownloaded() ;
}
uint32_t ChunkMap::getNumberOfChunks(uint64_t size)
{
uint64_t n = size/(uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE ;

15
libretroshare/src/ft/ftchunkmap.h
View file

@ -178,8 +178,22 @@ class ChunkMap
/// returns true is the file is complete
bool isComplete() const { return _file_is_complete ; }
/// returns info about currently downloaded chunks
void getChunksInfo(FileChunksInfo& info) const ;
/// input the result of the chunk hash checking
void setChunkCheckingResult(uint32_t chunk_number, bool succeed) ;
/// returns the current list of chunks to ask for a CRC, and a proposed source for each
void getChunksToCheck(std::vector<std::pair<unsigned int, std::list<std::string> > >& chunks_to_ask) ;
/// sets all chunks to checking state
void forceCheck() ;
/// Goes through all structures and computes the actual file completion. The true completion
/// gets lost when force checking the file.
void updateTotalDownloaded() ;
protected:
/// handles what size the last chunk has.
uint32_t sizeOfChunk(uint32_t chunk_number) const ;
@ -199,6 +213,7 @@ class ChunkMap
uint64_t _total_downloaded ; //! completion for the file
bool _file_is_complete ; //! set to true when the file is complete.
bool _assume_availability ; //! true if all sources always have the complete file.
std::vector<uint32_t> _chunks_checking_queue ; //! Queue of downloaded chunks to be checked.
};

8
libretroshare/src/ft/ftdata.h
View file

@ -38,10 +38,12 @@
#include <string>
#include <inttypes.h>
#include <retroshare/rstypes.h>
/*************** SEND INTERFACE *******************/
class CompressedChunkMap ;
class CRC32Map ;
class Sha1CheckSum ;
class ftDataSend
{
@ -66,6 +68,10 @@ class ftDataSend
virtual bool sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk crc map
virtual bool sendCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) = 0;
/// Send a request for a chunk crc map
virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) = 0;
/// Send a chunk crc map
virtual bool sendSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) = 0;
};

264
libretroshare/src/ft/ftdatamultiplex.cc
View file

@ -42,6 +42,9 @@ const uint32_t DMULTIPLEX_MIN = 10; /* 1ms sleep */
const uint32_t DMULTIPLEX_MAX = 1000; /* 1 sec sleep */
const double DMULTIPLEX_RELAX = 0.5; /* ??? */
static const uint32_t MAX_CHECKING_CHUNK_WAIT_DELAY = 120 ; //! TTL for an inactive chunk
const uint32_t MAX_SIMULTANEOUS_CRC_REQUESTS = 20 ;
/******
* #define MPLEX_DEBUG 1
*****/
@ -57,6 +60,7 @@ const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map request to be treated by server
const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server
const uint32_t FT_CLIENT_CHUNK_CRC_REQ = 0x0006; // chunk sha1 crc request to be treated
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
@ -281,7 +285,19 @@ bool ftDataMultiplex::recvCRC32MapRequest(const std::string& peerId, const std::
return true;
}
bool ftDataMultiplex::recvSingleChunkCrcRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_number)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
std::cerr << std::endl;
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(ftRequest(FT_CLIENT_CHUNK_CRC_REQ,peerId,hash,0,0,chunk_number,NULL));
return true;
}
class CRC32Thread: public RsThread
{
public:
@ -387,6 +403,14 @@ bool ftDataMultiplex::doWork()
handleRecvCRC32MapRequest(req.mPeerId,req.mHash) ;
break ;
case FT_CLIENT_CHUNK_CRC_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_CRC_REQ";
std::cerr << std::endl;
#endif
handleRecvChunkCrcRequest(req.mPeerId,req.mHash,req.mChunk) ;
break ;
default:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
@ -465,6 +489,93 @@ bool ftDataMultiplex::doWork()
return true;
}
bool ftDataMultiplex::recvSingleChunkCrc(const std::string& /*peerId*/, const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() Received crc of file " << hash << ", chunk " << chunk_number << ", crc=" << crc.toStdString() << std::endl;
//#endif
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() ERROR: No matching Client for CRC. This is an error. " << hash << " !" << std::endl;
/* error */
return false;
}
// store in the cache as well
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
if(sha1cache._map.size() == 0)
sha1cache._map = Sha1Map(it->second.mCreator->fileSize(),ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE) ;
sha1cache._map.set(chunk_number,crc) ;
// remove this chunk from the request list as well.
std::map<uint32_t,ChunkCheckSumSourceList>::iterator it2(sha1cache._to_ask.find(chunk_number)) ;
if(it2 != sha1cache._to_ask.end())
sha1cache._to_ask.erase(it2) ;
sha1cache._received.push_back(chunk_number) ;
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() stored in cache. " << std::endl;
//#endif
return true ;
}
bool ftDataMultiplex::dispatchReceivedChunkCheckSum()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
uint32_t MAX_CHECKSUM_CHECK_PER_FILE = 25 ;
for(std::map<std::string,Sha1CacheEntry>::iterator it(_cached_sha1maps.begin());it!=_cached_sha1maps.end();++it)
{
ftFileCreator *client = NULL ;
for(uint32_t n=0;n<MAX_CHECKSUM_CHECK_PER_FILE && n < it->second._received.size();)
{
if(client == NULL)
{
std::map<std::string, ftClient>::iterator itc = mClients.find(it->first);
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum(): treating hash " << it->first << std::endl;
if(itc == mClients.end())
{
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum() ERROR: No matching Client for hash. This is an error. Hash=" << it->first << std::endl;
/* error */
break ;
}
else
client = itc->second.mCreator ;
}
int chunk_number = it->second._received[n] ;
if(!it->second._map.isSet(chunk_number))
{
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum() ERROR: chunk " << chunk_number << " is supposed to be initialized but it was not received !!" << std::endl;
++n ;
continue ;
}
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum(): checking chunk " << chunk_number << " with hash " << it->second._map[chunk_number].toStdString() << std::endl;
client->verifyChunk(chunk_number,it->second._map[chunk_number]) ;
it->second._received[n] = it->second._received.back() ;
it->second._received.pop_back() ;
}
}
return true ;
}
bool ftDataMultiplex::recvCRC32Map(const std::string& /*peerId*/, const std::string& hash,const CRC32Map& crc_map)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -708,6 +819,121 @@ bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId,
return true ;
}
bool ftDataMultiplex::handleRecvChunkCrcRequest(const std::string& peerId, const std::string& hash, uint32_t chunk_number)
{
// look into the sha1sum cache
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() looking for chunk " << chunk_number << " for hash " << hash << std::endl;
Sha1CheckSum crc ;
bool found = false ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
sha1cache.last_activity = time(NULL) ; // update time_stamp
if(sha1cache._map.size() > 0 && sha1cache._map.isSet(chunk_number))
{
crc = sha1cache._map[chunk_number] ;
found = true ;
}
}
if(found)
{
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() found in cache ! Sending " << crc.toStdString() << std::endl;
mDataSend->sendSingleChunkCRC(peerId,hash,chunk_number,crc);
return true ;
}
std::map<std::string, ftFileProvider *>::iterator it ;
std::string filename ;
uint64_t filesize =0;
found = true ;
// 1 - look into the list of servers.Not clients ! Clients dont' have verified data.
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end())
found = false ;
}
// 2 - if not found, create a server.
//
if(!found)
{
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
//#endif
if(!handleSearchRequest(peerId,hash))
return false ;
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
std::cerr << std::endl;
//#endif
}
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end()) // handleSearchRequest should have filled mServers[hash], but we have been off-mutex since,
{
std::cerr << "Could definitely not find a provider for file " << hash << ". Maybe the file does not exist?" << std::endl;
return false ; // so it's safer to check again.
}
else
{
filesize = it->second->fileSize() ;
filename = it->second->fileName() ;
}
}
std::cerr << "Computing Sha1 for chunk " << chunk_number<< " of file " << filename << ", hash=" << hash << ", size=" << filesize << std::endl;
unsigned char *buf = new unsigned char[ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE] ;
FILE *fd = fopen(filename.c_str(),"r") ;
if(fd == NULL)
{
std::cerr << "Cannot read file " << filename << ". Something's wrong!" << std::endl;
delete buf ;
return false ;
}
uint32_t len ;
if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,SEEK_SET)!=0 || 0==(len = fread(buf,1,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,fd)))
{
std::cerr << "Cannot fseek/read from file " << filename << " at position " << (uint64_t)chunk_number * (uint64_t)ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE << std::endl;
fclose(fd) ;
}
fclose(fd) ;
crc = RsDirUtil::sha1sum(buf,len) ;
// update cache
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
if(sha1cache._map.size() == 0)
sha1cache._map = Sha1Map(filesize,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE) ;
sha1cache._map.set(chunk_number,crc) ;
}
std::cerr << "Sending CRC of chunk " << chunk_number<< " of file " << filename << ", hash=" << hash << ", size=" << filesize << ", crc=" << crc.toStdString() << std::endl;
mDataSend->sendSingleChunkCRC(peerId,hash,chunk_number,crc);
return true ;
}
bool ftDataMultiplex::handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
@ -905,6 +1131,44 @@ bool ftDataMultiplex::sendCRC32MapRequest(const std::string& peer_id,const std::
{
return mDataSend->sendCRC32MapRequest(peer_id,hash);
}
bool ftDataMultiplex::sendSingleChunkCRCRequests(const std::string& hash, const std::vector<std::pair<uint32_t,std::list<std::string> > >& to_ask)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
// Put all requested chunks in the request queue.
Sha1CacheEntry& ce(_cached_sha1maps[hash]) ;
for(uint32_t i=0;i<to_ask.size();++i)
{
ChunkCheckSumSourceList& list(ce._to_ask[to_ask[i].first]) ;
for(std::list<std::string>::const_iterator it(to_ask[i].second.begin());it!=to_ask[i].second.end();++it)
list[*it] = 0 ;
}
return true ;
}
void ftDataMultiplex::handlePendingCrcRequests()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
time_t now = time(NULL) ;
uint32_t n=0 ;
for(std::map<std::string,Sha1CacheEntry>::iterator it(_cached_sha1maps.begin());it!=_cached_sha1maps.end();++it)
for(std::map<uint32_t,ChunkCheckSumSourceList>::iterator it2(it->second._to_ask.begin());it2!=it->second._to_ask.end();++it2)
for(std::map<std::string,time_t>::iterator it3(it2->second.begin());it3!=it2->second.end();++it3)
if(it3->second + MAX_CHECKING_CHUNK_WAIT_DELAY < now) // do nothing, otherwise, ask again
{
mDataSend->sendSingleChunkCRCRequest(it3->first,it->first,it2->first);
it3->second = now ;
if(++n > MAX_SIMULTANEOUS_CRC_REQUESTS)
return ;
}
}
void ftDataMultiplex::deleteUnusedServers()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

24
libretroshare/src/ft/ftdatamultiplex.h
View file

@ -78,8 +78,17 @@ class ftRequest
void *mData;
};
typedef std::map<std::string,time_t> ChunkCheckSumSourceList ;
class Sha1CacheEntry
{
public:
Sha1Map _map ; // Map of available sha1 sums for every chunk.
time_t last_activity ; // This is used for removing unused entries.
std::vector<uint32_t> _received ; // received chunk ids. To bedispatched.
std::map<uint32_t,ChunkCheckSumSourceList> _to_ask ; // Chunks to ask to sources.
};
class ftDataMultiplex: public ftDataRecv, public RsQueueThread
{
@ -98,6 +107,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool FileDetails(const std::string &hash, uint32_t hintsflag, FileInfo &info);
void deleteUnusedServers() ;
void handlePendingCrcRequests() ;
/*************** SEND INTERFACE (calls ftDataSend) *******************/
@ -117,6 +127,11 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
/* called from a separate thread */
bool computeAndSendCRC32Map(const std::string& peerId, const std::string& hash) ;
/* called from a separate thread */
bool sendSingleChunkCRCRequests(const std::string& hash, const std::vector<std::pair<uint32_t, std::list<std::string > > >& to_ask) ;
bool dispatchReceivedChunkCheckSum() ;
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
@ -132,6 +147,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
virtual bool recvCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) ;
/// Receive a CRC map request
virtual bool recvCRC32MapRequest(const std::string& peer_id,const std::string& hash) ;
virtual bool recvSingleChunkCrcRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_id) ;
virtual bool recvSingleChunkCrc(const std::string& peer_id,const std::string& hash,uint32_t chunk_id,const Sha1CheckSum& sum) ;
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
// map is too old. This supposes that the caller will ask again in a few seconds.
@ -153,6 +171,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvChunkCrcRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_id) ;
/* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
@ -168,6 +187,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
std::list<CRC32Thread *> _crc32map_threads ;
std::map<std::string,std::pair<time_t,CRC32Map> > _cached_crc32maps ;
std::map<std::string,Sha1CacheEntry> _cached_sha1maps ; // one cache entry per file hash. Handled dynamically.
ftDataSend *mDataSend;
ftSearch *mSearch;
std::string mOwnId;

50
libretroshare/src/ft/ftfilecreator.cc
View file

@ -609,6 +609,56 @@ bool ftFileCreator::hashReceivedData(std::string& hash)
return RsDirUtil::getFileHash(file_name,hash,tmpsize) ;
}
void ftFileCreator::forceCheck()
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap.forceCheck();
}
void ftFileCreator::getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_to_ask)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap.getChunksToCheck(chunks_to_ask) ;
}
bool ftFileCreator::verifyChunk(uint32_t chunk_number,const Sha1CheckSum& sum)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
if(!locked_initializeFileAttrs() )
return false ;
static const uint32_t chunk_size = ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE ;
unsigned char *buff = new unsigned char[chunk_size] ;
uint32_t len ;
if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
{
Sha1CheckSum comp = RsDirUtil::sha1sum(buff,len) ;
if(sum == comp)
chunkMap.setChunkCheckingResult(chunk_number,true) ;
else
{
std::cerr << "Sum mismatch for chunk " << chunk_number << std::endl;
std::cerr << " Computed hash = " << comp.toStdString() << std::endl;
std::cerr << " Reference hash = " << sum.toStdString() << std::endl;
chunkMap.setChunkCheckingResult(chunk_number,false) ;
}
}
else
{
printf("Chunk verification: cannot fseek!\n") ;
chunkMap.setChunkCheckingResult(chunk_number,false) ;
}
delete[] buff ;
return true ;
}
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks)
{
{

12
libretroshare/src/ft/ftfilecreator.h
View file

@ -76,6 +76,18 @@ class ftFileCreator: public ftFileProvider
// incomplete_chunks: count of any bad or not yet downloaded chunk
//
bool crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks) ;
// Sets all chunks to checking state
//
void forceCheck() ;
bool verifyChunk(uint32_t, const Sha1CheckSum&) ;
// Looks into the chunkmap for downloaded chunks that have not yet been certified.
// For each of them, returns the chunk number and a source peer to ask the CRC to.
//
void getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_to_ask) ;
/*
* creation functions for FileCreator
*/

95
libretroshare/src/ft/ftserver.cc
View file

@ -227,6 +227,8 @@ void ftServer::run()
while(isRunning())
{
mFtDataplex->deleteUnusedServers() ;
mFtDataplex->handlePendingCrcRequests() ;
mFtDataplex->dispatchReceivedChunkCheckSum() ;
#ifdef WIN32
Sleep(5000);
#else
@ -876,8 +878,28 @@ bool ftServer::sendCRC32MapRequest(const std::string& peerId,const std::string&
mP3iface->SendFileCRC32MapRequest(rfi);
}
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendSingleChunkCRCRequest(const std::string& peerId,const std::string& hash,uint32_t chunk_number)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendSingleChunkCRCRequest(peerId,hash,chunk_number) ;
else
{
/* create a packet */
/* push to networking part */
RsFileSingleChunkCrcRequest *rfi = new RsFileSingleChunkCrcRequest();
/* id */
rfi->PeerId(peerId);
/* file info */
rfi->hash = hash; /* ftr->hash; */
rfi->chunk_number = chunk_number ;
mP3iface->SendFileSingleChunkCrcRequest(rfi);
}
return true ;
}
@ -901,8 +923,29 @@ bool ftServer::sendCRC32Map(const std::string& peerId,const std::string& hash,co
mP3iface->SendFileCRC32Map(rfi);
}
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendSingleChunkCRC(const std::string& peerId,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendSingleChunkCRC(peerId,hash,chunk_number,crc) ;
else
{
/* create a packet */
/* push to networking part */
RsFileSingleChunkCrc *rfi = new RsFileSingleChunkCrc();
/* id */
rfi->PeerId(peerId);
/* file info */
rfi->hash = hash; /* ftr->hash; */
rfi->check_sum = crc;
rfi->chunk_number = chunk_number;
mP3iface->SendFileSingleChunkCrc(rfi);
}
return true ;
}
@ -1146,6 +1189,8 @@ bool ftServer::handleFileData()
RsFileChunkMap *fcm;
RsFileCRC32MapRequest *fccrcmr;
RsFileCRC32Map *fccrcm;
RsFileSingleChunkCrcRequest *fscrcr;
RsFileSingleChunkCrc *fscrc;
int i_init = 0;
int i = 0;
@ -1290,6 +1335,48 @@ FileInfo(ffr);
delete fccrcm;
}
// now file chunk crc requests
i_init = i;
while((fscrcr = mP3iface -> GetFileSingleChunkCrcRequest()) != NULL )
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File CRC Request:" << std::endl;
}
fscrcr -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
/* incoming data */
mFtDataplex->recvSingleChunkCrcRequest(fscrcr->PeerId(), fscrcr->hash,fscrcr->chunk_number) ;
delete fscrcr;
}
// now file chunkmaps
i_init = i;
while((fscrc = mP3iface -> GetFileSingleChunkCrc()) != NULL )
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File Data:" << std::endl;
}
fscrc -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
/* incoming data */
mFtDataplex->recvSingleChunkCrc(fscrc->PeerId(), fscrc->hash,fscrc->chunk_number,fscrc->check_sum);
delete fscrcr;
}
if (i > 0)
{
return 1;

4
libretroshare/src/ft/ftserver.h
View file

@ -229,8 +229,8 @@ virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& h
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
virtual bool sendCRC32MapRequest(const std::string&, const std::string&) ;
virtual bool sendCRC32Map(const std::string&, const std::string&, const CRC32Map&) ;
virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) ;
virtual bool sendSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ;
/*************** Internal Transfer Fns *************************/
virtual int tick();

23
libretroshare/src/ft/fttransfermodule.cc
View file

@ -426,6 +426,22 @@ bool ftTransferModule::queryInactive()
mFileStatus.stat = ftFileStatus::PQIFILE_CHECKING ;
mFlag = FT_TM_FLAG_CHECKING;
}
else
{
// request for CRCs to ask
std::vector<std::pair<uint32_t,std::list<std::string> > > chunks_to_ask ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::queryInactive() : getting chunks to check." << std::endl;
#endif
mFileCreator->getChunksToCheck(chunks_to_ask) ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::queryInactive() : got " << chunks_to_ask.size() << " chunks." << std::endl;
#endif
mMultiplexor->sendSingleChunkCRCRequests(mHash,chunks_to_ask);
}
return true;
}
@ -611,11 +627,16 @@ void ftTransferModule::forceCheck()
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::forceCheck(): setting flags to force check." << std::endl ;
#endif
#ifndef USE_NEW_CHUNK_CHECKING_CODE
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map.
// setup flags for CRC state machine to work properly
// setup flags for CRC state machine to work properly
_crcmap_state = FT_TM_CRC_MAP_STATE_DONT_HAVE ;
_crcmap_last_asked_time = 0 ;
#else
mFileCreator->forceCheck() ;
#endif
}
bool ftTransferModule::checkCRC()