Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-25 07:29:33 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added deferred check of chunks during file transfer. Chunk sha1 sums are requested to the sources and checked for downloaded data.

Browse Source 
Validated chunks are shared to other peers. Force check is now very simple since it just turns all chunks into "needs checking" mode
and sums are asked to sources. Sources maintain a temporary cache of chunks. Since sums are requested sparsely, this should not 
affect the sources in terms of performance. We can still imagine precomputing and saving sha1 of chunks while hashing them.

For backward compatibility reasons, the following has been setup *temporarily* in this version:
- unvalidated chunks are still considered as already obtained, and are shared and saved
- force check has been disabled
- final file check is maintained
- in case of file fail, the old checking mode will be used.

All changes for next version are kept in the define 'USE_NEW_CHUNK_CHECKING_CODE' that will be made the default in a few weeks.
At start, I expect most chunk to stya yellow during download, until most sources are able to provide chunk hashs.




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@5019 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2012-03-15 19:55:43 +00:00
parent 7ab5b54266
commit 889a2b2433
31 changed files with 1540 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
libretroshare/src/ft/ftchunkmap.cc
View File

@ -151,9 +151,91 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
#ifdef DEBUG_FTCHUNK
std::cerr << "*** ChunkMap::dataReceived: Chunk is complete. Removing it." << std::endl ;
#endif
_map[n] = FileChunksInfo::CHUNK_DONE ;
_map[n] = FileChunksInfo::CHUNK_CHECKING ;
_chunks_checking_queue.push_back(n) ;
_slices_to_download.erase(itc) ;
updateTotalDownloaded() ;
}
}
void ChunkMap::updateTotalDownloaded()
{
_total_downloaded = 0 ;
_file_is_complete = true ;
// First, round over chunk map to get the raw info.
//
for(uint32_t i=0;i<_map.size();++i)
switch(_map[i])
{
#ifdef USE_NEW_CHUNK_CHECKING_CODE
case FileChunksInfo::CHUNK_CHECKING: _file_is_complete = false ;
#else
case FileChunksInfo::CHUNK_CHECKING: //_file_is_complete = false ;********WARNING******** Re-enable this when users have massively switched to chunk checking code
#endif
case FileChunksInfo::CHUNK_DONE: _total_downloaded += sizeOfChunk(i) ;
break ;
default:
_file_is_complete = false ;
}
// Then go through active chunks.
//
for(std::map<ChunkNumber,ChunkDownloadInfo>::const_iterator itc(_slices_to_download.begin());itc!=_slices_to_download.end();++itc)
{
if(_map[itc->first] == FileChunksInfo::CHUNK_CHECKING)
_total_downloaded -= sizeOfChunk(itc->first) ;
_total_downloaded += sizeOfChunk(itc->first) - itc->second._remains ;
if(_file_is_complete)
std::cerr << "ChunkMap::updateTotalDownloaded(): ERROR: file still has pending slices but all chunks are marked as DONE !!" << std::endl;
}
}
void ChunkMap::getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_crc_to_ask)
{
chunks_crc_to_ask.clear() ;
for(uint32_t i=0;i<_chunks_checking_queue.size();)
{
std::list<std::string> peers ;
for(std::map<std::string,SourceChunksInfo>::const_iterator it2(_peers_chunks_availability.begin());it2!=_peers_chunks_availability.end();++it2)
if(it2->second.cmap[_chunks_checking_queue[i]])
peers.push_back(it2->first) ;
if(peers.empty()) // no peers => can't ask!
{
++i ;
continue ;
}
chunks_crc_to_ask.push_back(std::pair<uint32_t,std::list<std::string> >(_chunks_checking_queue[i],peers)) ;
// remove that chunk from the queue
_chunks_checking_queue[i] = _chunks_checking_queue.back() ;
_chunks_checking_queue.pop_back() ;
}
}
void ChunkMap::setChunkCheckingResult(uint32_t chunk_number,bool check_succeeded)
{
// Find the chunk is the waiting queue. Remove it, and mark it as done.
//
if(_map[chunk_number] != FileChunksInfo::CHUNK_CHECKING)
{
std::cerr << "(EE) ChunkMap: asked to set checking result of chunk " << chunk_number<< " that is not marked as being checked!!" << std::endl;
return ;
}
if(check_succeeded)
{
_map[chunk_number] = FileChunksInfo::CHUNK_DONE ;
// We also check whether the file is complete or not.
_file_is_complete = true ;
@ -165,6 +247,11 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
break ;
}
}
else
{
_total_downloaded -= sizeOfChunk(chunk_number) ; // restore completion.
_map[chunk_number] = FileChunksInfo::CHUNK_OUTSTANDING ;
}
}
// Warning: a chunk may be empty, but still being downloaded, so asking new slices from it
@ -495,6 +582,17 @@ void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
#endif
}
void ChunkMap::forceCheck()
{
for(uint32_t i=0;i<_map.size();++i)
{
_map[i] = FileChunksInfo::CHUNK_CHECKING ;
_chunks_checking_queue.push_back(i) ;
}
updateTotalDownloaded() ;
}
uint32_t ChunkMap::getNumberOfChunks(uint64_t size)
{
uint64_t n = size/(uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE ;

15
libretroshare/src/ft/ftchunkmap.h
View File

@ -178,8 +178,22 @@ class ChunkMap
/// returns true is the file is complete
bool isComplete() const { return _file_is_complete ; }
/// returns info about currently downloaded chunks
void getChunksInfo(FileChunksInfo& info) const ;
/// input the result of the chunk hash checking
void setChunkCheckingResult(uint32_t chunk_number, bool succeed) ;
/// returns the current list of chunks to ask for a CRC, and a proposed source for each
void getChunksToCheck(std::vector<std::pair<unsigned int, std::list<std::string> > >& chunks_to_ask) ;
/// sets all chunks to checking state
void forceCheck() ;
/// Goes through all structures and computes the actual file completion. The true completion
/// gets lost when force checking the file.
void updateTotalDownloaded() ;
protected:
/// handles what size the last chunk has.
uint32_t sizeOfChunk(uint32_t chunk_number) const ;
@ -199,6 +213,7 @@ class ChunkMap
uint64_t _total_downloaded ; //! completion for the file
bool _file_is_complete ; //! set to true when the file is complete.
bool _assume_availability ; //! true if all sources always have the complete file.
std::vector<uint32_t> _chunks_checking_queue ; //! Queue of downloaded chunks to be checked.
};

8
libretroshare/src/ft/ftdata.h
View File

@ -38,10 +38,12 @@
#include <string>
#include <inttypes.h>
#include <retroshare/rstypes.h>
/*************** SEND INTERFACE *******************/
class CompressedChunkMap ;
class CRC32Map ;
class Sha1CheckSum ;
class ftDataSend
{
@ -66,6 +68,10 @@ class ftDataSend
virtual bool sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk crc map
virtual bool sendCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) = 0;
/// Send a request for a chunk crc map
virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) = 0;
/// Send a chunk crc map
virtual bool sendSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) = 0;
};

264
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -42,6 +42,9 @@ const uint32_t DMULTIPLEX_MIN = 10; /* 1ms sleep */
const uint32_t DMULTIPLEX_MAX = 1000; /* 1 sec sleep */
const double DMULTIPLEX_RELAX = 0.5; /* ??? */
static const uint32_t MAX_CHECKING_CHUNK_WAIT_DELAY = 120 ; //! TTL for an inactive chunk
const uint32_t MAX_SIMULTANEOUS_CRC_REQUESTS = 20 ;
/******
* #define MPLEX_DEBUG 1
*****/
@ -57,6 +60,7 @@ const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map request to be treated by server
const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server
const uint32_t FT_CLIENT_CHUNK_CRC_REQ = 0x0006; // chunk sha1 crc request to be treated
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
@ -281,7 +285,19 @@ bool ftDataMultiplex::recvCRC32MapRequest(const std::string& peerId, const std::
return true;
}
bool ftDataMultiplex::recvSingleChunkCrcRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_number)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
std::cerr << std::endl;
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(ftRequest(FT_CLIENT_CHUNK_CRC_REQ,peerId,hash,0,0,chunk_number,NULL));
return true;
}
class CRC32Thread: public RsThread
{
public:
@ -387,6 +403,14 @@ bool ftDataMultiplex::doWork()
handleRecvCRC32MapRequest(req.mPeerId,req.mHash) ;
break ;
case FT_CLIENT_CHUNK_CRC_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_CRC_REQ";
std::cerr << std::endl;
#endif
handleRecvChunkCrcRequest(req.mPeerId,req.mHash,req.mChunk) ;
break ;
default:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
@ -465,6 +489,93 @@ bool ftDataMultiplex::doWork()
return true;
}
bool ftDataMultiplex::recvSingleChunkCrc(const std::string& /*peerId*/, const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() Received crc of file " << hash << ", chunk " << chunk_number << ", crc=" << crc.toStdString() << std::endl;
//#endif
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() ERROR: No matching Client for CRC. This is an error. " << hash << " !" << std::endl;
/* error */
return false;
}
// store in the cache as well
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
if(sha1cache._map.size() == 0)
sha1cache._map = Sha1Map(it->second.mCreator->fileSize(),ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE) ;
sha1cache._map.set(chunk_number,crc) ;
// remove this chunk from the request list as well.
std::map<uint32_t,ChunkCheckSumSourceList>::iterator it2(sha1cache._to_ask.find(chunk_number)) ;
if(it2 != sha1cache._to_ask.end())
sha1cache._to_ask.erase(it2) ;
sha1cache._received.push_back(chunk_number) ;
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvSingleChunkCrc() stored in cache. " << std::endl;
//#endif
return true ;
}
bool ftDataMultiplex::dispatchReceivedChunkCheckSum()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
uint32_t MAX_CHECKSUM_CHECK_PER_FILE = 25 ;
for(std::map<std::string,Sha1CacheEntry>::iterator it(_cached_sha1maps.begin());it!=_cached_sha1maps.end();++it)
{
ftFileCreator *client = NULL ;
for(uint32_t n=0;n<MAX_CHECKSUM_CHECK_PER_FILE && n < it->second._received.size();)
{
if(client == NULL)
{
std::map<std::string, ftClient>::iterator itc = mClients.find(it->first);
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum(): treating hash " << it->first << std::endl;
if(itc == mClients.end())
{
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum() ERROR: No matching Client for hash. This is an error. Hash=" << it->first << std::endl;
/* error */
break ;
}
else
client = itc->second.mCreator ;
}
int chunk_number = it->second._received[n] ;
if(!it->second._map.isSet(chunk_number))
{
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum() ERROR: chunk " << chunk_number << " is supposed to be initialized but it was not received !!" << std::endl;
++n ;
continue ;
}
std::cerr << "ftDataMultiplex::dispatchReceivedChunkCheckSum(): checking chunk " << chunk_number << " with hash " << it->second._map[chunk_number].toStdString() << std::endl;
client->verifyChunk(chunk_number,it->second._map[chunk_number]) ;
it->second._received[n] = it->second._received.back() ;
it->second._received.pop_back() ;
}
}
return true ;
}
bool ftDataMultiplex::recvCRC32Map(const std::string& /*peerId*/, const std::string& hash,const CRC32Map& crc_map)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -708,6 +819,121 @@ bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId,
return true ;
}
bool ftDataMultiplex::handleRecvChunkCrcRequest(const std::string& peerId, const std::string& hash, uint32_t chunk_number)
{
// look into the sha1sum cache
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() looking for chunk " << chunk_number << " for hash " << hash << std::endl;
Sha1CheckSum crc ;
bool found = false ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
sha1cache.last_activity = time(NULL) ; // update time_stamp
if(sha1cache._map.size() > 0 && sha1cache._map.isSet(chunk_number))
{
crc = sha1cache._map[chunk_number] ;
found = true ;
}
}
if(found)
{
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() found in cache ! Sending " << crc.toStdString() << std::endl;
mDataSend->sendSingleChunkCRC(peerId,hash,chunk_number,crc);
return true ;
}
std::map<std::string, ftFileProvider *>::iterator it ;
std::string filename ;
uint64_t filesize =0;
found = true ;
// 1 - look into the list of servers.Not clients ! Clients dont' have verified data.
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end())
found = false ;
}
// 2 - if not found, create a server.
//
if(!found)
{
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
//#endif
if(!handleSearchRequest(peerId,hash))
return false ;
//#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
std::cerr << std::endl;
//#endif
}
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end()) // handleSearchRequest should have filled mServers[hash], but we have been off-mutex since,
{
std::cerr << "Could definitely not find a provider for file " << hash << ". Maybe the file does not exist?" << std::endl;
return false ; // so it's safer to check again.
}
else
{
filesize = it->second->fileSize() ;
filename = it->second->fileName() ;
}
}
std::cerr << "Computing Sha1 for chunk " << chunk_number<< " of file " << filename << ", hash=" << hash << ", size=" << filesize << std::endl;
unsigned char *buf = new unsigned char[ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE] ;
FILE *fd = fopen(filename.c_str(),"r") ;
if(fd == NULL)
{
std::cerr << "Cannot read file " << filename << ". Something's wrong!" << std::endl;
delete buf ;
return false ;
}
uint32_t len ;
if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,SEEK_SET)!=0 || 0==(len = fread(buf,1,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,fd)))
{
std::cerr << "Cannot fseek/read from file " << filename << " at position " << (uint64_t)chunk_number * (uint64_t)ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE << std::endl;
fclose(fd) ;
}
fclose(fd) ;
crc = RsDirUtil::sha1sum(buf,len) ;
// update cache
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
Sha1CacheEntry& sha1cache(_cached_sha1maps[hash]) ;
if(sha1cache._map.size() == 0)
sha1cache._map = Sha1Map(filesize,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE) ;
sha1cache._map.set(chunk_number,crc) ;
}
std::cerr << "Sending CRC of chunk " << chunk_number<< " of file " << filename << ", hash=" << hash << ", size=" << filesize << ", crc=" << crc.toStdString() << std::endl;
mDataSend->sendSingleChunkCRC(peerId,hash,chunk_number,crc);
return true ;
}
bool ftDataMultiplex::handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
@ -905,6 +1131,44 @@ bool ftDataMultiplex::sendCRC32MapRequest(const std::string& peer_id,const std::
{
return mDataSend->sendCRC32MapRequest(peer_id,hash);
}
bool ftDataMultiplex::sendSingleChunkCRCRequests(const std::string& hash, const std::vector<std::pair<uint32_t,std::list<std::string> > >& to_ask)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
// Put all requested chunks in the request queue.
Sha1CacheEntry& ce(_cached_sha1maps[hash]) ;
for(uint32_t i=0;i<to_ask.size();++i)
{
ChunkCheckSumSourceList& list(ce._to_ask[to_ask[i].first]) ;
for(std::list<std::string>::const_iterator it(to_ask[i].second.begin());it!=to_ask[i].second.end();++it)
list[*it] = 0 ;
}
return true ;
}
void ftDataMultiplex::handlePendingCrcRequests()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
time_t now = time(NULL) ;
uint32_t n=0 ;
for(std::map<std::string,Sha1CacheEntry>::iterator it(_cached_sha1maps.begin());it!=_cached_sha1maps.end();++it)
for(std::map<uint32_t,ChunkCheckSumSourceList>::iterator it2(it->second._to_ask.begin());it2!=it->second._to_ask.end();++it2)
for(std::map<std::string,time_t>::iterator it3(it2->second.begin());it3!=it2->second.end();++it3)
if(it3->second + MAX_CHECKING_CHUNK_WAIT_DELAY < now) // do nothing, otherwise, ask again
{
mDataSend->sendSingleChunkCRCRequest(it3->first,it->first,it2->first);
it3->second = now ;
if(++n > MAX_SIMULTANEOUS_CRC_REQUESTS)
return ;
}
}
void ftDataMultiplex::deleteUnusedServers()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

24
libretroshare/src/ft/ftdatamultiplex.h
View File

@ -78,8 +78,17 @@ class ftRequest
void *mData;
};
typedef std::map<std::string,time_t> ChunkCheckSumSourceList ;
class Sha1CacheEntry
{
public:
Sha1Map _map ; // Map of available sha1 sums for every chunk.
time_t last_activity ; // This is used for removing unused entries.
std::vector<uint32_t> _received ; // received chunk ids. To bedispatched.
std::map<uint32_t,ChunkCheckSumSourceList> _to_ask ; // Chunks to ask to sources.
};
class ftDataMultiplex: public ftDataRecv, public RsQueueThread
{
@ -98,6 +107,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool FileDetails(const std::string &hash, uint32_t hintsflag, FileInfo &info);
void deleteUnusedServers() ;
void handlePendingCrcRequests() ;
/*************** SEND INTERFACE (calls ftDataSend) *******************/
@ -117,6 +127,11 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
/* called from a separate thread */
bool computeAndSendCRC32Map(const std::string& peerId, const std::string& hash) ;
/* called from a separate thread */
bool sendSingleChunkCRCRequests(const std::string& hash, const std::vector<std::pair<uint32_t, std::list<std::string > > >& to_ask) ;
bool dispatchReceivedChunkCheckSum() ;
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
@ -132,6 +147,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
virtual bool recvCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) ;
/// Receive a CRC map request
virtual bool recvCRC32MapRequest(const std::string& peer_id,const std::string& hash) ;
virtual bool recvSingleChunkCrcRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_id) ;
virtual bool recvSingleChunkCrc(const std::string& peer_id,const std::string& hash,uint32_t chunk_id,const Sha1CheckSum& sum) ;
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
// map is too old. This supposes that the caller will ask again in a few seconds.
@ -153,6 +171,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvChunkCrcRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_id) ;
/* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
@ -168,6 +187,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
std::list<CRC32Thread *> _crc32map_threads ;
std::map<std::string,std::pair<time_t,CRC32Map> > _cached_crc32maps ;
std::map<std::string,Sha1CacheEntry> _cached_sha1maps ; // one cache entry per file hash. Handled dynamically.
ftDataSend *mDataSend;
ftSearch *mSearch;
std::string mOwnId;

50
libretroshare/src/ft/ftfilecreator.cc
View File

@ -609,6 +609,56 @@ bool ftFileCreator::hashReceivedData(std::string& hash)
return RsDirUtil::getFileHash(file_name,hash,tmpsize) ;
}
void ftFileCreator::forceCheck()
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap.forceCheck();
}
void ftFileCreator::getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_to_ask)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap.getChunksToCheck(chunks_to_ask) ;
}
bool ftFileCreator::verifyChunk(uint32_t chunk_number,const Sha1CheckSum& sum)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
if(!locked_initializeFileAttrs() )
return false ;
static const uint32_t chunk_size = ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE ;
unsigned char *buff = new unsigned char[chunk_size] ;
uint32_t len ;
if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
{
Sha1CheckSum comp = RsDirUtil::sha1sum(buff,len) ;
if(sum == comp)
chunkMap.setChunkCheckingResult(chunk_number,true) ;
else
{
std::cerr << "Sum mismatch for chunk " << chunk_number << std::endl;
std::cerr << " Computed hash = " << comp.toStdString() << std::endl;
std::cerr << " Reference hash = " << sum.toStdString() << std::endl;
chunkMap.setChunkCheckingResult(chunk_number,false) ;
}
}
else
{
printf("Chunk verification: cannot fseek!\n") ;
chunkMap.setChunkCheckingResult(chunk_number,false) ;
}
delete[] buff ;
return true ;
}
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks)
{
{

12
libretroshare/src/ft/ftfilecreator.h
View File

@ -76,6 +76,18 @@ class ftFileCreator: public ftFileProvider
// incomplete_chunks: count of any bad or not yet downloaded chunk
//
bool crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks) ;
// Sets all chunks to checking state
//
void forceCheck() ;
bool verifyChunk(uint32_t, const Sha1CheckSum&) ;
// Looks into the chunkmap for downloaded chunks that have not yet been certified.
// For each of them, returns the chunk number and a source peer to ask the CRC to.
//
void getChunksToCheck(std::vector<std::pair<uint32_t,std::list<std::string> > >& chunks_to_ask) ;
/*
* creation functions for FileCreator
*/

95
libretroshare/src/ft/ftserver.cc
View File

@ -227,6 +227,8 @@ void ftServer::run()
while(isRunning())
{
mFtDataplex->deleteUnusedServers() ;
mFtDataplex->handlePendingCrcRequests() ;
mFtDataplex->dispatchReceivedChunkCheckSum() ;
#ifdef WIN32
Sleep(5000);
#else
@ -876,8 +878,28 @@ bool ftServer::sendCRC32MapRequest(const std::string& peerId,const std::string&
mP3iface->SendFileCRC32MapRequest(rfi);
}
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendSingleChunkCRCRequest(const std::string& peerId,const std::string& hash,uint32_t chunk_number)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendSingleChunkCRCRequest(peerId,hash,chunk_number) ;
else
{
/* create a packet */
/* push to networking part */
RsFileSingleChunkCrcRequest *rfi = new RsFileSingleChunkCrcRequest();
/* id */
rfi->PeerId(peerId);
/* file info */
rfi->hash = hash; /* ftr->hash; */
rfi->chunk_number = chunk_number ;
mP3iface->SendFileSingleChunkCrcRequest(rfi);
}
return true ;
}
@ -901,8 +923,29 @@ bool ftServer::sendCRC32Map(const std::string& peerId,const std::string& hash,co
mP3iface->SendFileCRC32Map(rfi);
}
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendSingleChunkCRC(const std::string& peerId,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendSingleChunkCRC(peerId,hash,chunk_number,crc) ;
else
{
/* create a packet */
/* push to networking part */
RsFileSingleChunkCrc *rfi = new RsFileSingleChunkCrc();
/* id */
rfi->PeerId(peerId);
/* file info */
rfi->hash = hash; /* ftr->hash; */
rfi->check_sum = crc;
rfi->chunk_number = chunk_number;
mP3iface->SendFileSingleChunkCrc(rfi);
}
return true ;
}
@ -1146,6 +1189,8 @@ bool ftServer::handleFileData()
RsFileChunkMap *fcm;
RsFileCRC32MapRequest *fccrcmr;
RsFileCRC32Map *fccrcm;
RsFileSingleChunkCrcRequest *fscrcr;
RsFileSingleChunkCrc *fscrc;
int i_init = 0;
int i = 0;
@ -1290,6 +1335,48 @@ FileInfo(ffr);
delete fccrcm;
}
// now file chunk crc requests
i_init = i;
while((fscrcr = mP3iface -> GetFileSingleChunkCrcRequest()) != NULL )
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File CRC Request:" << std::endl;
}
fscrcr -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
/* incoming data */
mFtDataplex->recvSingleChunkCrcRequest(fscrcr->PeerId(), fscrcr->hash,fscrcr->chunk_number) ;
delete fscrcr;
}
// now file chunkmaps
i_init = i;
while((fscrc = mP3iface -> GetFileSingleChunkCrc()) != NULL )
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File Data:" << std::endl;
}
fscrc -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
/* incoming data */
mFtDataplex->recvSingleChunkCrc(fscrc->PeerId(), fscrc->hash,fscrc->chunk_number,fscrc->check_sum);
delete fscrcr;
}
if (i > 0)
{
return 1;

4
libretroshare/src/ft/ftserver.h
View File

@ -229,8 +229,8 @@ virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& h
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
virtual bool sendCRC32MapRequest(const std::string&, const std::string&) ;
virtual bool sendCRC32Map(const std::string&, const std::string&, const CRC32Map&) ;
virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) ;
virtual bool sendSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ;
/*************** Internal Transfer Fns *************************/
virtual int tick();

23
libretroshare/src/ft/fttransfermodule.cc
View File

@ -426,6 +426,22 @@ bool ftTransferModule::queryInactive()
mFileStatus.stat = ftFileStatus::PQIFILE_CHECKING ;
mFlag = FT_TM_FLAG_CHECKING;
}
else
{
// request for CRCs to ask
std::vector<std::pair<uint32_t,std::list<std::string> > > chunks_to_ask ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::queryInactive() : getting chunks to check." << std::endl;
#endif
mFileCreator->getChunksToCheck(chunks_to_ask) ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::queryInactive() : got " << chunks_to_ask.size() << " chunks." << std::endl;
#endif
mMultiplexor->sendSingleChunkCRCRequests(mHash,chunks_to_ask);
}
return true;
}
@ -611,11 +627,16 @@ void ftTransferModule::forceCheck()
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::forceCheck(): setting flags to force check." << std::endl ;
#endif
#ifndef USE_NEW_CHUNK_CHECKING_CODE
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map.
// setup flags for CRC state machine to work properly
// setup flags for CRC state machine to work properly
_crcmap_state = FT_TM_CRC_MAP_STATE_DONT_HAVE ;
_crcmap_last_asked_time = 0 ;
#else
mFileCreator->forceCheck() ;
#endif
}
bool ftTransferModule::checkCRC()

6
libretroshare/src/pqi/pqi.h
View File

@ -78,6 +78,12 @@ virtual int SendFileCRC32MapRequest(RsFileCRC32MapRequest *) = 0;
virtual RsFileCRC32Map *GetFileCRC32Map() = 0;
virtual int SendFileCRC32Map(RsFileCRC32Map *) = 0;
virtual RsFileSingleChunkCrcRequest *GetFileSingleChunkCrcRequest()=0;
virtual int SendFileSingleChunkCrcRequest(RsFileSingleChunkCrcRequest *ns)=0;
virtual RsFileSingleChunkCrc *GetFileSingleChunkCrc()=0;
virtual int SendFileSingleChunkCrc(RsFileSingleChunkCrc *ns)=0;
};
class P3Interface: public SearchInterface

43
libretroshare/src/pqi/pqihandler.cc
View File

@ -395,6 +395,14 @@ int pqihandler::SendFileCRC32Map(RsFileCRC32Map *ns)
{
return queueOutRsItem(ns) ;
}
int pqihandler::SendFileSingleChunkCrcRequest(RsFileSingleChunkCrcRequest *ns)
{
return queueOutRsItem(ns) ;
}
int pqihandler::SendFileSingleChunkCrc(RsFileSingleChunkCrc *ns)
{
return queueOutRsItem(ns) ;
}
int pqihandler::SendRsRawItem(RsRawItem *ns)
{
@ -572,6 +580,17 @@ void pqihandler::locked_SortnStoreItem(RsItem *item)
item = NULL;
break;
case RS_PKT_SUBTYPE_FI_CHUNK_CRC_REQUEST:
pqioutput(PQL_DEBUG_BASIC, pqihandlerzone, "SortnStore -> File Crc Request");
in_singlechunkcrc_request.push_back(item);
item = NULL;
break;
case RS_PKT_SUBTYPE_FI_CHUNK_CRC:
pqioutput(PQL_DEBUG_BASIC, pqihandlerzone, "SortnStore -> File CRC32Map");
in_singlechunkcrc.push_back(item);
item = NULL;
break;
default:
break; /* no match! */
@ -709,8 +728,32 @@ RsFileCRC32Map *pqihandler::GetFileCRC32Map()
}
return NULL;
}
RsFileSingleChunkCrcRequest *pqihandler::GetFileSingleChunkCrcRequest()
{
RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
if (in_singlechunkcrc_request.size() != 0)
{
RsFileSingleChunkCrcRequest *fi = dynamic_cast<RsFileSingleChunkCrcRequest *>(in_singlechunkcrc_request.front());
if (!fi) { delete in_singlechunkcrc_request.front(); }
in_singlechunkcrc_request.pop_front();
return fi;
}
return NULL;
}
RsFileSingleChunkCrc *pqihandler::GetFileSingleChunkCrc()
{
RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
if (in_singlechunkcrc.size() != 0)
{
RsFileSingleChunkCrc *fi = dynamic_cast<RsFileSingleChunkCrc *>(in_singlechunkcrc.front());
if (!fi) { delete in_singlechunkcrc.front(); }
in_singlechunkcrc.pop_front();
return fi;
}
return NULL;
}
RsRawItem *pqihandler::GetRsRawItem()
{
RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/

6
libretroshare/src/pqi/pqihandler.h
View File

@ -68,12 +68,16 @@ class pqihandler: public P3Interface, public pqiQoS
virtual int SendFileChunkMap(RsFileChunkMap *ns);
virtual int SendFileCRC32MapRequest(RsFileCRC32MapRequest *ns);
virtual int SendFileCRC32Map(RsFileCRC32Map *ns);
virtual int SendFileSingleChunkCrcRequest(RsFileSingleChunkCrcRequest *ns);
virtual int SendFileSingleChunkCrc(RsFileSingleChunkCrc *ns);
virtual RsFileRequest *GetFileRequest();
virtual RsFileData *GetFileData();
virtual RsFileChunkMapRequest *GetFileChunkMapRequest();
virtual RsFileChunkMap *GetFileChunkMap();
virtual RsFileCRC32MapRequest *GetFileCRC32MapRequest();
virtual RsFileCRC32Map *GetFileCRC32Map();
virtual RsFileSingleChunkCrcRequest *GetFileSingleChunkCrcRequest();
virtual RsFileSingleChunkCrc *GetFileSingleChunkCrc();
// Rest of P3Interface
virtual int tick();
@ -112,7 +116,7 @@ class pqihandler: public P3Interface, public pqiQoS
SecurityPolicy *globsec;
// Temporary storage...
std::list<RsItem *> in_result, in_search, in_request, in_data, in_service,in_chunkmap,in_chunkmap_request,in_crc32map_request,in_crc32map;
std::list<RsItem *> in_result, in_search, in_request, in_data, in_service,in_chunkmap,in_chunkmap_request,in_crc32map_request,in_crc32map,in_singlechunkcrc,in_singlechunkcrc_request;
private:

40
libretroshare/src/retroshare/rstypes.h
View File

@ -54,6 +54,20 @@ const uint32_t RS_PARTIALS_DIRECTORY = 0x0000 ;
const uint32_t RS_DOWNLOAD_DIRECTORY = 0x0001 ;
const uint32_t RS_CONFIG_DIRECTORY = 0x0002 ;
class Sha1CheckSum
{
public:
Sha1CheckSum() {}
explicit Sha1CheckSum(const uint8_t *twenty_bytes_digest) ; // inits form a 20-bytes digest.
explicit Sha1CheckSum(const std::string& fourty_bytes_string) ; // inits form a 40 bytes hexadecimal string.
std::string toStdString() const ;
bool operator==(const Sha1CheckSum& s) const ;
// private:
uint32_t fourbytes[5] ;
};
class TransferInfo
{
public:
@ -286,7 +300,7 @@ class CompressedChunkMap ;
class FileChunksInfo
{
public:
enum ChunkState { CHUNK_DONE=2, CHUNK_ACTIVE=1, CHUNK_OUTSTANDING=0 } ;
enum ChunkState { CHUNK_CHECKING=3, CHUNK_DONE=2, CHUNK_ACTIVE=1, CHUNK_OUTSTANDING=0 } ;
enum ChunkStrategy { CHUNK_STRATEGY_STREAMING, CHUNK_STRATEGY_RANDOM } ;
struct SliceInfo
@ -326,7 +340,11 @@ class CompressedChunkMap
_map.resize( getCompressedSize(uncompressed_data.size()),0 ) ;
for(uint32_t i=0;i<uncompressed_data.size();++i)
#ifdef USE_NEW_CHUNK_CHECKING_CODE
if(uncompressed_data[i]==FileChunksInfo::CHUNK_DONE)
#else
if(uncompressed_data[i]==FileChunksInfo::CHUNK_DONE || uncompressed_data[i] == FileChunksInfo::CHUNK_CHECKING)
#endif
set(i) ;
}
@ -353,27 +371,24 @@ class CompressedChunkMap
std::vector<uint32_t> _map ;
};
class CRC32Map
template<class CRCTYPE> class t_CRCMap
{
public:
// Build from a file.
//
CRC32Map(uint64_t file_size,uint32_t chunk_size)
t_CRCMap(uint64_t file_size,uint32_t chunk_size)
: _crcs( file_size/chunk_size + ( (file_size%chunk_size)>0)), _ccmap(file_size/chunk_size + ( (file_size%chunk_size)>0),0)
{
}
CRC32Map() {}
t_CRCMap() {}
// Compares two maps and returns the valid chunks in a compressed chunk map.
//
friend CompressedChunkMap compare(const CRC32Map& crc1,const CRC32Map& crc2) ;
inline void set(uint32_t i,const CRCTYPE& val) { _crcs[i] = val ; _ccmap.set(i) ; }
inline bool isSet(uint32_t i) const { return _ccmap[i] ; }
inline void set(uint32_t i,uint32_t val) { _crcs[i] = val ; _ccmap.set(i) ; }
inline uint32_t operator[](int i) const { return _crcs[i] ; }
inline const CRCTYPE& operator[](int i) const { return _crcs[i] ; }
inline uint32_t size() const { return _crcs.size() ; }
private:
std::vector<uint32_t> _crcs;
std::vector<CRCTYPE> _crcs;
CompressedChunkMap _ccmap ;
friend class RsTurtleFileCrcItem ;
@ -381,6 +396,9 @@ class CRC32Map
friend class RsFileCRC32Map ;
};
typedef t_CRCMap<uint32_t> CRC32Map ;
typedef t_CRCMap<Sha1CheckSum> Sha1Map ;
/* class which encapsulates download details */
class DwlDetails {
public:

4
libretroshare/src/serialiser/itempriorities.h
View File

@ -36,10 +36,12 @@ const uint8_t QOS_PRIORITY_RS_TURTLE_TUNNEL_OK = 6 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_CRC_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_CHUNK_CRC_REQUEST= 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_MAP_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_DATA = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_CRC = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_CHUNK_CRC = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_MAP = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_ITEM = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FORWARD_FILE_DATA= 2 ;
@ -48,10 +50,12 @@ const uint8_t QOS_PRIORITY_RS_TURTLE_FORWARD_FILE_DATA= 2 ;
//
const uint8_t QOS_PRIORITY_RS_FILE_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_FILE_CRC_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_CHUNK_CRC_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_FILE_MAP_REQUEST = 5 ;
const uint8_t QOS_PRIORITY_RS_CACHE_REQUEST = 4 ;
const uint8_t QOS_PRIORITY_RS_FILE_DATA = 3 ;
const uint8_t QOS_PRIORITY_RS_FILE_CRC = 3 ;
const uint8_t QOS_PRIORITY_RS_CHUNK_CRC = 5 ;
const uint8_t QOS_PRIORITY_RS_FILE_MAP = 3 ;
const uint8_t QOS_PRIORITY_RS_CACHE_ITEM = 3 ;

243
libretroshare/src/serialiser/rsbaseitems.cc
View File

@ -50,6 +50,8 @@ uint32_t RsFileItemSerialiser::size(RsItem *i)
RsFileChunkMap *rfcm;
RsFileCRC32MapRequest *rfcrcr;
RsFileCRC32Map *rfcrc;
RsFileSingleChunkCrcRequest *rfscrcr;
RsFileSingleChunkCrc *rfscrc;
if (NULL != (rfr = dynamic_cast<RsFileRequest *>(i)))
{
@ -75,6 +77,14 @@ uint32_t RsFileItemSerialiser::size(RsItem *i)
{
return sizeCRC32Map(rfcrc);
}
else if (NULL != (rfscrcr = dynamic_cast<RsFileSingleChunkCrcRequest *>(i)))
{
return sizeChunkCrcReq(rfscrcr);
}
else if (NULL != (rfscrc = dynamic_cast<RsFileSingleChunkCrc *>(i)))
{
return sizeChunkCrc(rfscrc);
}
return 0;
}
@ -88,6 +98,8 @@ bool RsFileItemSerialiser::serialise(RsItem *i, void *data, uint32_t *pktsize
RsFileChunkMap *rfcm;
RsFileCRC32MapRequest *rfcrcr;
RsFileCRC32Map *rfcrc;
RsFileSingleChunkCrcRequest *rfscrcr;
RsFileSingleChunkCrc *rfscrc;
if (NULL != (rfr = dynamic_cast<RsFileRequest *>(i)))
{
@ -113,6 +125,16 @@ bool RsFileItemSerialiser::serialise(RsItem *i, void *data, uint32_t *pktsize
{
return serialiseCRC32Map(rfcrc,data,pktsize);
}
else if (NULL != (rfscrcr = dynamic_cast<RsFileSingleChunkCrcRequest *>(i)))
{
return serialiseChunkCrcReq(rfscrcr,data,pktsize);
}
else if (NULL != (rfscrc = dynamic_cast<RsFileSingleChunkCrc *>(i)))
{
return serialiseChunkCrc(rfscrc,data,pktsize);
}
std::cerr << "RsFileItemSerialiser::serialize(): unhandled packet type !!" << std::endl;
return false;
}
@ -148,6 +170,12 @@ RsItem *RsFileItemSerialiser::deserialise(void *data, uint32_t *pktsize)
case RS_PKT_SUBTYPE_FI_CRC32_MAP:
return deserialiseCRC32Map(data, pktsize);
break;
case RS_PKT_SUBTYPE_FI_CHUNK_CRC_REQUEST:
return deserialiseChunkCrcReq(data, pktsize);
break;
case RS_PKT_SUBTYPE_FI_CHUNK_CRC:
return deserialiseChunkCrc(data, pktsize);
break;
default:
return NULL;
break;
@ -344,6 +372,27 @@ std::ostream& RsFileCRC32Map::print(std::ostream &out, uint16_t indent)
printRsItemEnd(out, "RsFileCRC32Map", indent);
return out;
}
std::ostream& RsFileSingleChunkCrcRequest::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileSingleChunkCrcRequest", indent);
uint16_t int_Indent = indent + 2;
printIndent(out, int_Indent); out << "PeerId: " << PeerId() << std::endl ;
printIndent(out, int_Indent); out << " hash: " << hash << std::endl ;
printIndent(out, int_Indent); out << " chunk: " << chunk_number << "..." << std::endl ;
printRsItemEnd(out, "RsFileSingleChunkCrcRequest", indent);
return out;
}
std::ostream& RsFileSingleChunkCrc::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileSingleChunkCrc", indent);
uint16_t int_Indent = indent + 2;
printIndent(out, int_Indent); out << "PeerId: " << PeerId() << std::endl ;
printIndent(out, int_Indent); out << " hash: " << hash << std::endl ;
printIndent(out, int_Indent); out << " chunk: " << chunk_number << "..." << std::endl ;
printIndent(out, int_Indent); out << " sha1: " << check_sum.toStdString() << "..." << std::endl ;
printRsItemEnd(out, "RsFileSingleChunkCrc", indent);
return out;
}
std::ostream& RsFileCRC32MapRequest::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileCRC32MapRequest", indent);
@ -487,6 +536,23 @@ uint32_t RsFileItemSerialiser::sizeChunkMap(RsFileChunkMap *item)
return s;
}
uint32_t RsFileItemSerialiser::sizeChunkCrc(RsFileSingleChunkCrc *item)
{
uint32_t s = 8; /* header */
s += GetTlvStringSize(item->hash) ; // hash
s += 4 ; // chunk number
s += 20 ; // sha1
return s;
}
uint32_t RsFileItemSerialiser::sizeChunkCrcReq(RsFileSingleChunkCrcRequest *item)
{
uint32_t s = 8; /* header */
s += GetTlvStringSize(item->hash) ; // hash
s += 4 ; // chunk number
return s;
}
uint32_t RsFileItemSerialiser::sizeCRC32MapReq(RsFileCRC32MapRequest *item)
{
uint32_t s = 8; /* header */
@ -734,6 +800,81 @@ bool RsFileItemSerialiser::serialiseChunkMapReq(RsFileChunkMapRequest *item,
return ok;
}
bool RsFileItemSerialiser::serialiseChunkCrcReq(RsFileSingleChunkCrcRequest *item, void *data, uint32_t *pktsize)
{
uint32_t tlvsize = sizeChunkCrcReq(item);
uint32_t offset = 0;
if (*pktsize < tlvsize)
return false; /* not enough space */
*pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize);
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl;
#endif
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_VALUE, item->hash);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number) ;
if (offset != tlvsize)
{
ok = false;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Size Error! " << std::endl;
#endif
}
return ok;
}
bool RsFileItemSerialiser::serialiseChunkCrc(RsFileSingleChunkCrc *item, void *data, uint32_t *pktsize)
{
uint32_t tlvsize = sizeChunkCrc(item);
uint32_t offset = 0;
if (*pktsize < tlvsize)
return false; /* not enough space */
*pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize);
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl;
#endif
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_VALUE, item->hash);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number) ;
ok &= setRawUInt32(data, tlvsize, &offset, item->check_sum.fourbytes[0]) ;
ok &= setRawUInt32(data, tlvsize, &offset, item->check_sum.fourbytes[1]) ;
ok &= setRawUInt32(data, tlvsize, &offset, item->check_sum.fourbytes[2]) ;
ok &= setRawUInt32(data, tlvsize, &offset, item->check_sum.fourbytes[3]) ;
ok &= setRawUInt32(data, tlvsize, &offset, item->check_sum.fourbytes[4]) ;
if (offset != tlvsize)
{
ok = false;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Size Error! " << std::endl;
#endif
}
return ok;
}
bool RsFileItemSerialiser::serialiseChunkMap(RsFileChunkMap *item, void *data, uint32_t *pktsize)
{
uint32_t tlvsize = sizeChunkMap(item);
@ -1200,7 +1341,109 @@ RsCacheItem *RsCacheItemSerialiser::deserialiseItem(void *data, uint32_t *pktsiz
return item;
}
RsFileSingleChunkCrcRequest *RsFileItemSerialiser::deserialiseChunkCrcReq(void *data, uint32_t *pktsize)
{
/* get the type and size */
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
uint32_t offset = 0;
if ((RS_PKT_VERSION1 != getRsItemVersion(rstype)) ||
(RS_PKT_CLASS_BASE != getRsItemClass(rstype)) ||
(RS_PKT_TYPE_FILE != getRsItemType(rstype)) ||
(RS_PKT_SUBTYPE_FI_CHUNK_CRC_REQUEST != getRsItemSubType(rstype)))
{
return NULL; /* wrong type */
}
if (*pktsize < rssize) /* check size */
return NULL; /* not enough data */
/* set the packet length */
*pktsize = rssize;
bool ok = true;
/* ready to load */
RsFileSingleChunkCrcRequest *item = new RsFileSingleChunkCrcRequest();
item->clear();
/* skip the header */
offset += 8;
uint8_t tmp ;
ok &= GetTlvString(data, *pktsize, &offset, TLV_TYPE_STR_VALUE, item->hash); // file hash
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
if (offset != rssize)
{
/* error */
delete item;
return NULL;
}
if (!ok)
{
delete item;
return NULL;
}
return item;
}
RsFileSingleChunkCrc *RsFileItemSerialiser::deserialiseChunkCrc(void *data, uint32_t *pktsize)
{
/* get the type and size */
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
uint32_t offset = 0;
if ((RS_PKT_VERSION1 != getRsItemVersion(rstype)) ||
(RS_PKT_CLASS_BASE != getRsItemClass(rstype)) ||
(RS_PKT_TYPE_FILE != getRsItemType(rstype)) ||
(RS_PKT_SUBTYPE_FI_CHUNK_CRC != getRsItemSubType(rstype)))
{
return NULL; /* wrong type */
}
if (*pktsize < rssize) /* check size */
return NULL; /* not enough data */
/* set the packet length */
*pktsize = rssize;
bool ok = true;
/* ready to load */
RsFileSingleChunkCrc *item = new RsFileSingleChunkCrc();
item->clear();
/* skip the header */
offset += 8;
uint8_t tmp ;
ok &= GetTlvString(data, *pktsize, &offset, TLV_TYPE_STR_VALUE, item->hash); // file hash
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
ok &= getRawUInt32(data, rssize, &offset, &(item->check_sum.fourbytes[0]));
ok &= getRawUInt32(data, rssize, &offset, &(item->check_sum.fourbytes[1]));
ok &= getRawUInt32(data, rssize, &offset, &(item->check_sum.fourbytes[2]));
ok &= getRawUInt32(data, rssize, &offset, &(item->check_sum.fourbytes[3]));
ok &= getRawUInt32(data, rssize, &offset, &(item->check_sum.fourbytes[4]));
if (offset != rssize)
{
/* error */
delete item;
return NULL;
}
if (!ok)
{
delete item;
return NULL;
}
return item;
}
/*************************************************************************/
/*************************************************************************/

45
libretroshare/src/serialiser/rsbaseitems.h
View File

@ -42,6 +42,8 @@ const uint8_t RS_PKT_SUBTYPE_FI_CHUNK_MAP_REQUEST = 0x04;
const uint8_t RS_PKT_SUBTYPE_FI_CHUNK_MAP = 0x05;
const uint8_t RS_PKT_SUBTYPE_FI_CRC32_MAP_REQUEST = 0x06;
const uint8_t RS_PKT_SUBTYPE_FI_CRC32_MAP = 0x07;
const uint8_t RS_PKT_SUBTYPE_FI_CHUNK_CRC_REQUEST = 0x08;
const uint8_t RS_PKT_SUBTYPE_FI_CHUNK_CRC = 0x09;
const uint8_t RS_PKT_SUBTYPE_CACHE_ITEM = 0x01;
const uint8_t RS_PKT_SUBTYPE_CACHE_REQUEST = 0x02;
@ -137,6 +139,24 @@ class RsFileCRC32MapRequest: public RsItem
std::ostream &print(std::ostream &out, uint16_t indent = 0);
};
class RsFileSingleChunkCrcRequest: public RsItem
{
public:
RsFileSingleChunkCrcRequest()
:RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_BASE, RS_PKT_TYPE_FILE, RS_PKT_SUBTYPE_FI_CHUNK_CRC_REQUEST)
{
setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC_REQUEST) ;
}
virtual ~RsFileSingleChunkCrcRequest() {}
virtual void clear() {}
std::string hash ; // hash of the file for which we request the crc
uint32_t chunk_number ; // chunk number
std::ostream &print(std::ostream &out, uint16_t indent = 0);
};
class RsFileCRC32Map: public RsItem
{
public:
@ -153,6 +173,25 @@ class RsFileCRC32Map: public RsItem
std::ostream &print(std::ostream &out, uint16_t indent = 0);
};
class RsFileSingleChunkCrc: public RsItem
{
public:
RsFileSingleChunkCrc()
:RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_BASE, RS_PKT_TYPE_FILE, RS_PKT_SUBTYPE_FI_CHUNK_CRC)
{
setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC) ;
}
virtual ~RsFileSingleChunkCrc() {}
virtual void clear() {}
std::string hash ; // hash of the file for which we request the chunk map
uint32_t chunk_number ;
Sha1CheckSum check_sum ; // CRC32 map of the file.
std::ostream &print(std::ostream &out, uint16_t indent = 0);
};
/**************************************************************************/
class RsFileItemSerialiser: public RsSerialType
@ -175,6 +214,8 @@ class RsFileItemSerialiser: public RsSerialType
virtual uint32_t sizeData(RsFileData *);
virtual uint32_t sizeChunkMapReq(RsFileChunkMapRequest *);
virtual uint32_t sizeChunkMap(RsFileChunkMap *);
virtual uint32_t sizeChunkCrcReq(RsFileSingleChunkCrcRequest *);
virtual uint32_t sizeChunkCrc(RsFileSingleChunkCrc *);
virtual uint32_t sizeCRC32MapReq(RsFileCRC32MapRequest *);
virtual uint32_t sizeCRC32Map(RsFileCRC32Map *);
@ -184,6 +225,8 @@ class RsFileItemSerialiser: public RsSerialType
virtual bool serialiseChunkMap(RsFileChunkMap *item, void *data, uint32_t *size);
virtual bool serialiseCRC32MapReq(RsFileCRC32MapRequest *item, void *data, uint32_t *size);
virtual bool serialiseCRC32Map(RsFileCRC32Map *item, void *data, uint32_t *size);
virtual bool serialiseChunkCrcReq(RsFileSingleChunkCrcRequest *item, void *data, uint32_t *size);
virtual bool serialiseChunkCrc(RsFileSingleChunkCrc *item, void *data, uint32_t *size);
virtual RsFileRequest *deserialiseReq(void *data, uint32_t *size);
virtual RsFileData *deserialiseData(void *data, uint32_t *size);
@ -191,6 +234,8 @@ class RsFileItemSerialiser: public RsSerialType
virtual RsFileChunkMap *deserialiseChunkMap(void *data, uint32_t *size);
virtual RsFileCRC32MapRequest *deserialiseCRC32MapReq(void *data, uint32_t *size);
virtual RsFileCRC32Map *deserialiseCRC32Map(void *data, uint32_t *size);
virtual RsFileSingleChunkCrcRequest *deserialiseChunkCrcReq(void *data, uint32_t *size);
virtual RsFileSingleChunkCrc *deserialiseChunkCrc(void *data, uint32_t *size);
};
/**************************************************************************/

24
libretroshare/src/tests/util/Makefile Normal file
View File

@ -0,0 +1,24 @@
RS_TOP_DIR = ../..
DHT_TOP_DIR = ../../../../libbitdht/src
##### Define any flags that are needed for this section #######
###############################################################
###############################################################
include $(RS_TOP_DIR)/tests/scripts/config.mk
###############################################################
TESTOBJ = dirtest.o sha1_test.o
TESTS = dirtest sha1_test
all: tests
sha1_test: sha1_test.o
$(CC) $(CFLAGS) -o sha1_test sha1_test.o $(LIBS)
dirtest: dirtest.o
$(CC) $(CFLAGS) -o dirtest dirtest.o $(LIBS)
###############################################################
include $(RS_TOP_DIR)/tests/scripts/rules.mk
###############################################################

80
libretroshare/src/tests/util/sha1_test.cc Normal file
View File

@ -0,0 +1,80 @@
/*
* "$Id: dirtest.cc,v 1.1 2007-02-19 20:08:30 rmf24 Exp $"
*
* RetroShare C++ Interface.
*
* Copyright 2012-2012 by Cyril Soler
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "util/rsdir.h"
#include <iostream>
#include <list>
#include <string>
#include <stdio.h>
void printHelp(int argc,char *argv[])
{
std::cerr << argv[0] << ": test RS sha1sum class implementation." << std::endl;
std::cerr << "Usage: " << argv[0] << " [test file]" << std::endl ;
}
int main(int argc,char *argv[])
{
if(argc != 2)
{
printHelp(argc,argv) ;
return -1 ;
}
FILE *f = fopen(argv[1],"r") ;
if(f == NULL)
{
std::cerr << "Cannot open file " << argv[1] << " for read !" << std::endl;
return -1 ;
}
std::cerr << "Testing sha1" << std::endl;
uint32_t SIZE = 1024*1024 ;
unsigned char *buf = new unsigned char[SIZE] ;
int len = fread(buf,1,SIZE,f) ;
std::cerr << "Read " << len << " bytes" << std::endl;
Sha1CheckSum sum = RsDirUtil::sha1sum(buf,len) ;
std::cerr << std::hex << sum.fourbytes[0] << std::endl;
std::cerr << "New method : " << sum.toStdString() << std::endl;
std::string hash ;
uint64_t size ;
RsDirUtil::getFileHash(argv[1],hash,size) ;
std::cerr << "Old method : " << hash << std::endl;
Sha1CheckSum H(hash) ;
std::cerr << "Hashed transformed: " << H.toStdString() << std::endl;
return 0 ;
}

137
libretroshare/src/turtle/p3turtle.cc
View File

@ -1001,6 +1001,12 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST: handleRecvFileCRC32MapRequest(dynamic_cast<RsTurtleFileCrcRequestItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC : handleRecvChunkCRC(dynamic_cast<RsTurtleChunkCrcItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST: handleRecvChunkCRCRequest(dynamic_cast<RsTurtleChunkCrcRequestItem *>(item)) ;
break ;
default:
std::cerr << "WARNING: Unknown packet type received: id=" << (void*)(item->PacketSubType()) << ". Is somebody trying to poison you ?" << std::endl ;
#ifdef P3TURTLE_DEBUG
@ -1226,7 +1232,78 @@ void p3turtle::handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem *item)
_ft_server->getMultiplexer()->recvCRC32MapRequest(vpid,hash) ;
}
void p3turtle::handleRecvChunkCRC(RsTurtleChunkCrcItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map Request item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: chunk crc request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file crc request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvSingleChunkCrc(vpid,hash,item->chunk_number,item->check_sum) ;
}
void p3turtle::handleRecvChunkCRCRequest(RsTurtleChunkCrcRequestItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map Request item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: chunk crc request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file crc request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvSingleChunkCrcRequest(vpid,hash,item->chunk_number) ;
}
void p3turtle::handleRecvFileCRC32Map(RsTurtleFileCrcItem *item)
{
#ifdef P3TURTLE_DEBUG
@ -1435,7 +1512,67 @@ void p3turtle::sendChunkMap(const std::string& peerId,const std::string& ,const
#endif
sendItem(item) ;
}
void p3turtle::sendSingleChunkCRC(const std::string& peerId,const std::string&,uint32_t chunk_number,const Sha1CheckSum& crc)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::sendCRC32MapRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleChunkCrcItem *item = new RsTurtleChunkCrcItem;
item->tunnel_id = tunnel_id ;
item->chunk_number = chunk_number ;
item->check_sum = crc ;
item->PeerId(tunnel.local_dst) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map request to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendSingleChunkCRCRequest(const std::string& peerId,const std::string&,uint32_t chunk_number)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::sendCRC32MapRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleChunkCrcRequestItem *item = new RsTurtleChunkCrcRequestItem;
item->tunnel_id = tunnel_id ;
item->chunk_number = chunk_number ;
item->PeerId(tunnel.local_dst) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map request to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendCRC32MapRequest(const std::string& peerId,const std::string& )
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/

8
libretroshare/src/turtle/p3turtle.h
View File

@ -301,6 +301,12 @@ class p3turtle: public p3Service, /*public pqiMonitor,*/ public RsTurtle,/* publ
/// Send a crc32 map of this file to the given peer
void sendCRC32Map(const std::string& peerId, const std::string& hash,const CRC32Map& cmap) ;
/// Send a request for the CRC of a single chunk of this file to the given peer
void sendSingleChunkCRCRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_number) ;
/// Send a crc32 map of this file to the given peer
void sendSingleChunkCRC(const std::string& peerId, const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& sum) ;
private:
//--------------------------- Admin/Helper functions -------------------------//
@ -352,6 +358,8 @@ class p3turtle: public p3Service, /*public pqiMonitor,*/ public RsTurtle,/* publ
void handleRecvFileMap(RsTurtleFileMapItem*);
void handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem*);
void handleRecvFileCRC32Map(RsTurtleFileCrcItem*);
void handleRecvChunkCRCRequest(RsTurtleChunkCrcRequestItem*);
void handleRecvChunkCRC(RsTurtleChunkCrcItem*);
//------ Functions connecting the turtle router to other components.----------//

163
libretroshare/src/turtle/rsturtleitem.cc
View File

@ -149,7 +149,27 @@ uint32_t RsTurtleFileCrcRequestItem::serial_size()
return s ;
}
uint32_t RsTurtleChunkCrcItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // chunk number
s += 20 ; // check_sum
return s ;
}
uint32_t RsTurtleChunkCrcRequestItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // chunk number
return s ;
}
uint32_t RsTurtleFileCrcItem::serial_size()
{
uint32_t s = 0 ;
@ -202,6 +222,8 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST : return new RsTurtleFileCrcRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC : return new RsTurtleFileCrcItem(data,*size) ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem(data,*size) ;
default:
std::cerr << "Unknown packet type in RsTurtle!" << std::endl ;
@ -322,6 +344,40 @@ bool RsTurtleFileCrcRequestItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
bool RsTurtleChunkCrcRequestItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleChunkCrcRequestItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_number);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleFileCrcItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
@ -363,7 +419,45 @@ bool RsTurtleFileCrcItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
bool RsTurtleChunkCrcItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleChunkCrcRequestItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_number);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[0]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[1]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[2]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[3]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[4]);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleStringSearchRequestItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
@ -644,7 +738,34 @@ RsTurtleFileCrcItem::RsTurtleFileCrcItem(void *data,uint32_t pktsize)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleChunkCrcItem::RsTurtleChunkCrcItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC)
{
setPriorityLevel(QOS_PRIORITY_RS_TURTLE_CHUNK_CRC) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id) ;
ok &= getRawUInt32(data, pktsize, &offset, &chunk_number) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[0]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[1]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[2]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[3]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[4]) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleFileCrcRequestItem::RsTurtleFileCrcRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST)
{
@ -667,7 +788,29 @@ RsTurtleFileCrcRequestItem::RsTurtleFileCrcRequestItem(void *data,uint32_t pktsi
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleChunkCrcRequestItem::RsTurtleChunkCrcRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST)
{
setPriorityLevel(QOS_PRIORITY_RS_TURTLE_CHUNK_CRC_REQUEST) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id) ;
ok &= getRawUInt32(data, pktsize, &offset, &chunk_number) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize)
: RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT)
{
@ -1119,3 +1262,23 @@ std::ostream& RsTurtleFileCrcRequestItem::print(std::ostream& o, uint16_t)
return o ;
}
std::ostream& RsTurtleChunkCrcRequestItem::print(std::ostream& o, uint16_t)
{
o << "Chunk CRC request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk num : " << chunk_number << std::endl ;
return o ;
}
std::ostream& RsTurtleChunkCrcItem::print(std::ostream& o, uint16_t)
{
o << "Chunk CRC request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk num : " << chunk_number << std::endl ;
o << " sha1 sum : " << check_sum.toStdString() << std::endl ;
return o ;
}

44
libretroshare/src/turtle/rsturtleitem.h
View File

@ -22,6 +22,8 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ;
/***********************************************************************************/
/* Basic Turtle Item Class */
@ -281,14 +283,32 @@ class RsTurtleFileCrcRequestItem: public RsTurtleGenericTunnelItem
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
// CompressedChunkMap _map ; // list of chunks for which we need the CRC
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleChunkCrcRequestItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleChunkCrcRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST) { setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC_REQUEST);}
RsTurtleChunkCrcRequestItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return false ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_SERVER ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
uint32_t chunk_number ; // id of the chunk to CRC.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleFileCrcItem: public RsTurtleGenericTunnelItem
{
@ -312,6 +332,26 @@ class RsTurtleFileCrcItem: public RsTurtleGenericTunnelItem
virtual uint32_t serial_size() ;
};
class RsTurtleChunkCrcItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleChunkCrcItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC) { setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC);}
RsTurtleChunkCrcItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return true ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_CLIENT ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
uint32_t chunk_number ;
Sha1CheckSum check_sum ;
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
/***********************************************************************************/
/* Turtle Serialiser class */
/***********************************************************************************/

91
libretroshare/src/util/rsdir.cc
View File

@ -43,6 +43,7 @@
#include <sstream>
#include <fstream>
#include <stdexcept>
#if defined(WIN32) || defined(__CYGWIN__)
#include "util/rsstring.h"
@ -624,18 +625,96 @@ bool RsDirUtil::getFileHash(const std::string& filepath, std::string &hash, uint
SHA1_Final(&sha_buf[0], sha_ctx);
std::ostringstream tmpout;
for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
{
tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
}
hash = tmpout.str();
hash = Sha1CheckSum(sha_buf).toStdString() ;
delete sha_ctx;
fclose(fd);
return true;
}
/* Function to hash, and get details of a file */
Sha1CheckSum RsDirUtil::sha1sum(unsigned char *data, uint32_t size)
{
SHA_CTX sha_ctx ;
if(SHA_DIGEST_LENGTH != 20)
throw std::runtime_error("Warning: can't compute Sha1Sum with sum size != 20") ;
SHA1_Init(&sha_ctx);
while(size > 512)
{
SHA1_Update(&sha_ctx, data, 512);
data = &data[512] ;
size -= 512 ;
}
SHA1_Update(&sha_ctx, data, size);
unsigned char sha_buf[SHA_DIGEST_LENGTH];
SHA1_Final(&sha_buf[0], &sha_ctx);
return Sha1CheckSum(sha_buf) ;
}
bool Sha1CheckSum::operator==(const Sha1CheckSum& s) const
{
for(int i=0;i<5;++i)
if(fourbytes[i] != s.fourbytes[i])
return false ;
return true ;
}
std::string Sha1CheckSum::toStdString() const
{
std::ostringstream tmpout;
for(int i = 0; i < 5; i++)
for(int j = 0; j < 4; j++)
tmpout << std::setw(2) << std::setfill('0') << std::hex << ((fourbytes[i] >> (8*j)) & 0xff ) ;
return tmpout.str() ;
}
Sha1CheckSum::Sha1CheckSum(const uint8_t *buf)
{
int n=0;
for(int i = 0; i < 5; i++)
{
fourbytes[i] = 0 ;
for(int j = 0; j < 4; j++)
fourbytes[i] += buf[n++] << (8*j) ;
}
}
Sha1CheckSum::Sha1CheckSum(const std::string& s)
{
int n=0;
if(s.length() != 40)
throw std::runtime_error("Sha1CheckSum::Sha1CheckSum: can only init from 40 chars hexadecimal string") ;
for(int i = 0; i < 5; ++i)
{
fourbytes[i] = 0 ;
for(int j = 0; j < 4; ++j)
{
for(int k=0;k<2;++k)
{
char b = s[n++] ;
if(b >= 'A' && b <= 'F')
fourbytes[i] += (b-'A'+10) << ((4*(1-k))+8*j) ;
else if(b >= 'a' && b <= 'f')
fourbytes[i] += (b-'a'+10) << ((4*(1-k))+8*j) ;
else if(b >= '0' && b <= '9')
fourbytes[i] += (b-'0') << ((4*(1-k))+8*j) ;
else
throw std::runtime_error("Sha1CheckSum::Sha1CheckSum: can't init from non pure hexadecimal string") ;
}
}
}
}
bool RsDirUtil::renameFile(const std::string& from, const std::string& to)
{
int loops = 0;

5
libretroshare/src/util/rsdir.h
View File

@ -32,9 +32,10 @@
#include <list>
#include <stdint.h>
class CRC32Map ;
class RsThread;
#include <retroshare/rstypes.h>
namespace RsDirUtil {
std::string getTopDir(const std::string&);
@ -67,6 +68,8 @@ bool cleanupDirectory(const std::string& dir, const std::list<std::string> &
bool hashFile(const std::string& filepath, std::string &name, std::string &hash, uint64_t &size);
bool getFileHash(const std::string& filepath,std::string &hash, uint64_t &size, RsThread *thread = NULL);
Sha1CheckSum sha1sum(uint8_t *data,uint32_t size) ;
std::wstring getWideTopDir(std::wstring);
std::wstring getWideRootDir(std::wstring);

4
retroshare-gui/src/gui/FileTransferInfoWidget.cpp
View File

@ -50,6 +50,7 @@ FileTransferInfoWidget::FileTransferInfoWidget(QWidget * /*parent*/, Qt::WFlags
downloadedPixmap.load(":images/graph-downloaded.png");
downloadingPixmap.load(":images/graph-downloading.png");
notDownloadPixmap.load(":images/graph-notdownload.png");
checkingPixmap.load(":images/graph-checking.png");
setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding);
}
@ -141,6 +142,9 @@ void FileTransferInfoWidget::draw(const FileInfo& nfo,const FileChunksInfo& info
case FileChunksInfo::CHUNK_ACTIVE: painter->drawPixmap(target, downloadingPixmap, source);
break ;
case FileChunksInfo::CHUNK_CHECKING: painter->drawPixmap(target, checkingPixmap, source);
break ;
case FileChunksInfo::CHUNK_OUTSTANDING: painter->drawPixmap(target, notDownloadPixmap, source);
break ;
default: ;

1
retroshare-gui/src/gui/FileTransferInfoWidget.h
View File

@ -55,6 +55,7 @@ private:
QPixmap downloadedPixmap;
QPixmap downloadingPixmap;
QPixmap notDownloadPixmap;
QPixmap checkingPixmap;
std::string _file_hash ;
};

6
retroshare-gui/src/gui/TransfersDialog.cpp
View File

@ -333,8 +333,14 @@ TransfersDialog::TransfersDialog(QWidget *parent)
resumeAct = new QAction(QIcon(IMAGE_RESUME), tr("Resume"), this);
connect(resumeAct, SIGNAL(triggered()), this, SLOT(resumeFileTransfer()));
#ifdef USE_NEW_CHUNK_CHECKING_CODE
// *********WARNING**********
// csoler: this has been suspended because it needs the file transfer to consider a file as complete only if all chunks are
// verified by hash. As users are goign to slowly switch to new checking code, this will not be readily available.
//
forceCheckAct = new QAction(QIcon(IMAGE_CANCEL), tr( "Force Check" ), this );
connect( forceCheckAct , SIGNAL( triggered() ), this, SLOT( forceCheck() ) );
#endif
cancelAct = new QAction(QIcon(IMAGE_CANCEL), tr( "Cancel" ), this );
connect( cancelAct , SIGNAL( triggered() ), this, SLOT( cancel() ) );

29
retroshare-gui/src/gui/TransfersDialog.ui
View File

@ -744,8 +744,8 @@ p, li { white-space: pre-wrap; }
<rect>
<x>0</x>
<y>0</y>
<width>596</width>
<height>101</height>
<width>598</width>
<height>99</height>
</rect>
</property>
</widget>
@ -798,7 +798,7 @@ p, li { white-space: pre-wrap; }
</item>
</layout>
</item>
<item row="0" column="2">
<item row="0" column="3">
<layout class="QGridLayout">
<item row="0" column="0">
<widget class="QLabel" name="label_7">
@ -819,7 +819,7 @@ p, li { white-space: pre-wrap; }
</item>
</layout>
</item>
<item row="0" column="3">
<item row="0" column="4">
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
@ -832,6 +832,27 @@ p, li { white-space: pre-wrap; }
</property>
</spacer>
</item>
<item row="0" column="2">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QLabel" name="label_10">
<property name="text">
<string/>
</property>
<property name="pixmap">
<pixmap resource="images.qrc">:/images/graph-checking.png</pixmap>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_9">
<property name="text">
<string>Needs checking</string>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</item>
</layout>

1
retroshare-gui/src/gui/images.qrc
View File

@ -107,6 +107,7 @@
<file>images/graph-downloaded.png</file>
<file>images/graph-downloading.png</file>
<file>images/graph-notdownload.png</file>
<file>images/graph-checking.png</file>
<file>images/add-friend24.png</file>
<file>images/add-share24.png</file>
<file>images/add_24x24.png</file>

BIN
retroshare-gui/src/gui/images/graph-checking.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 447 B