Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

First implementation of per-chunk CRC32 check. This is triggered

Browse Source 
- by the right-click+Force Check on files.
- when a global hash on a downloaded file does not match the announced hash.

When a CRC map check is ordered, the CRC map is requested to one of the sources for the current file download.
When received, all downloaded chunks are checked w.r.t the reference CRC and marked as not done if the CRCs do not match.
The exchange of CRC32 map and requests has been tested, as well as CRC map checking during download (force check).

To be implemented soon:
    - caching of CRC32 maps (although these are fast to compute)
    - CRC32 map packets for normal downloads. For now these work only for turtle tunnels.
    - handling of errors if the CRC never comes. For now, the download will stay stuck in "Checking..." mode.

So, don't play too much with the force check feature for now...





git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@3310 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2010-07-21 23:14:10 +00:00
parent db034b026f
commit 8bfc74485a
27 changed files with 1012 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
libretroshare/src/ft/ftchunkmap.cc
View File

@ -419,17 +419,28 @@ void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
#endif
}
uint32_t ChunkMap::getNumberOfChunks(uint64_t size)
{
uint64_t n = size/(uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE ;
if(size % (uint64_t)CHUNKMAP_FIXED_CHUNK_SIZE != 0)
++n ;
uint32_t value = n & 0xffffffffull ;
if((uint64_t)value != n)
{
std::cerr << "ERROR: number of chunks is a totally absurd value. File size is " << size << ", chunks are " << n << "!!" << std::endl ;
return 0 ;
}
return value ;
}
void ChunkMap::buildPlainMap(uint64_t size, CompressedChunkMap& map)
{
uint32_t chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE) ; // 1MB chunks
uint64_t n = size/(uint64_t)chunk_size ;
if(size % (uint64_t)chunk_size != 0)
++n ;
uint32_t n = getNumberOfChunks(size) ;
map = CompressedChunkMap(n,~uint32_t(0)) ;
}

4
libretroshare/src/ft/ftchunkmap.h
View File

@ -144,6 +144,9 @@ class ChunkMap
//
static void buildPlainMap(uint64_t size,CompressedChunkMap& map) ;
/// Computes the number of chunks for the given file size.
static uint32_t getNumberOfChunks(uint64_t size) ;
/// This function is used by the parent ftFileProvider to know whether the chunk can be sent or not.
bool isChunkAvailable(uint64_t offset, uint32_t chunk_size) const ;
@ -162,6 +165,7 @@ class ChunkMap
bool isComplete() const { return _file_is_complete ; }
void getChunksInfo(FileChunksInfo& info) const ;
protected:
/// handles what size the last chunk has.
uint32_t sizeOfChunk(uint32_t chunk_number) const ;

25
libretroshare/src/ft/ftcontroller.cc
View File

@ -63,6 +63,7 @@
* #define CONTROL_DEBUG 1
* #define DEBUG_DWLQUEUE 1
*****/
//#define CONTROL_DEBUG 1
static const uint32_t SAVE_TRANSFERS_DELAY = 61 ; // save transfer progress every 61 seconds.
static const uint32_t INACTIVE_CHUNKS_CHECK_DELAY = 60 ; // time after which an inactive chunk is released
@ -980,10 +981,12 @@ bool ftController::alreadyHaveFile(const std::string& hash)
return false ;
}
bool ftController::FileRequest(std::string fname, std::string hash,
uint64_t size, std::string dest, uint32_t flags,
std::list<std::string> &srcIds)
bool ftController::FileRequest(const std::string& fname, const std::string& hash,
uint64_t size, const std::string& dest, uint32_t flags,
const std::list<std::string> &_srcIds)
{
std::list<std::string> srcIds(_srcIds) ;
/* check if we have the file */
if(alreadyHaveFile(hash))
@ -1047,8 +1050,8 @@ bool ftController::FileRequest(std::string fname, std::string hash,
}
FileInfo info;
std::list<std::string>::iterator it;
std::list<TransferInfo>::iterator pit;
std::list<std::string>::const_iterator it;
std::list<TransferInfo>::const_iterator pit;
#ifdef CONTROL_DEBUG
std::cerr << "ftController::FileRequest(" << fname << ",";
@ -1077,7 +1080,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
std::map<std::string, ftFileControl*>::iterator dit = mDownloads.find(hash);
std::map<std::string, ftFileControl*>::const_iterator dit = mDownloads.find(hash);
if (dit != mDownloads.end())
{
@ -1312,7 +1315,7 @@ bool ftController::setChunkStrategy(const std::string& hash,FileChunksInfo::Chun
return true ;
}
bool ftController::FileCancel(std::string hash)
bool ftController::FileCancel(const std::string& hash)
{
rsTurtle->stopMonitoringFileTunnels(hash) ;
@ -1420,10 +1423,18 @@ bool ftController::FileControl(const std::string& hash, uint32_t flags)
mit->second->mState = ftFileControl::PAUSED ;
std::cerr << "setting state to " << ftFileControl::PAUSED << std::endl ;
break;
case RS_FILE_CTRL_START:
mit->second->mState = ftFileControl::DOWNLOADING ;
std::cerr << "setting state to " << ftFileControl::DOWNLOADING << std::endl ;
break;
case RS_FILE_CTRL_FORCE_CHECK:
// mit->second->mState = ftFileControl::CHECKING_HASH ;
mit->second->mTransfer->forceCheck();
std::cerr << "setting state to " << ftFileControl::CHECKING_HASH << std::endl ;
break;
default:
return false;
}

14
libretroshare/src/ft/ftcontroller.h
View File

@ -99,9 +99,9 @@ class ftFileControl
class ftPendingRequest
{
public:
ftPendingRequest(std::string fname, std::string hash,
uint64_t size, std::string dest, uint32_t flags,
std::list<std::string> &srcIds)
ftPendingRequest(const std::string& fname, const std::string& hash,
uint64_t size, const std::string& dest, uint32_t flags,
const std::list<std::string> &srcIds)
: mName(fname), mHash(hash), mSize(size),
mDest(dest), mFlags(flags),mSrcIds(srcIds) { return; }
@ -137,9 +137,9 @@ class ftController: public CacheTransfer, public RsThread, public pqiMonitor, pu
/********************** Controller Access **********************/
/***************************************************************/
bool FileRequest(std::string fname, std::string hash,
uint64_t size, std::string dest, uint32_t flags,
std::list<std::string> &sourceIds);
bool FileRequest(const std::string& fname, const std::string& hash,
uint64_t size, const std::string& dest, uint32_t flags,
const std::list<std::string> &sourceIds);
/// Do we already have this file, either in download or in file lists ?
bool alreadyHaveFile(const std::string& hash) ;
@ -150,7 +150,7 @@ class ftController: public CacheTransfer, public RsThread, public pqiMonitor, pu
uint32_t freeDiskSpaceLimit() const ;
void setFreeDiskSpaceLimit(uint32_t size_in_mb) ;
bool FileCancel(std::string hash);
bool FileCancel(const std::string& hash);
bool FileControl(const std::string& hash, uint32_t flags);
bool FileClearCompleted();
bool FlagFileComplete(std::string hash);

5
libretroshare/src/ft/ftdata.h
View File

@ -41,6 +41,7 @@
/*************** SEND INTERFACE *******************/
class CompressedChunkMap ;
class CRC32Map ;
class ftDataSend
{
@ -57,6 +58,10 @@ class ftDataSend
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk map
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) = 0;
/// Send a request for a chunk crc map
virtual bool sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk crc map
virtual bool sendCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) = 0;
};

96
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -53,7 +53,8 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
const uint32_t FT_DATA = 0x0001; // data cuhnk to be stored
const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map reuqest to be treated by server
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map request to be treated by server
const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
@ -250,7 +251,19 @@ bool ftDataMultiplex::recvChunkMapRequest(const std::string& peerId, const std::
return true;
}
bool ftDataMultiplex::recvCRCMapRequest(const std::string& peerId, const std::string& hash/*,const CompressedChunkMap& map*/)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
std::cerr << std::endl;
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(ftRequest(FT_CRC32MAP_REQ,peerId,hash,0,0,0,NULL));
return true;
}
/*********** BACKGROUND THREAD OPERATIONS ***********/
bool ftDataMultiplex::workQueued()
@ -326,6 +339,14 @@ bool ftDataMultiplex::doWork()
handleRecvServerChunkMapRequest(req.mPeerId,req.mHash) ;
break ;
case FT_CRC32MAP_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CRC32_MAP_REQ";
std::cerr << std::endl;
#endif
handleRecvCRC32MapRequest(req.mPeerId,req.mHash,CompressedChunkMap()) ;
break ;
default:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
@ -362,6 +383,27 @@ bool ftDataMultiplex::doWork()
return true;
}
bool ftDataMultiplex::recvCRCMap(const std::string& peerId, const std::string& hash,const CRC32Map& crc_map)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
std::cerr << "ftDataMultiplex::recvCRCMap() ERROR: No matching Client for CRC32map. This is an error. " << hash << " !" << std::endl;
/* error */
return false;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvCRCMap() Passing crc map of file " << hash << ", to FT Module" << std::endl;
#endif
(it->second).mModule->addCRC32Map(crc_map);
return true ;
}
// A chunk map has arrived. It can be two different situations:
// - an uploader has sent his chunk map, so we need to store it in the corresponding ftFileProvider
// - a source for a download has sent his chunk map, so we need to send it to the corresponding ftFileCreator.
@ -412,6 +454,53 @@ bool ftDataMultiplex::recvChunkMap(const std::string& peerId, const std::string&
return false;
}
bool ftDataMultiplex::handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash,const CompressedChunkMap&)
{
std::map<std::string, ftFileProvider *>::iterator it ;
bool found = true ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end())
found = false ;
}
if(!found)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
#endif
if(!handleSearchRequest(peerId,hash))
return false ;
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
std::cerr << std::endl;
#endif
}
CRC32Map cmap ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end()) // handleSearchRequest should have filled mServers[hash], but we have been off-mutex since,
return false ; // so it's safer to check again.
else if(!it->second->getCRC32Map(cmap))
return false ;
}
std::cerr << "File CRC32 map was successfully computed. Sending it." << std::endl ;
mDataSend->sendCRC32Map(peerId,hash,cmap);
return true ;
}
bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
@ -639,7 +728,10 @@ bool ftDataMultiplex::sendChunkMapRequest(const std::string& peer_id,const std::
{
return mDataSend->sendChunkMapRequest(peer_id,hash);
}
bool ftDataMultiplex::sendCRCMapRequest(const std::string& peer_id,const std::string& hash,const CompressedChunkMap&)
{
return mDataSend->sendCRC32MapRequest(peer_id,hash);
}
void ftDataMultiplex::deleteUnusedServers()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

13
libretroshare/src/ft/ftdatamultiplex.h
View File

@ -110,18 +110,24 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
/* Server/client Send */
bool sendChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/* Client Send */
bool sendCRCMapRequest(const std::string& peerId, const std::string& hash,const CompressedChunkMap& chnks) ;
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
/* Server Recv */
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
/// Receive a request for a chunk map
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ;
/// Receive a chunk map
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
/* Server Recv */
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
/// Receive a CRC map
virtual bool recvCRCMap(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) ;
/// Receive a CRC map request
virtual bool recvCRCMapRequest(const std::string& peer_id,const std::string& hash) ;
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
// map is too old. This supposes that the caller will ask again in a few seconds.
@ -142,6 +148,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool handleSearchRequest(const std::string& peerId, const std::string& hash);
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash, const CompressedChunkMap& map) ;
/* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);

50
libretroshare/src/ft/ftfilecreator.cc
View File

@ -472,13 +472,55 @@ bool ftFileCreator::hashReceivedData(std::string& hash)
// long time. Therefore, we must pay attention not to call this function
// at a time file_name nor hash can be modified, which is easy.
//
if(!finished())
return false ;
return RsDirUtil::hashFile(file_name,hash) ;
}
#include <stdexcept>
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref)
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks)
{
// still to be implemented.
throw std::runtime_error(std::string("Unimplemented function ") + __PRETTY_FUNCTION__) ;
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
CompressedChunkMap map ;
chunkMap.getAvailabilityMap(map) ;
uint32_t nb_chunks = ref.size() ;
static const uint32_t chunk_size = ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE ;
std::cerr << "ftFileCreator::crossCheckChunkMap(): comparing chunks..." << std::endl;
if(!locked_initializeFileAttrs() )
return false ;
unsigned char *buff = new unsigned char[chunk_size] ;
incomplete_chunks = 0 ;
bad_chunks = 0 ;
uint32_t len = 0 ;
for(uint32_t i=0;i<nb_chunks;++i)
if(map[i])
if(fseek(fd,(uint64_t)i * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
{
if( RsDirUtil::rs_CRC32(buff,len) != ref[i])
{
++bad_chunks ;
++incomplete_chunks ;
map.reset(i) ;
}
}
else
return false ;
else
++incomplete_chunks ;
delete[] buff ;
if(bad_chunks > 0)
chunkMap.setAvailabilityMap(map) ;
}
closeFile() ;
return true ;
}

14
libretroshare/src/ft/ftfilecreator.h
View File

@ -64,13 +64,11 @@ class ftFileCreator: public ftFileProvider
bool hashReceivedData(std::string& hash) ;
// Computes a CRC32 map of all chunks, for comparison with reference, and re-starting invalid chunks.
//
bool CRC32ReceivedData(CRC32Map& crc_map) ;
// Checks the CRC32 of all chunks against the given CRC32 map. Re-flag the bad chunks as being void.
// bad_chunks: count of achieved chunks that don't match the CRC
// incomplete_chunks: count of any bad or not yet downloaded chunk
//
bool crossCheckChunkMap(const CRC32Map& crc_map) ;
bool crossCheckChunkMap(const CRC32Map& ref,uint32_t& bad_chunks,uint32_t& incomplete_chunks) ;
/*
* creation functions for FileCreator
*/
@ -107,6 +105,12 @@ class ftFileCreator: public ftFileProvider
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
void setAvailabilityMap(const CompressedChunkMap& cmap) ;
// Provides a complete per-chunk CRC32 map to client who want to check their data.
// This is overloads ftFileProvider, but returns false, because we can't ensure that unchecked chunks
// will provide a CRC32 that is faithful to the original hash.
//
virtual bool getCRC32Map(CRC32Map& crc_map) { return false ; }
// This is called when receiving the availability map from a source peer, for the file being handled.
//
void setSourceMap(const std::string& peer_id,const CompressedChunkMap& map) ;

13
libretroshare/src/ft/ftfileprovider.cc
View File

@ -292,3 +292,16 @@ int ftFileProvider::initializeFileAttrs()
return 1;
}
bool ftFileProvider::getCRC32Map(CRC32Map& crc_map)
{
if(!initializeFileAttrs())
{
std::cerr << "ftFileProvider::getCRC32Map(...): ERROR: can't initialize file !" << std::endl ;
return false ;
}
std::cerr << "ftFileProvider::getClientMap(): computing CRC32 map for file " << file_name << " (" << hash << ")" << std::endl ;
return RsDirUtil::crc32File(fd,mSize,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,crc_map) ;
}

6
libretroshare/src/ft/ftfileprovider.h
View File

@ -55,6 +55,12 @@ class ftFileProvider
//
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
// Provides a complete per-chunk CRC32 map to client who want to check their data.
// This is derived in ftFileCreator, but returns false, because we can't ensure that unchecked chunks
// will provide a CRC32 that is faithful to the original hash.
//
virtual bool getCRC32Map(CRC32Map& crc_map) ;
// a ftFileProvider feeds a distant peer. To display what the peers already has, we need to store/read this info.
void getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old) ;
void setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap) ;

33
libretroshare/src/ft/ftserver.cc
View File

@ -281,7 +281,7 @@ bool ftServer::checkHash(const std::string& hash,std::string& error_string)
return true ;
}
bool ftServer::FileRequest(std::string fname, std::string hash, uint64_t size, std::string dest, uint32_t flags, std::list<std::string> srcIds)
bool ftServer::FileRequest(const std::string& fname, const std::string& hash, uint64_t size, const std::string& dest, uint32_t flags, const std::list<std::string>& srcIds)
{
std::string error_string ;
@ -319,8 +319,7 @@ FileChunksInfo::ChunkStrategy ftServer::defaultChunkStrategy()
{
return mFtController->defaultChunkStrategy() ;
}
bool ftServer::FileCancel(std::string hash)
bool ftServer::FileCancel(const std::string& hash)
{
// Remove from both queue and ftController, by default.
//
@ -329,7 +328,7 @@ bool ftServer::FileCancel(std::string hash)
return true ;
}
bool ftServer::FileControl(std::string hash, uint32_t flags)
bool ftServer::FileControl(const std::string& hash, uint32_t flags)
{
return mFtController->FileControl(hash, flags);
}
@ -743,6 +742,8 @@ bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string&
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMapRequest(peerId,hash) ;
else
std::cerr << "ftServer: Warning: not sending chunk map request to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
@ -753,12 +754,36 @@ bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,co
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMap(peerId,hash,map) ;
else
std::cerr << "ftServer: Warning: not sending chunk map to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendCRC32MapRequest(const std::string& peerId,const std::string& hash)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendCRC32MapRequest(peerId,hash) ;
else
std::cerr << "ftServer: Warning: not sending CRC map request to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendCRC32Map(const std::string& peerId,const std::string& hash,const CRC32Map& map)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendCRC32Map(peerId,hash,map) ;
else
std::cerr << "ftServer: Warning: not sending CRC map to peer " << peerId << ", because it's not a turtle tunnel." << std::endl ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */
//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */

10
libretroshare/src/ft/ftserver.h
View File

@ -120,10 +120,9 @@ ftController *getController() const { return mFtController ; }
/***
* Control of Downloads
***/
virtual bool FileRequest(std::string fname, std::string hash, uint64_t size,
std::string dest, uint32_t flags, std::list<std::string> srcIds);
virtual bool FileCancel(std::string hash);
virtual bool FileControl(std::string hash, uint32_t flags);
virtual bool FileRequest(const std::string& fname, const std::string& hash, uint64_t size, const std::string& dest, uint32_t flags, const std::list<std::string>& srcIds);
virtual bool FileCancel(const std::string& hash);
virtual bool FileControl(const std::string& hash, uint32_t flags);
virtual bool FileClearCompleted();
virtual bool setChunkStrategy(const std::string& hash,FileChunksInfo::ChunkStrategy s) ;
virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ;
@ -213,6 +212,9 @@ virtual bool sendData(const std::string& peerId, const std::string& hash, uint64
virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) ;
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) ;
virtual bool sendCRC32MapRequest(const std::string&, const std::string&) ;
virtual bool sendCRC32Map(const std::string&, const std::string&, const CRC32Map&) ;
/*************** Internal Transfer Fns *************************/

169
libretroshare/src/ft/fttransfermodule.cc
View File

@ -28,6 +28,7 @@
*****/
//#define FT_DEBUG 1
#include <rsiface/rsturtle.h>
#include "fttransfermodule.h"
/*************************************************************************
@ -52,6 +53,7 @@ const uint32_t FT_TM_MAX_RESETS = 5;
const uint32_t FT_TM_MINIMUM_CHUNK = 128; /* ie 1/8Kb / sec */
const uint32_t FT_TM_RESTART_DOWNLOAD = 20; /* 20 seconds */
const uint32_t FT_TM_DOWNLOAD_TIMEOUT = 10; /* 10 seconds */
const uint32_t FT_TM_CRC_MAP_MAX_WAIT_PER_GIG = 10; /* 10 seconds */
const double FT_TM_MAX_INCREASE = 1.00;
const double FT_TM_MIN_INCREASE = -0.10;
@ -59,6 +61,11 @@ const int32_t FT_TM_FAST_RTT = 1.0;
const int32_t FT_TM_STD_RTT = 5.0;
const int32_t FT_TM_SLOW_RTT = 9.0;
const uint32_t FT_TM_CRC_MAP_STATE_DONT_HAVE = 0 ;
const uint32_t FT_TM_CRC_MAP_STATE_NOCHECK = 1 ;
const uint32_t FT_TM_CRC_MAP_STATE_ASKED = 2 ;
const uint32_t FT_TM_CRC_MAP_STATE_HAVE = 3 ;
#define FT_TM_FLAG_DOWNLOADING 0
#define FT_TM_FLAG_CANCELED 1
#define FT_TM_FLAG_COMPLETE 2
@ -505,7 +512,7 @@ bool ftTransferModule::isCheckingHash()
#ifdef FT_DEBUG
std::cerr << "isCheckingHash(): mFlag=" << mFlag << std::endl;
#endif
return mFlag == FT_TM_FLAG_CHECKING ;
return mFlag == FT_TM_FLAG_CHECKING || mFlag == FT_TM_FLAG_CHUNK_CRC;
}
class HashThread: public RsThread
@ -580,9 +587,11 @@ bool ftTransferModule::checkFile()
}
delete _hash_thread ;
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map. But for now, cancel file transfer.
mFileStatus.stat = ftFileStatus::PQIFILE_FAIL_CANCEL ;
}
forceCheck() ;
return true ;
cancelFileTransferUpward() ;
std::cerr << "(EE) ftTransferModule::checkFile(): File verification failed for hash " << mHash << "! Asking for CRC map. mFlag=4. For now: cancelling file transfer." << std::endl ;
//askForCRCMap() ;
@ -590,28 +599,172 @@ bool ftTransferModule::checkFile()
return false ;
}
void ftTransferModule::forceCheck()
{
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::forceCheck(): setting flags to force check." << std::endl ;
#endif
mFlag = FT_TM_FLAG_CHUNK_CRC ; // Ask for CRC map. But for now, cancel file transfer.
// setup flags for CRC state machine to work properly
_crcmap_state = FT_TM_CRC_MAP_STATE_DONT_HAVE ;
_crcmap_last_asked_time = 0 ;
_crcmap_last_source_id = -1 ;
}
bool ftTransferModule::checkCRC()
{
// We have a finite state machine here.
//
// The states are, for each chunk, and what should be done:
// DONT_HAVE
// -> ask for the chunk CRC
// ASKED
// -> do nothing
// RECEIVED
// -> check the chunk
// CHECKED
// -> do nothing
//
// CRCs are asked by group of CRC_REQUEST_MAX_SIZE chunks at a time. The response may contain a different
// number of CRCs, depending on the availability.
//
// Server side:
// - Only complete files can compute CRCs, as the CRCs of downloaded chunks in an unchecked file are un-verified.
// - CRCs of files are cached, so that they don't get computed twice.
//
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ;
#endif
if(_crc32map.empty())
return false ;
// Loop over chunks. Collect the ones to ask for, and hash the ones received.
// If we have
//
time_t now = time(NULL) ;
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
switch(_crcmap_state)
{
case FT_TM_CRC_MAP_STATE_NOCHECK:
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ;
#endif
break ;
case FT_TM_CRC_MAP_STATE_ASKED:
std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: last time is " << _crcmap_last_asked_time << std::endl ;
std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: now is " << now << std::endl ;
std::cerr << "Limit is " << (uint64_t)_crcmap_last_asked_time + (uint64_t)(FT_TM_CRC_MAP_MAX_WAIT_PER_GIG * (1+mSize/float(1024ull*1024ull*1024ull))) << std::endl ;
if( (uint64_t)_crcmap_last_asked_time + (uint64_t)(FT_TM_CRC_MAP_MAX_WAIT_PER_GIG * (1+mSize/float(1024ull*1024ull*1024ull))) > (uint64_t)now)
{
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ;
#endif
break ;
}
#ifdef FT_DEBUG
else
std::cerr << "ftTransferModule::checkCRC(): state is ASKED, but time is too long. Asking again." << std::endl ;
#endif
case FT_TM_CRC_MAP_STATE_DONT_HAVE:
{
// Ask the ones we should ask for. We use a very coarse strategy now: we
// send the request to a random source. We'll make this more sensible
// later.
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): state is DONT_HAVE. Selecting a source for asking a CRC map." << std::endl ;
#endif
// if(_crcmap_last_source_id < 0)
// _crcmap_last_source_id=lrand48() ;
// _crcmap_last_source_id = (_crcmap_last_source_id+1)%mFileSources.size() ;
int n=0 ;
bool found = false ;
std::map<std::string,peerInfo>::const_iterator mit ;
for(mit = mFileSources.begin();mit != mFileSources.end();++mit)
if(rsTurtle->isTurtlePeer(mit->first))
{
found=true ;
break ;
}
if(found)
{
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): sending CRC map request to source " << mit->first << std::endl ;
#endif
_crcmap_last_asked_time = now ;
mMultiplexor->sendCRCMapRequest(mit->first,mHash,CompressedChunkMap());
_crcmap_state = FT_TM_CRC_MAP_STATE_ASKED ;
}
else
std::cerr << "ERROR: No file source to ask a chunkmap to!" << std::endl ;
}
break ;
case FT_TM_CRC_MAP_STATE_HAVE:
{
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ;
#endif
// The CRCmap is complete. Let's cross check it with existing chunks in ftFileCreator !
//
uint32_t bad_chunks ;
uint32_t incomplete_chunks ;
mFileCreator->crossCheckChunkMap(_crc32map) ;
if(!mFileCreator->crossCheckChunkMap(_crcmap,bad_chunks,incomplete_chunks))
{
std::cerr << "ftTransferModule::checkCRC(): could not check CRC chunks!" << std::endl ;
return false;
}
// go back to download stage, if not all chunks are correct.
//
if(bad_chunks > 0)
{
mFlag = FT_TM_FLAG_DOWNLOADING ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): Done. Restarting download." << std::endl ;
std::cerr << "ftTransferModule::checkCRC(): Done. " << bad_chunks << " bad chunks found. Restarting download for these chunks only." << std::endl ;
#endif
}
else if(incomplete_chunks > 0)
{
mFlag = FT_TM_FLAG_DOWNLOADING ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): Done. all chunks ok. Continuing download for remaining chunks." << std::endl ;
#endif
}
else
{
mFlag = FT_TM_FLAG_COMPLETE ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): Done. CRC check is ok, file is complete." << std::endl ;
#endif
}
_crcmap_state = FT_TM_CRC_MAP_STATE_NOCHECK ;
}
}
return true ;
}
void ftTransferModule::addCRC32Map(const CRC32Map& crc_map)
{
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
// Note: for now, we only send complete CRC32 maps, so the crc_map is always complete. When we
// send partial CRC maps, we will have to check them for completness.
//
_crcmap_state = FT_TM_CRC_MAP_STATE_HAVE ;
_crcmap = crc_map ;
}
void ftTransferModule::adjustSpeed()
{

7
libretroshare/src/ft/fttransfermodule.h
View File

@ -144,6 +144,8 @@ public:
bool cancelFileTransferUpward();
bool completeFileTransfer();
bool isCheckingHash() ;
void forceCheck() ;
void addCRC32Map(const CRC32Map& map) ;
//interface to multiplex module
bool recvFileData(std::string peerId, uint64_t offset,
@ -189,7 +191,10 @@ private:
double desiredRate;
double actualRate;
CRC32Map _crc32map ;
CRC32Map _crcmap ;
uint32_t _crcmap_state ;
time_t _crcmap_last_asked_time ;
int _crcmap_last_source_id ;
ftFileStatus mFileStatus; //used for pause/resume file transfer

6
libretroshare/src/rsiface/rsfiles.h
View File

@ -111,14 +111,14 @@ class RsFiles
***/
/// Returns false is we already have the file. Otherwise, initiates the dl and returns true.
virtual bool FileRequest(std::string fname, std::string hash, uint64_t size, std::string dest, uint32_t flags, std::list<std::string> srcIds) = 0;
virtual bool FileCancel(std::string hash) = 0;
virtual bool FileRequest(const std::string& fname, const std::string& hash, uint64_t size, const std::string& dest, uint32_t flags, const std::list<std::string>& srcIds) = 0;
virtual bool FileCancel(const std::string& hash) = 0;
virtual bool setChunkStrategy(const std::string& hash,FileChunksInfo::ChunkStrategy) = 0;
virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) = 0;
virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() = 0;
virtual uint32_t freeDiskSpaceLimit() const =0;
virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) =0;
virtual bool FileControl(std::string hash, uint32_t flags) = 0;
virtual bool FileControl(const std::string& hash, uint32_t flags) = 0;
virtual bool FileClearCompleted() = 0;
/***

15
libretroshare/src/rsiface/rstypes.h
View File

@ -344,16 +344,25 @@ class CRC32Map
public:
// Build from a file.
//
CRC32Map(const std::string& fname,uint32_t chunk_size) ;
CRC32Map(uint64_t file_size,uint32_t chunk_size)
: _crcs( file_size/chunk_size + ( (file_size%chunk_size)>0)), _map(file_size/chunk_size + ( (file_size%chunk_size)>0),0)
{
}
CRC32Map() {}
// Compares two maps and returns the valid chunks in a compressed chunk map.
//
friend CompressedChunkMap compare(const CRC32Map& crc1,const CRC32Map& crc2) ;
bool empty() const { return _map.empty() ; }
void set(uint32_t i,uint32_t val) { _crcs[i] = val ; _map.set(i) ; }
uint32_t operator[](int i) const { return _crcs[i] ; }
uint32_t size() const { return _crcs.size() ; }
private:
std::vector<uint32_t> _map ;
std::vector<uint32_t> _crcs;
CompressedChunkMap _map ;
friend class RsTurtleFileCrcItem ;
};
/* class which encapsulates download details */

7
libretroshare/src/tests/ft/Makefile
View File

@ -7,11 +7,11 @@ RS_TOP_DIR = ../..
include $(RS_TOP_DIR)/tests/scripts/config.mk
###############################################################
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o fttransfermoduletest.o
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o fttransfermoduletest.o ftcrc32test.o
#ftserver1test.o ftserver2test.o ftserver3test.o
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest fttransfermoduletest
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest fttransfermoduletest ftcrc32test
#ftserver1test ftserver2test fttransfermoduletest ftserver3test
all: tests
@ -40,6 +40,9 @@ ftserver2test : ftserver2test.o
ftserver3test : ftserver3test.o
$(CC) $(CFLAGS) -o ftserver3test ftserver3test.o $(LIBS)
ftcrc32test : ftcrc32test.o
$(CC) $(CFLAGS) -O0 -g -o ftcrc32test ftcrc32test.o $(LIBS)
###############################################################
include $(RS_TOP_DIR)/tests/scripts/rules.mk
###############################################################

117
libretroshare/src/tests/ft/ftcrc32test.cc Normal file
View File

@ -0,0 +1,117 @@
/*
* libretroshare/src/ft: ftcrc32test.cc
*
* File Transfer for RetroShare.
*
* Copyright 2008 by Cyril Soler
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include <iostream>
#include <rsiface/rstypes.h>
#include <util/rsdir.h>
#include <ft/ftchunkmap.h>
#ifdef WIN32
#include "util/rswin.h"
#endif
#include "util/rsdir.h"
void usage(char *name)
{
std::cerr << "Computes a CRC32 map of a file." << std::endl;
std::cerr << "Usage: " << name << " -h <path>" << std::endl;
}
int main(int argc, char **argv)
{
int c;
uint32_t period = 1;
uint32_t dPeriod = 600; /* default 10 minutes */
std::list<std::string> hashList;
while(-1 != (c = getopt(argc, argv, "f:h:")))
{
switch (c)
{
case 'f':
hashList.push_back(std::string(optarg));
break;
default:
usage(argv[0]);
break;
}
}
uint32_t flags = 0;
for(std::list<std::string>::const_iterator it(hashList.begin()); it != hashList.end(); it++)
{
std::cerr << "Hashing file :" << *it << std::endl ;
std::string hash ;
RsDirUtil::hashFile( *it,hash) ;
std::cerr << "Hash = " << hash << std::endl;
FILE *f = fopen( (*it).c_str(),"rb" ) ;
if(f == NULL)
{
std::cerr << "Could not open this file! Sorry." << std::endl ;
return 0 ;
}
CRC32Map crc_map ;
if(fseek(f,0,SEEK_END))
{
std::cerr << "Could not fseek to end of this file! Sorry." << std::endl ;
fclose(f) ;
return 0 ;
}
uint64_t size = ftell(f) ;
if(fseek(f,0,SEEK_SET))
{
std::cerr << "Could not fseek to beginning of this file! Sorry." << std::endl ;
fclose(f) ;
return 0 ;
}
std::cerr << "File size:" << size << std::endl ;
RsDirUtil::crc32File(f,size,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,crc_map) ;
fclose(f) ;
std::cerr << "Got this CRC map: "<< std::endl ;
for(uint32_t i=0;i<crc_map.size();++i)
std::cerr << (void*)crc_map[i] ;
std::cerr << std::endl;
return 1 ;
}
}

192
libretroshare/src/turtle/p3turtle.cc
View File

@ -23,6 +23,8 @@
*
*/
//#define P3TURTLE_DEBUG
#include <stdexcept>
#include <stdlib.h>
#include <assert.h>
@ -856,8 +858,14 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST: handleRecvFileMapRequest(dynamic_cast<RsTurtleFileMapRequestItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_FILE_CRC : handleRecvFileCRC32Map(dynamic_cast<RsTurtleFileCrcItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST: handleRecvFileCRC32MapRequest(dynamic_cast<RsTurtleFileCrcRequestItem *>(item)) ;
break ;
default:
std::cerr << "WARNING: Unknown packet type received: id=" << (void*)(item->PacketSubType()) << ". Is somebody trying to DOS you ?" << std::endl ;
std::cerr << "WARNING: Unknown packet type received: id=" << (void*)(item->PacketSubType()) << ". Is somebody trying to poison you ?" << std::endl ;
#ifdef P3TURTLE_DEBUG
exit(-1) ;
#endif
@ -1043,6 +1051,79 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
_ft_server->getMultiplexer()->recvChunkMap(vpid,hash,item->compressed_map,item->direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
}
void p3turtle::handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map Request item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file crc request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvCRCMapRequest(vpid,hash) ;
}
void p3turtle::handleRecvFileCRC32Map(RsTurtleFileCrcItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: got file CRC32 map with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file map." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// We should check that there is no backward call to the turtle router!
//
vpid = tunnel.vpid ;
hash = tunnel.hash ;
}
_ft_server->getMultiplexer()->recvCRCMap(vpid,hash,item->crc_map) ;
}
// Send a data request into the correct tunnel for the given file hash
void p3turtle::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t, uint64_t offset, uint32_t chunksize)
{
@ -1210,55 +1291,66 @@ void p3turtle::sendChunkMap(const std::string& peerId,const std::string& hash,co
sendItem(item) ;
}
// This function is actually not needed: Search request to the turtle router are:
// - distant search requests, handled by the router
// - search requests over files being downloaded, handled by rsFiles !!
//
//bool p3turtle::search(std::string hash, uint64_t, uint32_t hintflags, FileInfo &info) const
//{
// if(! (hintflags & RS_FILE_HINTS_TURTLE)) // this should not happen, but it's a security.
// return false;
//
// RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
//
//#ifdef P3TURTLE_DEBUG
// std::cerr << "p3turtle: received file search request for hash " << hash << "." << std::endl ;
//#endif
//
// std::map<TurtleFileHash,TurtleFileHashInfo>::const_iterator it = _incoming_file_hashes.find(hash) ;
//
// if(_incoming_file_hashes.end() != it)
// {
// info.fname = it->second.name;
// info.size = it->second.size;
// info.hash = it->first;
//
// for(unsigned int i=0;i<it->second.tunnels.size();++i)
// {
// TransferInfo ti;
// ti.peerId = getTurtlePeerId(it->second.tunnels[i]);
// ti.name = "Distant peer for hash=" + hash ;
// ti.tfRate = 0;
// info.peers.push_back(ti);
// }
//
//#ifdef P3TURTLE_DEBUG
// std::cerr << " Found these tunnels for that hash:. "<< std::endl ;
// for(unsigned int i=0;i<it->second.tunnels.size();++i)
// std::cerr << " " << (void*)it->second.tunnels[i] << std::endl ;
//
// std::cerr << " answered yes. "<< std::endl ;
//#endif
// return true ;
// }
// else
// {
//#ifdef P3TURTLE_DEBUG
// std::cerr << " responding false." << std::endl ;
//#endif
// return false ;
// }
//}
void p3turtle::sendCRC32MapRequest(const std::string& peerId,const std::string& hash)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::sendCRC32MapRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleFileCrcRequestItem *item = new RsTurtleFileCrcRequestItem;
item->tunnel_id = tunnel_id ;
// item->crc_map = cmap ;
item->PeerId(tunnel.local_dst) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map request to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendCRC32Map(const std::string& peerId,const std::string& hash,const CRC32Map& cmap)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::senddataRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleFileCrcItem *item = new RsTurtleFileCrcItem ;
item->tunnel_id = tunnel_id ;
item->crc_map = cmap ;
item->PeerId(tunnel.local_src) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
bool p3turtle::isTurtlePeer(const std::string& peer_id) const
{

8
libretroshare/src/turtle/p3turtle.h
View File

@ -289,6 +289,12 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
/// Send a chunk map of this file to the given peer
void sendChunkMap(const std::string& peerId, const std::string& hash,const CompressedChunkMap& cmap) ;
/// Send a request for the crc32 map of this file to the given peer
void sendCRC32MapRequest(const std::string& peerId, const std::string& hash) ;
/// Send a crc32 map of this file to the given peer
void sendCRC32Map(const std::string& peerId, const std::string& hash,const CRC32Map& cmap) ;
private:
//--------------------------- Admin/Helper functions -------------------------//
@ -338,6 +344,8 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
void handleRecvFileData(RsTurtleFileDataItem *item);
void handleRecvFileMapRequest(RsTurtleFileMapRequestItem*);
void handleRecvFileMap(RsTurtleFileMapItem*);
void handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem*);
void handleRecvFileCRC32Map(RsTurtleFileCrcItem*);
//------ Functions connecting the turtle router to other components.----------//

185
libretroshare/src/turtle/rsturtleitem.cc
View File

@ -5,6 +5,7 @@
#include "turtletypes.h"
#include "rsturtleitem.h"
//#define P3TURTLE_DEBUG
// -----------------------------------------------------------------------------------//
// -------------------------------- Serialization. --------------------------------- //
// -----------------------------------------------------------------------------------//
@ -139,6 +140,31 @@ uint32_t RsTurtleFileMapItem::serial_size()
return s ;
}
uint32_t RsTurtleFileCrcRequestItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
return s ;
}
uint32_t RsTurtleFileCrcItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // size of _map
s += 4 ; // size of _crcs
s += 4 * crc_map._crcs.size() ;
s += 4 * crc_map._map._map.size() ;
return s ;
}
//
// ---------------------------------- Serialization ----------------------------------//
//
@ -174,6 +200,8 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
case RS_TURTLE_SUBTYPE_FILE_DATA : return new RsTurtleFileDataItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST : return new RsTurtleFileMapRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST : return new RsTurtleFileCrcRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC : return new RsTurtleFileCrcItem(data,*size) ;
default:
std::cerr << "Unknown packet type in RsTurtle!" << std::endl ;
@ -260,6 +288,82 @@ bool RsTurtleFileMapItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
bool RsTurtleFileCrcRequestItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleFileCrcRequestItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleFileCrcItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleFileCrcItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, crc_map._map._map.size());
ok &= setRawUInt32(data, tlvsize, &offset, crc_map._crcs.size());
for(uint32_t i=0;i<crc_map._map._map.size() && ok;++i)
ok &= setRawUInt32(data, tlvsize, &offset, crc_map._map._map[i]);
for(uint32_t i=0;i<crc_map._crcs.size() && ok;++i)
ok &= setRawUInt32(data, tlvsize, &offset, crc_map._crcs[i]);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleStringSearchRequestItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
@ -504,6 +608,62 @@ RsTurtleFileMapRequestItem::RsTurtleFileMapRequestItem(void *data,uint32_t pktsi
#endif
}
RsTurtleFileCrcItem::RsTurtleFileCrcItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
uint32_t s1,s2 ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id);
ok &= getRawUInt32(data, pktsize, &offset, &s1) ;
ok &= getRawUInt32(data, pktsize, &offset, &s2) ;
crc_map._map._map.resize(s1) ;
crc_map._crcs.resize(s2) ;
for(uint32_t i=0;i<s1 && ok;++i)
ok &= getRawUInt32(data, pktsize, &offset, &(crc_map._map._map[i])) ;
for(uint32_t i=0;i<s2 && ok;++i)
ok &= getRawUInt32(data, pktsize, &offset, &(crc_map._crcs[i])) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleFileCrcRequestItem::RsTurtleFileCrcRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map request item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id);
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize)
: RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT)
{
@ -925,3 +1085,28 @@ std::ostream& RsTurtleFileMapRequestItem::print(std::ostream& o, uint16_t)
return o ;
}
std::ostream& RsTurtleFileCrcItem::print(std::ostream& o, uint16_t)
{
o << "File CRC item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " map : " ;
for(uint32_t i=0;i<crc_map._map._map.size();++i)
o << (void*)crc_map._map._map[i] << std::endl ;
o << " CRC : " ;
for(uint32_t i=0;i<crc_map._crcs.size();++i)
o << (void*)crc_map._crcs[i] << std::endl ;
return o ;
}
std::ostream& RsTurtleFileCrcRequestItem::print(std::ostream& o, uint16_t)
{
o << "File CRC request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
return o ;
}

46
libretroshare/src/turtle/rsturtleitem.h
View File

@ -20,6 +20,8 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ;
const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0xfe ; // test value
/***********************************************************************************/
/* Basic Turtle Item Class */
@ -269,6 +271,50 @@ class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem
virtual uint32_t serial_size() ;
};
class RsTurtleFileCrcRequestItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleFileCrcRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST) {}
RsTurtleFileCrcRequestItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return false ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_SERVER ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
// CompressedChunkMap _map ; // list of chunks for which we need the CRC
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleFileCrcItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleFileCrcItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC) {}
RsTurtleFileCrcItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return true ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_CLIENT ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
CRC32Map crc_map ;// Map info for the file in compressed format. Each *bit* in the array uint's says "I have" or "I don't have"
// by default, we suppose the peer has all the chunks. This info will thus be and-ed
// with the default file map for this source.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
/***********************************************************************************/
/* Turtle Serialiser class */
/***********************************************************************************/

37
libretroshare/src/util/rsdir.cc
View File

@ -31,6 +31,7 @@
#include "util/rsdir.h"
#include "pqi/pqinotify.h"
#include <rsiface/rstypes.h>
#include <string>
#include <iostream>
#include <algorithm>
@ -113,10 +114,12 @@ class CRC32Table
uint32_t *_data ;
};
uint32_t rs_CRC32(const unsigned char *data,uint32_t len)
static const CRC32Table crc32_table ;
uint32_t RsDirUtil::rs_CRC32(const unsigned char *data,uint32_t _len)
{
static const CRC32Table crc32_table ;
uint32_t a = 0xffffffff ;
int64_t len = _len ;
for(const unsigned char *buf=data;len>=0;len--)
a = (a >> 8) ^ crc32_table[ (a & 0xff) ^ *buf++] ;
@ -124,6 +127,36 @@ uint32_t rs_CRC32(const unsigned char *data,uint32_t len)
return a ^ 0xffffffff ;
}
bool RsDirUtil::crc32File(FILE *fd, uint64_t file_size,uint32_t chunk_size, CRC32Map& crc_map)
{
if(fseek(fd,0,SEEK_SET) != 0)
{
std::cerr << "crc32File(): cannot fseek to beginnign of the file !!" << std::endl ;
return false ;
}
crc_map = CRC32Map(file_size,chunk_size) ;
unsigned char *buff = new unsigned char[chunk_size] ;
int len ;
uint64_t total_size = 0 ;
uint32_t nb_chunk = 0;
while((len = fread(buff,(size_t)1, (size_t)chunk_size, fd)) > 0)
{
crc_map.set(nb_chunk++,rs_CRC32(buff,len)) ;
total_size += len ;
}
delete[] buff ;
if(file_size != total_size)
{
std::cerr << "RsDirUtil::crc32File(): ERROR. The file size does not match the announced size." << std::endl ;
return false ;
}
return true ;
}
bool RsDirUtil::hashFile(const std::string& f_hash, std::string& hash)
{
FILE *fd;

8
libretroshare/src/util/rsdir.h
View File

@ -32,6 +32,8 @@
#include <list>
#include <stdint.h>
class CRC32Map ;
namespace RsDirUtil {
std::string getTopDir(std::string);
@ -47,8 +49,14 @@ bool hashFile(const std::string& full_path,std::string& hash) ;
bool renameFile(const std::string& from,const std::string& to) ;
bool createBackup (std::string sFilename, unsigned int nCount = 5);
// returns the CRC32 of the data of length len
//
uint32_t rs_CRC32(const unsigned char *data,uint32_t len) ;
// Computes the CRC32 map of a complete file, with given size and chunk size.
//
bool crc32File(FILE *f,uint64_t file_size,uint32_t chunk_size,CRC32Map& map) ;
int breakupDirList(std::string path,
std::list<std::string> &subdirs);

11
retroshare-gui/src/gui/TransfersDialog.cpp
View File

@ -216,6 +216,9 @@ TransfersDialog::TransfersDialog(QWidget *parent)
resumeAct = new QAction(QIcon(IMAGE_RESUME), tr("Resume"), this);
connect(resumeAct, SIGNAL(triggered()), this, SLOT(resumeFileTransfer()));
forceCheckAct = new QAction(QIcon(IMAGE_CANCEL), tr( "Force Check" ), this );
connect( forceCheckAct , SIGNAL( triggered() ), this, SLOT( forceCheck() ) );
cancelAct = new QAction(QIcon(IMAGE_CANCEL), tr( "Cancel" ), this );
connect( cancelAct , SIGNAL( triggered() ), this, SLOT( cancel() ) );
@ -381,7 +384,10 @@ void TransfersDialog::downloadListCostumPopupMenu( QPoint point )
}
if(info.downloadStatus != FT_STATE_COMPLETE)
{
contextMnu.addAction( forceCheckAct);
contextMnu.addAction( cancelAct);
}
contextMnu.addSeparator();
}
@ -940,6 +946,11 @@ QString TransfersDialog::getPeerName(const std::string& id) const
return res ;
}
void TransfersDialog::forceCheck()
{
if (!controlTransferFile(RS_FILE_CTRL_FORCE_CHECK))
std::cerr << "resumeFileTransfer(): can't force check file transfer" << std::endl;
}
void TransfersDialog::cancel()
{
bool first = true;

2
retroshare-gui/src/gui/TransfersDialog.h
View File

@ -72,6 +72,7 @@ private slots:
void downloadListCostumPopupMenu( QPoint point );
void cancel();
void forceCheck();
/** removes finished Downloads**/
void clearcompleted();
@ -137,6 +138,7 @@ private:
QAction* showdowninfoAct;
QAction* playAct;
QAction* cancelAct;
QAction* forceCheckAct;
QAction* clearcompletedAct;
QAction* copylinkAct;
QAction* pastelinkAct;