Added deferred check of chunks during file transfer. Chunk sha1 sums are requested to the sources and checked for downloaded data.

Validated chunks are shared to other peers. Force check is now very simple since it just turns all chunks into "needs checking" mode
and sums are asked to sources. Sources maintain a temporary cache of chunks. Since sums are requested sparsely, this should not 
affect the sources in terms of performance. We can still imagine precomputing and saving sha1 of chunks while hashing them.

For backward compatibility reasons, the following has been setup *temporarily* in this version:
- unvalidated chunks are still considered as already obtained, and are shared and saved
- force check has been disabled
- final file check is maintained
- in case of file fail, the old checking mode will be used.

All changes for next version are kept in the define 'USE_NEW_CHUNK_CHECKING_CODE' that will be made the default in a few weeks.
At start, I expect most chunk to stya yellow during download, until most sources are able to provide chunk hashs.




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@5019 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2012-03-15 19:55:43 +00:00
parent 7ab5b54266
commit 889a2b2433
31 changed files with 1540 additions and 35 deletions

View file

@ -1001,6 +1001,12 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST: handleRecvFileCRC32MapRequest(dynamic_cast<RsTurtleFileCrcRequestItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC : handleRecvChunkCRC(dynamic_cast<RsTurtleChunkCrcItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST: handleRecvChunkCRCRequest(dynamic_cast<RsTurtleChunkCrcRequestItem *>(item)) ;
break ;
default:
std::cerr << "WARNING: Unknown packet type received: id=" << (void*)(item->PacketSubType()) << ". Is somebody trying to poison you ?" << std::endl ;
#ifdef P3TURTLE_DEBUG
@ -1226,7 +1232,78 @@ void p3turtle::handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem *item)
_ft_server->getMultiplexer()->recvCRC32MapRequest(vpid,hash) ;
}
void p3turtle::handleRecvChunkCRC(RsTurtleChunkCrcItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map Request item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: chunk crc request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file crc request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvSingleChunkCrc(vpid,hash,item->chunk_number,item->check_sum) ;
}
void p3turtle::handleRecvChunkCRCRequest(RsTurtleChunkCrcRequestItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file CRC32 Map Request item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: chunk crc request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file crc request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvSingleChunkCrcRequest(vpid,hash,item->chunk_number) ;
}
void p3turtle::handleRecvFileCRC32Map(RsTurtleFileCrcItem *item)
{
#ifdef P3TURTLE_DEBUG
@ -1435,7 +1512,67 @@ void p3turtle::sendChunkMap(const std::string& peerId,const std::string& ,const
#endif
sendItem(item) ;
}
void p3turtle::sendSingleChunkCRC(const std::string& peerId,const std::string&,uint32_t chunk_number,const Sha1CheckSum& crc)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::sendCRC32MapRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleChunkCrcItem *item = new RsTurtleChunkCrcItem;
item->tunnel_id = tunnel_id ;
item->chunk_number = chunk_number ;
item->check_sum = crc ;
item->PeerId(tunnel.local_dst) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map request to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendSingleChunkCRCRequest(const std::string& peerId,const std::string&,uint32_t chunk_number)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::sendCRC32MapRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleChunkCrcRequestItem *item = new RsTurtleChunkCrcRequestItem;
item->tunnel_id = tunnel_id ;
item->chunk_number = chunk_number ;
item->PeerId(tunnel.local_dst) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending CRC32 map request to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendCRC32MapRequest(const std::string& peerId,const std::string& )
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/

View file

@ -301,6 +301,12 @@ class p3turtle: public p3Service, /*public pqiMonitor,*/ public RsTurtle,/* publ
/// Send a crc32 map of this file to the given peer
void sendCRC32Map(const std::string& peerId, const std::string& hash,const CRC32Map& cmap) ;
/// Send a request for the CRC of a single chunk of this file to the given peer
void sendSingleChunkCRCRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_number) ;
/// Send a crc32 map of this file to the given peer
void sendSingleChunkCRC(const std::string& peerId, const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& sum) ;
private:
//--------------------------- Admin/Helper functions -------------------------//
@ -352,6 +358,8 @@ class p3turtle: public p3Service, /*public pqiMonitor,*/ public RsTurtle,/* publ
void handleRecvFileMap(RsTurtleFileMapItem*);
void handleRecvFileCRC32MapRequest(RsTurtleFileCrcRequestItem*);
void handleRecvFileCRC32Map(RsTurtleFileCrcItem*);
void handleRecvChunkCRCRequest(RsTurtleChunkCrcRequestItem*);
void handleRecvChunkCRC(RsTurtleChunkCrcItem*);
//------ Functions connecting the turtle router to other components.----------//

View file

@ -149,7 +149,27 @@ uint32_t RsTurtleFileCrcRequestItem::serial_size()
return s ;
}
uint32_t RsTurtleChunkCrcItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // chunk number
s += 20 ; // check_sum
return s ;
}
uint32_t RsTurtleChunkCrcRequestItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // chunk number
return s ;
}
uint32_t RsTurtleFileCrcItem::serial_size()
{
uint32_t s = 0 ;
@ -202,6 +222,8 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST : return new RsTurtleFileCrcRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_CRC : return new RsTurtleFileCrcItem(data,*size) ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem(data,*size) ;
default:
std::cerr << "Unknown packet type in RsTurtle!" << std::endl ;
@ -322,6 +344,40 @@ bool RsTurtleFileCrcRequestItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
bool RsTurtleChunkCrcRequestItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleChunkCrcRequestItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_number);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleFileCrcItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
@ -363,7 +419,45 @@ bool RsTurtleFileCrcItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
bool RsTurtleChunkCrcItem::serialize(void *data,uint32_t& pktsize)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "RsTurtleChunkCrcRequestItem::serialize(): serializing packet:" << std::endl ;
print(std::cerr,2) ;
#endif
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_number);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[0]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[1]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[2]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[3]);
ok &= setRawUInt32(data, tlvsize, &offset, check_sum.fourbytes[4]);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
}
return ok;
}
bool RsTurtleStringSearchRequestItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
@ -644,7 +738,34 @@ RsTurtleFileCrcItem::RsTurtleFileCrcItem(void *data,uint32_t pktsize)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleChunkCrcItem::RsTurtleChunkCrcItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC)
{
setPriorityLevel(QOS_PRIORITY_RS_TURTLE_CHUNK_CRC) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id) ;
ok &= getRawUInt32(data, pktsize, &offset, &chunk_number) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[0]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[1]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[2]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[3]) ;
ok &= getRawUInt32(data, pktsize, &offset, &check_sum.fourbytes[4]) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleFileCrcRequestItem::RsTurtleFileCrcRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST)
{
@ -667,7 +788,29 @@ RsTurtleFileCrcRequestItem::RsTurtleFileCrcRequestItem(void *data,uint32_t pktsi
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleChunkCrcRequestItem::RsTurtleChunkCrcRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST)
{
setPriorityLevel(QOS_PRIORITY_RS_TURTLE_CHUNK_CRC_REQUEST) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id) ;
ok &= getRawUInt32(data, pktsize, &offset, &chunk_number) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize)
: RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT)
{
@ -1119,3 +1262,23 @@ std::ostream& RsTurtleFileCrcRequestItem::print(std::ostream& o, uint16_t)
return o ;
}
std::ostream& RsTurtleChunkCrcRequestItem::print(std::ostream& o, uint16_t)
{
o << "Chunk CRC request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk num : " << chunk_number << std::endl ;
return o ;
}
std::ostream& RsTurtleChunkCrcItem::print(std::ostream& o, uint16_t)
{
o << "Chunk CRC request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk num : " << chunk_number << std::endl ;
o << " sha1 sum : " << check_sum.toStdString() << std::endl ;
return o ;
}

View file

@ -22,6 +22,8 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ;
/***********************************************************************************/
/* Basic Turtle Item Class */
@ -281,14 +283,32 @@ class RsTurtleFileCrcRequestItem: public RsTurtleGenericTunnelItem
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
// CompressedChunkMap _map ; // list of chunks for which we need the CRC
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleChunkCrcRequestItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleChunkCrcRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST) { setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC_REQUEST);}
RsTurtleChunkCrcRequestItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return false ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_SERVER ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
uint32_t chunk_number ; // id of the chunk to CRC.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleFileCrcItem: public RsTurtleGenericTunnelItem
{
@ -312,6 +332,26 @@ class RsTurtleFileCrcItem: public RsTurtleGenericTunnelItem
virtual uint32_t serial_size() ;
};
class RsTurtleChunkCrcItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleChunkCrcItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_CHUNK_CRC) { setPriorityLevel(QOS_PRIORITY_RS_CHUNK_CRC);}
RsTurtleChunkCrcItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return true ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return DIRECTION_CLIENT ; }
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
uint32_t chunk_number ;
Sha1CheckSum check_sum ;
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
/***********************************************************************************/
/* Turtle Serialiser class */
/***********************************************************************************/