- implemented a visualisation for currently handled chunks, availability maps, and transfer info

- implemented transfer protocol for chunk availability maps between peers (not enabled yet though)
- suppressed rsiface directory from retroshare-gui
- moved notifyqt.{h,cpp} to gui/

next moves: 
	- send availability maps to clients; 
	- connect turtle search to unfinished files;
	- test multisource download with unfinished files.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1939 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2009-12-28 21:11:00 +00:00
parent 1a86871556
commit f4a2eaecce
19 changed files with 622 additions and 747 deletions

View file

@ -205,7 +205,7 @@ void p3turtle::manageTunnels()
time_t now = time(NULL) ;
bool tunnel_campain = false ;
if(now > _last_tunnel_campaign_time+REGULAR_TUNNEL_DIGGING_TIME)
if(now > _last_tunnel_campaign_time + REGULAR_TUNNEL_DIGGING_TIME)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Tunnel management: flaging all hashes for tunnels digging." << std::endl ;
@ -819,14 +819,12 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
// Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete.
tunnel.time_stamp = time(NULL) ;
// also update the hash time stamp to show that it's actually being downloaded.
_incoming_file_hashes[tunnel.hash].time_stamp = time(NULL) ;
// Let's figure out whether this reuqest is for us or not.
if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer
{
std::map<TurtleFileHash,TurtleFileHashInfo>::const_iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
std::map<TurtleFileHash,TurtleFileHashInfo>::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
#endif
@ -848,6 +846,9 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
vpid = tunnel.vpid ;
hash = tunnel.hash ;
size = hash_info.size ;
// also update the hash time stamp to show that it's actually being downloaded.
it->second.time_stamp = time(NULL) ;
}
else // No, it's a request we should forward down the pipe.
{
@ -876,6 +877,73 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
}
void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file Map item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(item->tunnel_id)) ;
if(it == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: got file map with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it->second) ;
// Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete.
tunnel.time_stamp = time(NULL) ;
// Let's figure out whether this reuqest is for us or not.
if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer
{
std::map<TurtleFileHash,TurtleFileHashInfo>::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
#endif
if(it==_incoming_file_hashes.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ;
#endif
return ;
}
const TurtleFileHashInfo& hash_info(it->second) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " This is an endpoint for this file map." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// also update the hash time stamp to show that it's actually being downloaded.
it->second.time_stamp = time(NULL) ;
// we should check that there is no backward call to the turtle router!
//
_ft_server->getMultiplexer()->recvFileMap(tunnel.vpid,tunnel.hash,item->chunk_size,item->nb_chunks,item->compressed_map) ;
}
else // No, it's a request we should forward down the pipe.
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Forwarding file map to peer " << tunnel.local_src << std::endl ;
#endif
RsTurtleFileMapItem *res_item = new RsTurtleFileMapItem(*item) ;
res_item->PeerId(tunnel.local_src) ;
sendItem(res_item) ;
return ;
}
}
}
// Send a data request into the correct tunnel for the given file hash
void p3turtle::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t, uint64_t offset, uint32_t chunksize)
{
@ -953,61 +1021,61 @@ void p3turtle::sendFileData(const std::string& peerId, const std::string& hash,
sendItem(item) ;
}
bool p3turtle::search(std::string hash, uint64_t, uint32_t hintflags, FileInfo &info) const
{
if(! (hintflags & RS_FILE_HINTS_TURTLE)) // this should not happen, but it's a security.
return false;
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: received file search request for hash " << hash << "." << std::endl ;
#endif
std::map<TurtleFileHash,TurtleFileHashInfo>::const_iterator it = _incoming_file_hashes.find(hash) ;
if(_incoming_file_hashes.end() != it)
{
info.fname = it->second.name;
info.size = it->second.size;
info.hash = it->first;
for(unsigned int i=0;i<it->second.tunnels.size();++i)
{
TransferInfo ti;
ti.peerId = getTurtlePeerId(it->second.tunnels[i]);
ti.name = "Distant peer for hash=" + hash ;
ti.tfRate = 0;
info.peers.push_back(ti);
}
#ifdef P3TURTLE_DEBUG
std::cerr << " Found these tunnels for that hash:. "<< std::endl ;
for(unsigned int i=0;i<it->second.tunnels.size();++i)
std::cerr << " " << (void*)it->second.tunnels[i] << std::endl ;
std::cerr << " answered yes. "<< std::endl ;
#endif
return true ;
}
else
{
#ifdef P3TURTLE_DEBUG
std::cerr << " responding false." << std::endl ;
#endif
return false ;
}
}
// This function is actually not needed: Search request to the turtle router are:
// - distant search requests, handled by the router
// - search requests over files being downloaded, handled by rsFiles !!
//
//bool p3turtle::search(std::string hash, uint64_t, uint32_t hintflags, FileInfo &info) const
//{
// if(! (hintflags & RS_FILE_HINTS_TURTLE)) // this should not happen, but it's a security.
// return false;
//
// RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
//
//#ifdef P3TURTLE_DEBUG
// std::cerr << "p3turtle: received file search request for hash " << hash << "." << std::endl ;
//#endif
//
// std::map<TurtleFileHash,TurtleFileHashInfo>::const_iterator it = _incoming_file_hashes.find(hash) ;
//
// if(_incoming_file_hashes.end() != it)
// {
// info.fname = it->second.name;
// info.size = it->second.size;
// info.hash = it->first;
//
// for(unsigned int i=0;i<it->second.tunnels.size();++i)
// {
// TransferInfo ti;
// ti.peerId = getTurtlePeerId(it->second.tunnels[i]);
// ti.name = "Distant peer for hash=" + hash ;
// ti.tfRate = 0;
// info.peers.push_back(ti);
// }
//
//#ifdef P3TURTLE_DEBUG
// std::cerr << " Found these tunnels for that hash:. "<< std::endl ;
// for(unsigned int i=0;i<it->second.tunnels.size();++i)
// std::cerr << " " << (void*)it->second.tunnels[i] << std::endl ;
//
// std::cerr << " answered yes. "<< std::endl ;
//#endif
// return true ;
// }
// else
// {
//#ifdef P3TURTLE_DEBUG
// std::cerr << " responding false." << std::endl ;
//#endif
// return false ;
// }
//}
bool p3turtle::isTurtlePeer(const std::string& peer_id) const
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
return _virtual_peers.find(peer_id) != _virtual_peers.end() ;
// if(it->second.tunnels.empty())
// return false ;
//
}
std::string p3turtle::getTurtlePeerId(TurtleTunnelId tid) const
@ -1472,7 +1540,7 @@ void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item)
bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,FileInfo& info)
{
return rsFiles->FileDetails(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY, info);
return rsFiles->FileDetails(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY | RS_FILE_HINTS_DOWNLOAD, info);
}
static std::string printNumber(uint64_t num,bool hex=false)
@ -1480,7 +1548,7 @@ static std::string printNumber(uint64_t num,bool hex=false)
if(hex)
{
char tmp[100] ;
sprintf(tmp,"0x%08lx",num) ;
sprintf(tmp,"0x%08x%08x", uint32_t(num >> 32),uint32_t(num & ( (1<<32)-1 ))) ;
return std::string(tmp) ;
}
else

View file

@ -131,6 +131,11 @@
//
// Questions:
// - should tunnels be re-used ? nope. The only useful case would be when two peers are exchanging files, which happens quite rarely.
//
// TODO:
// - derive most packets from RsTurtleTunnelPacketItem, that can be handled
// as raw data through a tunnel. This will avoid replicating most of the
// tunnel handling code.
#ifndef MRK_PQI_TURTLE_H
@ -152,6 +157,7 @@ class p3AuthMgr;
class p3ConnectMgr;
class ftDataMultiplex;
class RsSerialiser;
static const int TURTLE_MAX_SEARCH_DEPTH = 6 ;
// This class is used to keep trace of requests (searches and tunnels).
@ -197,7 +203,7 @@ class TurtleFileHashInfo
// p3Config | ConfigChanged() | used to load/save .cfg file for turtle variales.
// -----------+------------------+------------------------------------------------------
//
class p3turtle: public p3Service, public pqiMonitor, public RsTurtle, public ftSearch, public p3Config
class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public ftSearch */ public p3Config
{
public:
p3turtle(p3ConnectMgr *cm,ftServer *m);
@ -250,10 +256,10 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle, public ftS
//
virtual int tick();
/************* from ftSearch *******************/
// Search function. This function looks into the file hashes currently handled , and sends back info.
//
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const ;
// /************* from ftSearch *******************/
// // Search function. This function looks into the file hashes currently handled , and sends back info.
// //
// virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const ;
/************* from p3Config *******************/
virtual RsSerialiser *setupSerialiser() ;
@ -304,6 +310,7 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle, public ftS
void handleTunnelResult(RsTurtleTunnelOkItem *item);
void handleRecvFileRequest(RsTurtleFileRequestItem *item);
void handleRecvFileData(RsTurtleFileDataItem *item);
void handleRecvFileMap(RsTurtleFileMapItem*);
//------ Functions connecting the turtle router to other components.----------//

View file

@ -113,6 +113,22 @@ uint32_t RsTurtleFileDataItem::serial_size()
return s ;
}
uint32_t RsTurtleFileMapItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // chunk_size
s += 4 ; // nb_chunks
s += 4 ; // compressed_map.size()
s += 4 * compressed_map.size() ;
return s ;
}
//
// ---------------------------------- Serialization ----------------------------------//
//
@ -146,6 +162,7 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_REQUEST : return new RsTurtleFileRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_DATA : return new RsTurtleFileDataItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ;
default:
std::cerr << "Unknown packet type in RsTurtle!" << std::endl ;
@ -162,6 +179,44 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
}
bool RsTurtleFileMapItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_size);
ok &= setRawUInt32(data, tlvsize, &offset, nb_chunks);
ok &= setRawUInt32(data, tlvsize, &offset, compressed_map.size());
for(uint32_t i=0;i<compressed_map.size() && ok;++i)
ok &= setRawUInt32(data, tlvsize, &offset, compressed_map[i]);
if (offset != tlvsize)
{
ok = false;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
#endif
}
return ok;
}
bool RsTurtleStringSearchRequestItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
@ -350,6 +405,30 @@ bool RsTurtleSearchResultItem::serialize(void *data,uint32_t& pktsize)
return ok;
}
RsTurtleFileMapItem::RsTurtleFileMapItem(void *data,uint32_t pktsize)
: RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " type = search result" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
uint32_t rssize = getRsItemSize(data);
/* add mandatory parts first */
bool ok = true ;
uint32_t s ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id);
ok &= getRawUInt32(data, pktsize, &offset, &chunk_size);
ok &= getRawUInt32(data, pktsize, &offset, &nb_chunks) ;
ok &= getRawUInt32(data, pktsize, &offset, &s) ;
compressed_map.resize(s) ;
for(uint32_t i=0;i<s && ok;++i)
ok &= getRawUInt32(data, pktsize, &offset, &(compressed_map[i])) ;
}
RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize)
: RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT)
{
@ -742,3 +821,18 @@ std::ostream& RsTurtleFileDataItem::print(std::ostream& o, uint16_t)
return o ;
}
std::ostream& RsTurtleFileMapItem::print(std::ostream& o, uint16_t)
{
o << "File map item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk size: " << chunk_size << std::endl ;
o << " nb chunks : " << nb_chunks << std::endl ;
o << " map : " ;
for(uint32_t i=0;i<compressed_map.size();++i)
o << (void*)compressed_map[i] << std::endl ;
return o ;
}

View file

@ -17,6 +17,7 @@ const uint8_t RS_TURTLE_SUBTYPE_TUNNEL_CLOSED = 0x06 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ;
const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
/***********************************************************************************/
/* Basic Turtle Item Class */
@ -142,36 +143,6 @@ class RsTurtleTunnelOkItem: public RsTurtleItem
virtual uint32_t serial_size() ;
};
#ifdef A_VIRER
class RsTurtleCloseTunnelItem: public RsTurtleItem
{
public:
RsTurtleCloseTunnelItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_CLOSE_TUNNEL) {}
RsTurtleCloseTunnelItem(void *data,uint32_t size) ; // deserialization
uint32_t tunnel_id ; // id of the tunnel to close.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
protected:
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleTunnelClosedItem: public RsTurtleItem
{
public:
RsTurtleTunnelClosedItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_TUNNEL_CLOSED) {}
RsTurtleTunnelClosedItem(void *data,uint32_t size) ; // deserialization
uint32_t tunnel_id ; // id of the tunnel to close.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
protected:
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
#endif
/***********************************************************************************/
/* Turtle File Transfer item classes */
/***********************************************************************************/
@ -210,6 +181,27 @@ class RsTurtleFileDataItem: public RsTurtleItem
virtual uint32_t serial_size() ;
};
class RsTurtleFileMapItem: public RsTurtleItem
{
public:
RsTurtleFileMapItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP) {}
RsTurtleFileMapItem(void *data,uint32_t size) ; // deserialization
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
uint32_t chunk_size ; // fixed size of chunks, as seen from the source, for the given map.
uint32_t nb_chunks ; // number of chunks in the file. The last two infos are redundant, as we can recompute
// this info from the file size, but this allows a security check.
std::vector<uint32_t> compressed_map ; // Map info for the file in compressed format. Each *bit* in the array uint's says "I have" or "I don't have"
// by default, we suppose the peer has all the chunks. This info will thus be and-ed
// with the default file map for this source.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
/***********************************************************************************/
/* Turtle Serialiser class */
/***********************************************************************************/