From b83e894640c31b53284c61aa55771aee66332b3a Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 2 Jan 2010 21:30:19 +0000 Subject: [PATCH] - added a generic RsItem to the turtle router and the methods to route it. This makes the code much more elegant. - suppressed a cross mutex lock bug that happened rarely while digging tunnels - changed FileDetails in ftServer so that it's now possiblt to search for hashes of files being downloaded - improved the search code in ftdatamultiplex - added some comments to the turtle code git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1964 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/ft/ftdatamultiplex.cc | 40 ++- libretroshare/src/ft/ftfilecreator.cc | 16 +- libretroshare/src/ft/ftserver.cc | 27 +- libretroshare/src/turtle/p3turtle.cc | 407 ++++++++++++----------- libretroshare/src/turtle/p3turtle.h | 118 ++++--- libretroshare/src/turtle/rsturtleitem.cc | 6 +- libretroshare/src/turtle/rsturtleitem.h | 66 +++- 7 files changed, 388 insertions(+), 292 deletions(-) diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index 9a52e4f51..4c0a59db0 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -127,36 +127,44 @@ bool ftDataMultiplex::FileDetails(std::string hash, uint32_t hintsflag, FileI #endif RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - std::map::iterator sit; - sit = mServers.find(hash); - if (sit != mServers.end()) + + if(hintsflag & RS_FILE_HINTS_DOWNLOAD) { + std::map::iterator cit; + if (mClients.end() != (cit = mClients.find(hash))) + { #ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::FileDetails()"; - std::cerr << " Found ftFileProvider!"; - std::cerr << std::endl; + std::cerr << "ftDataMultiplex::FileDetails()"; + std::cerr << " Found ftFileCreator!"; + std::cerr << std::endl; #endif - (sit->second)->FileDetails(info); - return true; + //(cit->second).mModule->FileDetails(info); + (cit->second).mCreator->FileDetails(info); + return true; + } } - std::map::iterator cit; - if (mClients.end() != (cit = mClients.find(hash))) + if(hintsflag & RS_FILE_HINTS_UPLOAD) { + std::map::iterator sit; + sit = mServers.find(hash); + if (sit != mServers.end()) + { #ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::FileDetails()"; - std::cerr << " Found ftFileCreator!"; - std::cerr << std::endl; + std::cerr << "ftDataMultiplex::FileDetails()"; + std::cerr << " Found ftFileProvider!"; + std::cerr << std::endl; #endif - //(cit->second).mModule->FileDetails(info); - (cit->second).mCreator->FileDetails(info); - return true; + (sit->second)->FileDetails(info); + return true; + } } + #ifdef MPLEX_DEBUG std::cerr << "ftDataMultiplex::FileDetails()"; std::cerr << " Found nothing"; diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index 25f02b8e6..7d4d9c061 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -34,15 +34,6 @@ ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash, #endif RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - - /* initialise the Transfer Lists */ -// mStart = recvd; -// mEnd = recvd; - -#ifdef TO_DO - // we should init the chunk map with some bit array saying what is received and what is not!! - chunkMap.setTotalReceived(recvd) ; -#endif } bool ftFileCreator::getFileData(uint64_t offset, uint32_t &chunk_size, void *data) @@ -423,6 +414,13 @@ void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size, { RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + // At this point, we should cancel all file chunks that are asked to the + // peer and which this peer actually doesn't possesses. Otherwise, the transfer may get stuck. + // This should be done by: + // - first setting the peer availability map + // - then asking the chunkmap which chunks are being downloaded, but actually shouldn't + // - cancelling them in the ftFileCreator, so that they can be re-asked later to another peer. + // chunkMap.setPeerAvailabilityMap(peer_id,chunk_size,nb_chunks,compressed_map) ; } diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 464c3eac6..124efbfa0 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -367,24 +367,19 @@ bool ftServer::FileUploads(std::list &hashs) bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) { - bool found = false; if (hintflags & RS_FILE_HINTS_DOWNLOAD) - { - //found = mFtDataplex->FileDetails(hash, hintflags, info); - // - // Use Controller for download searches. - found = mFtController->FileDetails(hash, info); - } - else if (hintflags & RS_FILE_HINTS_UPLOAD) - { - found = mFtDataplex->FileDetails(hash, hintflags, info); - } + if(mFtController->FileDetails(hash, info)) + return true ; - if (!found) - { - found = mFtSearch->search(hash, 0, hintflags, info); - } - return found; + if(hintflags & RS_FILE_HINTS_UPLOAD) + if(mFtDataplex->FileDetails(hash, hintflags, info)) + return true ; + + if(hintflags & ~(RS_FILE_HINTS_UPLOAD | RS_FILE_HINTS_DOWNLOAD)) + if(mFtSearch->search(hash, 0, hintflags, info)) + return true ; + + return false; } /***************************************************************/ diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index fd8ff210e..3c85b7b8e 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -515,11 +515,6 @@ uint32_t p3turtle::generatePersonalFilePrint(const TurtleFileHash& hash,bool b) // int p3turtle::handleIncoming() { -#ifdef P3TURTLE_DEBUG -// std::cerr << "p3turtle::handleIncoming()"; -// std::cerr << std::endl; -#endif - int nhandled = 0; // While messages read // @@ -529,32 +524,34 @@ int p3turtle::handleIncoming() { nhandled++; - switch(item->PacketSubType()) + RsTurtleGenericTunnelItem *gti = dynamic_cast(item) ; + + if(gti != NULL) + routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels. + else /// These packets should be destroyed by the client. { - case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST: - case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast(item)) ; - break ; + /// Special packets that require specific treatment, because tunnels do not exist for these packets. + /// These packets are destroyed here, after treatment. + // + switch(item->PacketSubType()) + { + case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST: + case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast(item)) ; + break ; - case RS_TURTLE_SUBTYPE_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; - break ; + case RS_TURTLE_SUBTYPE_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; + break ; - case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : handleTunnelRequest(dynamic_cast(item)) ; - break ; + case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : handleTunnelRequest(dynamic_cast(item)) ; + break ; - case RS_TURTLE_SUBTYPE_TUNNEL_OK : handleTunnelResult(dynamic_cast(item)) ; - break ; - - case RS_TURTLE_SUBTYPE_FILE_REQUEST : handleRecvFileRequest(dynamic_cast(item)) ; - break ; - - case RS_TURTLE_SUBTYPE_FILE_DATA : handleRecvFileData(dynamic_cast(item)) ; - break ; - - // Here will also come handling of file transfer requests, tunnel digging/closing, etc. - default: - std::cerr << "p3turtle::handleIncoming: Unknown packet subtype " << item->PacketSubType() << std::endl ; + case RS_TURTLE_SUBTYPE_TUNNEL_OK : handleTunnelResult(dynamic_cast(item)) ; + break ; + default: + std::cerr << "p3turtle::handleIncoming: Unknown packet subtype " << item->PacketSubType() << std::endl ; + } + delete item; } - delete item; } return nhandled; @@ -725,6 +722,88 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) // --------------------------------- File Transfer. -------------------------------- // // -----------------------------------------------------------------------------------// +// Routing of turtle tunnel items in a generic manner. Most tunnel packets will use this function, except packets designed for +// contructing the tunnels and searching, namely TurtleSearchRequests/Results and OpenTunnel/TunnelOkItems +// +void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) +{ +#ifdef P3TURTLE_DEBUG + std::cerr << "p3Turtle: treating generic tunnel item:" << std::endl ; + item->print(std::cerr,1) ; +#endif + RsTurtleGenericTunnelItem::Direction direction = item->travelingDirection() ; + + { + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + + // look for the tunnel id. + // + std::map::iterator it(_local_tunnels.find(item->tunnelId())) ; + + if(it == _local_tunnels.end()) + { +#ifdef P3TURTLE_DEBUG + std::cerr << "p3turtle: got file map with unknown tunnel id " << (void*)item->tunnelId() << std::endl ; +#endif + return ; + } + + TurtleTunnel& tunnel(it->second) ; + + // Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete. + if(item->shouldStampTunnel()) + tunnel.time_stamp = time(NULL) ; + + // Let's figure out whether this packet is for us or not. + + if(direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT && tunnel.local_src != mConnMgr->getOwnId()) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Forwarding generic item to peer " << tunnel.local_src << std::endl ; +#endif + item->PeerId(tunnel.local_src) ; + + sendItem(item) ; + return ; + } + + if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER && tunnel.local_dst != mConnMgr->getOwnId()) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Forwarding generic item to peer " << tunnel.local_dst << std::endl ; +#endif + item->PeerId(tunnel.local_dst) ; + + sendItem(item) ; + return ; + } + } + + // The packet was not forwarded, so it is for us. Let's treat it. + // This is done off-mutex, to avoid various deadlocks + // + if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER) + switch(item->PacketSubType()) + { + case RS_TURTLE_SUBTYPE_FILE_REQUEST: handleRecvFileRequest(dynamic_cast(item)) ; + break ; + default: + std::cerr << "Unknown server packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ; + exit(-1) ; + } + + if(direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) + switch(item->PacketSubType()) + { + case RS_TURTLE_SUBTYPE_FILE_DATA : handleRecvFileData(dynamic_cast(item)) ; + break ; + default: + std::cerr << "Unknown client packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ; + exit(-1) ; + } + + delete item ; +} void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item) { @@ -742,9 +821,9 @@ void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item) { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - std::map::iterator it(_local_tunnels.find(item->tunnel_id)) ; + std::map::iterator it2(_local_tunnels.find(item->tunnel_id)) ; - if(it == _local_tunnels.end()) + if(it2 == _local_tunnels.end()) { #ifdef P3TURTLE_DEBUG std::cerr << "p3turtle: got file request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ; @@ -752,36 +831,19 @@ void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item) return ; } - TurtleTunnel& tunnel(it->second) ; - - // Let's figure out whether this reuqest is for us or not. - - if(tunnel.local_dst == mConnMgr->getOwnId()) // Yes, we have to pass on the request to the data multiplexer - { - std::map::const_iterator it(_outgoing_file_hashes.find(tunnel.hash)) ; + TurtleTunnel& tunnel(it2->second) ; + std::map::const_iterator it(_outgoing_file_hashes.find(tunnel.hash)) ; #ifdef P3TURTLE_DEBUG - assert(!tunnel.hash.empty()) ; - assert(it != _outgoing_file_hashes.end()) ; + assert(!tunnel.hash.empty()) ; + assert(it != _outgoing_file_hashes.end()) ; - std::cerr << " This is an endpoint for this file request." << std::endl ; - std::cerr << " Forwarding data request to the multiplexer." << std::endl ; - std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; + std::cerr << " This is an endpoint for this file request." << std::endl ; + std::cerr << " Forwarding data request to the multiplexer." << std::endl ; + std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; #endif - // _ft_server->getMultiplexer()->recvDataRequest(tunnel.vpid,tunnel.hash,it->second.size,item->chunk_offset,item->chunk_size) ; - // - size = it->second.size ; - vpid = tunnel.vpid ; - hash = tunnel.hash ; - } - else // No, it's a request we should forward down the pipe. - { - RsTurtleFileRequestItem *res_item = new RsTurtleFileRequestItem(*item) ; - - res_item->PeerId(tunnel.local_dst) ; - - sendItem(res_item) ; - return ; - } + size = it->second.size ; + vpid = tunnel.vpid ; + hash = tunnel.hash ; } // This call is voluntarily off-mutex gards because it can cause cross mutex locks with the multiplexer. @@ -796,18 +858,16 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item) std::cerr << "p3Turtle: received file data item:" << std::endl ; item->print(std::cerr,1) ; #endif - // This is a new request. Let's add it to the request map, and forward it to - // open peers. - TurtleVirtualPeerId vpid ; uint64_t size ; TurtleFileHash hash ; + { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - std::map::iterator it(_local_tunnels.find(item->tunnel_id)) ; + std::map::iterator it2(_local_tunnels.find(item->tunnel_id)) ; - if(it == _local_tunnels.end()) + if(it2 == _local_tunnels.end()) { #ifdef P3TURTLE_DEBUG std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ; @@ -815,68 +875,40 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item) return ; } - TurtleTunnel& tunnel(it->second) ; - - // Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete. - tunnel.time_stamp = time(NULL) ; - - // Let's figure out whether this reuqest is for us or not. - - if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer - { - std::map::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ; + TurtleTunnel& tunnel(it2->second) ; + std::map::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ; #ifdef P3TURTLE_DEBUG - assert(!tunnel.hash.empty()) ; + assert(!tunnel.hash.empty()) ; #endif - if(it==_incoming_file_hashes.end()) - { -#ifdef P3TURTLE_DEBUG - std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ; -#endif - return ; - } - - const TurtleFileHashInfo& hash_info(it->second) ; -#ifdef P3TURTLE_DEBUG - std::cerr << " This is an endpoint for this data chunk." << std::endl ; - std::cerr << " Forwarding data to the multiplexer." << std::endl ; - std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; -#endif - //_ft_server->getMultiplexer()->recvData(tunnel.vpid,tunnel.hash,hash_info.size,item->chunk_offset,item->chunk_size,item->chunk_data) ; - vpid = tunnel.vpid ; - hash = tunnel.hash ; - size = hash_info.size ; - - // also update the hash time stamp to show that it's actually being downloaded. - it->second.time_stamp = time(NULL) ; - } - else // No, it's a request we should forward down the pipe. + if(it==_incoming_file_hashes.end()) { #ifdef P3TURTLE_DEBUG - std::cerr << " Forwarding data chunk to peer " << tunnel.local_src << std::endl ; + std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ; #endif - RsTurtleFileDataItem *res_item = new RsTurtleFileDataItem(*item) ; - - res_item->chunk_data = malloc(res_item->chunk_size) ; - - if(res_item->chunk_data == NULL) - { - std::cerr << "p3turtle: Warning: failed malloc of " << res_item->chunk_size << " bytes for received data packet." << std::endl ; - return ; - } - memcpy(res_item->chunk_data,item->chunk_data,res_item->chunk_size) ; - - res_item->PeerId(tunnel.local_src) ; - - sendItem(res_item) ; return ; } + + const TurtleFileHashInfo& hash_info(it->second) ; +#ifdef P3TURTLE_DEBUG + std::cerr << " This is an endpoint for this data chunk." << std::endl ; + std::cerr << " Forwarding data to the multiplexer." << std::endl ; + std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; +#endif + //_ft_server->getMultiplexer()->recvData(tunnel.vpid,tunnel.hash,hash_info.size,item->chunk_offset,item->chunk_size,item->chunk_data) ; + vpid = tunnel.vpid ; + hash = tunnel.hash ; + size = hash_info.size ; + + // also update the hash time stamp to show that it's actually being downloaded. + it->second.time_stamp = time(NULL) ; } + _ft_server->getMultiplexer()->recvData(vpid,hash,size,item->chunk_offset,item->chunk_size,item->chunk_data) ; item->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted // down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData } + void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item) { #ifdef P3TURTLE_DEBUG @@ -886,61 +918,42 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item) { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - std::map::iterator it(_local_tunnels.find(item->tunnel_id)) ; + std::map::iterator it2(_local_tunnels.find(item->tunnel_id)) ; - if(it == _local_tunnels.end()) + if(it2 == _local_tunnels.end()) { #ifdef P3TURTLE_DEBUG - std::cerr << "p3turtle: got file map with unknown tunnel id " << (void*)item->tunnel_id << std::endl ; + std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ; #endif return ; } - TurtleTunnel& tunnel(it->second) ; + TurtleTunnel& tunnel(it2->second) ; - // Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete. - tunnel.time_stamp = time(NULL) ; - - // Let's figure out whether this reuqest is for us or not. - - if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer - { - std::map::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ; + std::map::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ; #ifdef P3TURTLE_DEBUG - assert(!tunnel.hash.empty()) ; + assert(!tunnel.hash.empty()) ; #endif - if(it==_incoming_file_hashes.end()) - { -#ifdef P3TURTLE_DEBUG - std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ; -#endif - return ; - } - - const TurtleFileHashInfo& hash_info(it->second) ; -#ifdef P3TURTLE_DEBUG - std::cerr << " This is an endpoint for this file map." << std::endl ; - std::cerr << " Forwarding data to the multiplexer." << std::endl ; - std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; -#endif - // also update the hash time stamp to show that it's actually being downloaded. - it->second.time_stamp = time(NULL) ; - - // we should check that there is no backward call to the turtle router! - // - _ft_server->getMultiplexer()->recvFileMap(tunnel.vpid,tunnel.hash,item->chunk_size,item->nb_chunks,item->compressed_map) ; - } - else // No, it's a request we should forward down the pipe. + if(it==_incoming_file_hashes.end()) { #ifdef P3TURTLE_DEBUG - std::cerr << " Forwarding file map to peer " << tunnel.local_src << std::endl ; + std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ; #endif - RsTurtleFileMapItem *res_item = new RsTurtleFileMapItem(*item) ; - res_item->PeerId(tunnel.local_src) ; - - sendItem(res_item) ; return ; } + + const TurtleFileHashInfo& hash_info(it->second) ; +#ifdef P3TURTLE_DEBUG + std::cerr << " This is an endpoint for this file map." << std::endl ; + std::cerr << " Forwarding data to the multiplexer." << std::endl ; + std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; +#endif + // also update the hash time stamp to show that it's actually being downloaded. + it->second.time_stamp = time(NULL) ; + + // we should check that there is no backward call to the turtle router! + // + _ft_server->getMultiplexer()->recvFileMap(tunnel.vpid,tunnel.hash,item->chunk_size,item->nb_chunks,item->compressed_map) ; } } @@ -962,8 +975,6 @@ void p3turtle::sendDataRequest(const std::string& peerId, const std::string& has TurtleTunnelId tunnel_id = it->second ; TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ; -// tunnel.time_stamp = time(NULL) ; - #ifdef P3TURTLE_DEBUG assert(hash == tunnel.hash) ; #endif @@ -1157,7 +1168,9 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) item->print(std::cerr,0) ; #endif // If the item contains an already handled tunnel request, give up. This - // happens when the same tunnel request gets relayed by different peers + // happens when the same tunnel request gets relayed by different peers. We + // have to be very careful here, not to call ftController while mTurtleMtx is + // locked. // { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ @@ -1169,62 +1182,71 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) #endif return ; } - - // This is a new request. Let's add it to the request map, and forward it to - // open peers. + // This is a new request. Let's add it to the request map, and forward + // it to open peers, while the mutex is locked, so no-one can trigger the + // lock before the data is consistent. TurtleRequestInfo& req( _tunnel_requests_origins[item->request_id] ) ; req.origin = item->PeerId() ; req.time_stamp = time(NULL) ; + } - // If it's not for us, perform a local search. If something found, forward the search result back. + // If it's not for us, perform a local search. If something found, forward the search result back. + // We're off-mutex here. - if(item->PeerId() != mConnMgr->getOwnId()) + bool found = false ; + FileInfo info ; + + if(item->PeerId() != mConnMgr->getOwnId()) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Request not from us. Performing local search" << std::endl ; +#endif + found = (_sharing_strategy != SHARE_FRIENDS_ONLY || item->depth < 2) && performLocalHashSearch(item->file_hash,info) ; + } + + { + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + + if(found) { - FileInfo info ; #ifdef P3TURTLE_DEBUG - std::cerr << " Request not from us. Performing local search" << std::endl ; + std::cerr << " Local hash found. Sending tunnel ok to origin (" << item->PeerId() << ")." << std::endl ; #endif - if((_sharing_strategy != SHARE_FRIENDS_ONLY || item->depth < 2) && performLocalHashSearch(item->file_hash,info)) - { -#ifdef P3TURTLE_DEBUG - std::cerr << " Local hash found. Sending tunnel ok to origin (" << item->PeerId() << ")." << std::endl ; -#endif - // Send back tunnel ok to the same guy - // - RsTurtleTunnelOkItem *res_item = new RsTurtleTunnelOkItem ; + // Send back tunnel ok to the same guy + // + RsTurtleTunnelOkItem *res_item = new RsTurtleTunnelOkItem ; - res_item->request_id = item->request_id ; - res_item->tunnel_id = item->partial_tunnel_id ^ generatePersonalFilePrint(item->file_hash,false) ; - res_item->PeerId(item->PeerId()) ; + res_item->request_id = item->request_id ; + res_item->tunnel_id = item->partial_tunnel_id ^ generatePersonalFilePrint(item->file_hash,false) ; + res_item->PeerId(item->PeerId()) ; - sendItem(res_item) ; + sendItem(res_item) ; - // Note in the tunnels list that we have an ending tunnel here. - TurtleTunnel tt ; - tt.local_src = item->PeerId() ; - tt.hash = item->file_hash ; - tt.local_dst = mConnMgr->getOwnId() ; // this means us - tt.time_stamp = time(NULL) ; + // Note in the tunnels list that we have an ending tunnel here. + TurtleTunnel tt ; + tt.local_src = item->PeerId() ; + tt.hash = item->file_hash ; + tt.local_dst = mConnMgr->getOwnId() ; // this means us + tt.time_stamp = time(NULL) ; - _local_tunnels[res_item->tunnel_id] = tt ; + _local_tunnels[res_item->tunnel_id] = tt ; - // We add a virtual peer for that tunnel+hash combination. - // - addDistantPeer(item->file_hash,res_item->tunnel_id) ; + // We add a virtual peer for that tunnel+hash combination. + // + addDistantPeer(item->file_hash,res_item->tunnel_id) ; - // Store the size of the file, to be able to re-form data requests to the multiplexer. - // - _outgoing_file_hashes[item->file_hash] = info ; + // Store the size of the file, to be able to re-form data requests to the multiplexer. + // + _outgoing_file_hashes[item->file_hash] = info ; - // We return straight, because when something is found, there's no need to digg a tunnel further. - return ; - } -#ifdef P3TURTLE_DEBUG - else - std::cerr << " No hash found locally, or local file not allowed for distant peers. Forwarding. " << std::endl ; -#endif + // We return straight, because when something is found, there's no need to digg a tunnel further. + return ; } +#ifdef P3TURTLE_DEBUG + else + std::cerr << " No hash found locally, or local file not allowed for distant peers. Forwarding. " << std::endl ; +#endif } // If search depth not too large, also forward this search request to all other peers. @@ -1538,6 +1560,9 @@ void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item) rsicontrol->getNotify().notifyTurtleSearchResult(item->request_id,item->result) ; } +/// Warning: this function should never be called while the turtle mutex is locked. +/// Otherwize this is a possible source of cross-lock with the File mutex. +// bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,FileInfo& info) { return rsFiles->FileDetails(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY | RS_FILE_HINTS_DOWNLOAD, info); diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 3d6ed4c0c..b8cd22c5c 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -229,82 +229,93 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f // virtual void monitorFileTunnels(const std::string& name,const std::string& file_hash,uint64_t size) ; - // This should be called when canceling a file download, so that the turtle router stops - // handling tunnels for this file. - // + /// This should be called when canceling a file download, so that the turtle router stops + /// handling tunnels for this file. + /// virtual void stopMonitoringFileTunnels(const std::string& file_hash) ; - // get info about tunnels + /// get info about tunnels virtual void getInfo(std::vector >&, std::vector >&, std::vector >&, std::vector >&) const ; /************* from pqiMonitor *******************/ - // Informs the turtle router that some peers are (dis)connected. This should initiate digging new tunnels, - // and closing other tunnels. - // + /// Informs the turtle router that some peers are (dis)connected. This should initiate digging new tunnels, + /// and closing other tunnels. + /// virtual void statusChange(const std::list &plist); /************* from pqiMonitor *******************/ - // This function does many things: - // - It handles incoming and outgoing packets - // - it sorts search requests and forwards search results upward. - // - it cleans unused (tunnel+search) requests. - // - it maintains the pool of tunnels, for each request file hash. - // + /// This function does many things: + /// - It handles incoming and outgoing packets + /// - it sorts search requests and forwards search results upward. + /// - it cleans unused (tunnel+search) requests. + /// - it maintains the pool of tunnels, for each request file hash. + /// virtual int tick(); -// /************* from ftSearch *******************/ -// // Search function. This function looks into the file hashes currently handled , and sends back info. -// // -// virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const ; - /************* from p3Config *******************/ virtual RsSerialiser *setupSerialiser() ; virtual std::list saveList(bool& cleanup) ; virtual bool loadList(std::list load) ; /************* Communication with ftserver *******************/ - // Does the turtle router manages tunnels to this peer ? (this is not a - // real id, but a fake one, that the turtle router is capable of connecting with a tunnel id). + /// Does the turtle router manages tunnels to this peer ? (this is not a + /// real id, but a fake one, that the turtle router is capable of connecting with a tunnel id). bool isTurtlePeer(const std::string& peer_id) const ; - // Examines the peer id, finds the turtle tunnel in it, and respond yes if the tunnel is ok and operational. + /// Examines the peer id, finds the turtle tunnel in it, and respond yes if the tunnel is ok and operational. bool isOnline(const std::string& peer_id) const ; - // Returns a unique peer id, corresponding to the given tunnel. + /// Returns a unique peer id, corresponding to the given tunnel. std::string getTurtlePeerId(TurtleTunnelId tid) const ; - // returns the list of virtual peers for all tunnels. + /// returns the list of virtual peers for all tunnels. void getVirtualPeersList(std::list& list) ; - // Send a data request into the correct tunnel for the given file hash + /// Send a data request into the correct tunnel for the given file hash void sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) ; - // Send file data into the correct tunnel for the given file hash + /// Send file data into the correct tunnel for the given file hash void sendFileData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) ; private: //--------------------------- Admin/Helper functions -------------------------// - uint32_t generatePersonalFilePrint(const TurtleFileHash&,bool) ; /// Generates a cyphered combination of ownId() and file hash - uint32_t generateRandomRequestId() ; /// Generates a random uint32_t number. + /// Generates a cyphered combination of ownId() and file hash + uint32_t generatePersonalFilePrint(const TurtleFileHash&,bool) ; - void autoWash() ; /// Auto cleaning of unused tunnels, search requests and tunnel requests. + /// Generates a random uint32_t number. + uint32_t generateRandomRequestId() ; + + /// Auto cleaning of unused tunnels, search requests and tunnel requests. + void autoWash() ; //------------------------------ Tunnel handling -----------------------------// - TurtleRequestId diggTunnel(const TurtleFileHash& hash) ; /// initiates tunnels from here to any peers having the given file hash - void addDistantPeer(const TurtleFileHash&, TurtleTunnelId) ; /// adds info related to a new virtual peer. + /// initiates tunnels from here to any peers having the given file hash + TurtleRequestId diggTunnel(const TurtleFileHash& hash) ; + + /// adds info related to a new virtual peer. + void addDistantPeer(const TurtleFileHash&, TurtleTunnelId) ; //----------------------------- Routing functions ----------------------------// - void manageTunnels() ; /// Handle tunnel digging for current file hashes - void locked_closeTunnel(TurtleTunnelId tid) ; /// closes a given tunnel. Should be called with mutex set. - int handleIncoming(); /// Main routing function + /// Handle tunnel digging for current file hashes + void manageTunnels() ; - void handleSearchRequest(RsTurtleSearchRequestItem *item); /// specific routing functions for handling particular packets. + /// closes a given tunnel. Should be called with mutex set. + void locked_closeTunnel(TurtleTunnelId tid) ; + + /// Main routing function + int handleIncoming(); + + /// Generic routing function for all tunnel packets that derive from RsTurtleGenericTunnelItem + void routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) ; + + /// specific routing functions for handling particular packets. + void handleSearchRequest(RsTurtleSearchRequestItem *item); void handleSearchResult(RsTurtleSearchResultItem *item); void handleTunnelRequest(RsTurtleOpenTunnelItem *item); void handleTunnelResult(RsTurtleTunnelOkItem *item); @@ -314,13 +325,13 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f //------ Functions connecting the turtle router to other components.----------// - // Performs a search calling local cache and search structure. + /// Performs a search calling local cache and search structure. void performLocalSearch(const std::string& match_string,std::list& result) ; - // Returns a search result upwards (possibly to the gui) + /// Returns a search result upwards (possibly to the gui) void returnSearchResult(RsTurtleSearchResultItem *item) ; - // Returns true if the file with given hash is hosted locally. + /// Returns true if the file with given hash is hosted locally. bool performLocalHashSearch(const TurtleFileHash& hash,FileInfo& info) ; //--------------------------- Local variables --------------------------------// @@ -332,21 +343,38 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f mutable RsMutex mTurtleMtx; - std::map _search_requests_origins ; /// keeps trace of who emmitted a given search request - std::map _tunnel_requests_origins ; /// keeps trace of who emmitted a tunnel request - std::map _incoming_file_hashes ; /// stores adequate tunnels for each file hash locally managed - std::map _outgoing_file_hashes ; /// stores file info for each file we provide. - std::map _local_tunnels ; /// local tunnels, stored by ids (Either transiting or ending). - std::map _virtual_peers ; /// Peers corresponding to each tunnel. - std::vector _hashes_to_remove ; /// Hashes marked to be deleted. + /// keeps trace of who emmitted a given search request + std::map _search_requests_origins ; + + /// keeps trace of who emmitted a tunnel request + std::map _tunnel_requests_origins ; + + /// stores adequate tunnels for each file hash locally managed + std::map _incoming_file_hashes ; + + /// stores file info for each file we provide. + std::map _outgoing_file_hashes ; + + /// local tunnels, stored by ids (Either transiting or ending). + std::map _local_tunnels ; + + /// Peers corresponding to each tunnel. + std::map _virtual_peers ; + + /// Hashes marked to be deleted. + std::vector _hashes_to_remove ; time_t _last_clean_time ; time_t _last_tunnel_management_time ; time_t _last_tunnel_campaign_time ; std::list _online_peers; - bool _force_digg_new_tunnels ; /// used to force digging new tunnels + + /// used to force digging new tunnels + bool _force_digg_new_tunnels ; + #ifdef P3TURTLE_DEBUG + // debug function void dumpState() ; #endif }; diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index c1a2518b2..37728f58b 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -406,7 +406,7 @@ bool RsTurtleSearchResultItem::serialize(void *data,uint32_t& pktsize) } RsTurtleFileMapItem::RsTurtleFileMapItem(void *data,uint32_t pktsize) - : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP) + : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP) { #ifdef P3TURTLE_DEBUG std::cerr << " type = search result" << std::endl ; @@ -630,7 +630,7 @@ bool RsTurtleFileRequestItem::serialize(void *data,uint32_t& pktsize) } RsTurtleFileRequestItem::RsTurtleFileRequestItem(void *data,uint32_t pktsize) - : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) + : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) { #ifdef P3TURTLE_DEBUG std::cerr << " type = file request" << std::endl ; @@ -662,7 +662,7 @@ RsTurtleFileDataItem::~RsTurtleFileDataItem() free(chunk_data) ; } RsTurtleFileDataItem::RsTurtleFileDataItem(void *data,uint32_t pktsize) - : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_DATA) + : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_DATA) { #ifdef P3TURTLE_DEBUG std::cerr << " type = file request" << std::endl ; diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 340580a02..e5bf36d95 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -36,6 +36,7 @@ class RsTurtleItem: public RsItem /***********************************************************************************/ /* Turtle Search Item classes */ +/* Specific packets */ /***********************************************************************************/ class RsTurtleSearchResultItem: public RsTurtleItem @@ -143,16 +144,48 @@ class RsTurtleTunnelOkItem: public RsTurtleItem virtual uint32_t serial_size() ; }; +/***********************************************************************************/ +/* Generic turtle packets for tunnels */ +/***********************************************************************************/ + +class RsTurtleGenericTunnelItem: public RsTurtleItem +{ + public: + RsTurtleGenericTunnelItem(uint8_t sub_packet_id) : RsTurtleItem(sub_packet_id) {} + + typedef enum { DIRECTION_CLIENT, DIRECTION_SERVER } Direction ; + + /// Does this packet stamps tunnels when it passes through ? + /// This is used for keeping trace weither tunnels are active or not. + + virtual bool shouldStampTunnel() const = 0 ; + + /// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to + /// indicate which tunnel they are travelling through. + + virtual TurtleTunnelId tunnelId() const = 0 ; + + /// Indicate weither the packet is a client packet (goign back to the + /// client) or a server packet (going to the server. Typically file + /// requests are server packets, whereas file data are client packets. + + virtual Direction travelingDirection() const = 0 ; +}; + /***********************************************************************************/ /* Turtle File Transfer item classes */ /***********************************************************************************/ -class RsTurtleFileRequestItem: public RsTurtleItem +class RsTurtleFileRequestItem: public RsTurtleGenericTunnelItem { public: - RsTurtleFileRequestItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) {} + RsTurtleFileRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) {} RsTurtleFileRequestItem(void *data,uint32_t size) ; // deserialization + virtual bool shouldStampTunnel() const { return false ; } + virtual TurtleTunnelId tunnelId() const { return tunnel_id ; } + virtual Direction travelingDirection() const { return DIRECTION_SERVER ; } + uint32_t tunnel_id ; // id of the tunnel to travel through uint64_t chunk_offset ; uint32_t chunk_size ; @@ -163,13 +196,17 @@ class RsTurtleFileRequestItem: public RsTurtleItem virtual uint32_t serial_size() ; }; -class RsTurtleFileDataItem: public RsTurtleItem +class RsTurtleFileDataItem: public RsTurtleGenericTunnelItem { public: - RsTurtleFileDataItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_DATA) {} + RsTurtleFileDataItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_DATA) {} ~RsTurtleFileDataItem() ; RsTurtleFileDataItem(void *data,uint32_t size) ; // deserialization + virtual bool shouldStampTunnel() const { return true ; } + virtual TurtleTunnelId tunnelId() const { return tunnel_id ; } + virtual Direction travelingDirection() const { return DIRECTION_CLIENT ; } + uint32_t tunnel_id ; // id of the tunnel to travel through uint64_t chunk_offset ; // offset in the file uint32_t chunk_size ; // size of the file chunk @@ -181,20 +218,25 @@ class RsTurtleFileDataItem: public RsTurtleItem virtual uint32_t serial_size() ; }; -class RsTurtleFileMapItem: public RsTurtleItem +class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem { public: - RsTurtleFileMapItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP) {} + RsTurtleFileMapItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP) {} RsTurtleFileMapItem(void *data,uint32_t size) ; // deserialization - uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source - uint32_t chunk_size ; // fixed size of chunks, as seen from the source, for the given map. - uint32_t nb_chunks ; // number of chunks in the file. The last two infos are redundant, as we can recompute - // this info from the file size, but this allows a security check. + virtual bool shouldStampTunnel() const { return false ; } + virtual TurtleTunnelId tunnelId() const { return tunnel_id ; } + virtual Direction travelingDirection() const { return direction ; } + + Direction direction ; // travel direction for this packet (server/client) + uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source + uint32_t chunk_size ; // fixed size of chunks, as seen from the source, for the given map. + uint32_t nb_chunks ; // number of chunks in the file. The last two infos are redundant, as we can recompute + // this info from the file size, but this allows a security check. std::vector compressed_map ; // Map info for the file in compressed format. Each *bit* in the array uint's says "I have" or "I don't have" - // by default, we suppose the peer has all the chunks. This info will thus be and-ed - // with the default file map for this source. + // by default, we suppose the peer has all the chunks. This info will thus be and-ed + // with the default file map for this source. virtual std::ostream& print(std::ostream& o, uint16_t) ;