- added a generic RsItem to the turtle router and the methods to route it. This makes the code much more elegant.

- suppressed a cross mutex lock bug that happened rarely while digging tunnels
- changed FileDetails in ftServer so that it's now possiblt to search for hashes of files being downloaded
- improved the search code in ftdatamultiplex
- added some comments to the turtle code



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1964 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftdatamultiplex.cc

@@ -127,21 +127,9 @@ bool    ftDataMultiplex::FileDetails(std::string hash, uint32_t hintsflag, FileI
 #endif
 
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
-	std::map<std::string, ftFileProvider *>::iterator sit;
+
-	sit = mServers.find(hash);
+	if(hintsflag & RS_FILE_HINTS_DOWNLOAD)
-	if (sit != mServers.end())
 	{
-
-#ifdef MPLEX_DEBUG
-		std::cerr << "ftDataMultiplex::FileDetails()";
-		std::cerr << " Found ftFileProvider!";
-		std::cerr << std::endl;
-#endif
-
-		(sit->second)->FileDetails(info);
-		return true;
-	}
-
 		std::map<std::string, ftClient>::iterator cit;
 		if (mClients.end() != (cit = mClients.find(hash)))
 		{
@@ -156,6 +144,26 @@ bool    ftDataMultiplex::FileDetails(std::string hash, uint32_t hintsflag, FileI
 			(cit->second).mCreator->FileDetails(info);
 			return true;
 		}
+	}
+
+	if(hintsflag & RS_FILE_HINTS_UPLOAD)
+	{
+		std::map<std::string, ftFileProvider *>::iterator sit;
+		sit = mServers.find(hash);
+		if (sit != mServers.end())
+		{
+
+#ifdef MPLEX_DEBUG
+			std::cerr << "ftDataMultiplex::FileDetails()";
+			std::cerr << " Found ftFileProvider!";
+			std::cerr << std::endl;
+#endif
+
+			(sit->second)->FileDetails(info);
+			return true;
+		}
+	}
+
 
 #ifdef MPLEX_DEBUG
 	std::cerr << "ftDataMultiplex::FileDetails()";

libretroshare/src/ft/ftfilecreator.cc

@@ -34,15 +34,6 @@ ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash,
 #endif
 
 	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
-
-	/* initialise the Transfer Lists */
-//	mStart = recvd;
-//	mEnd = recvd;
-
-#ifdef TO_DO
-	// we should init the chunk map with some bit array saying what is received and what is not!!
-	chunkMap.setTotalReceived(recvd) ;
-#endif
 }
 
 bool ftFileCreator::getFileData(uint64_t offset, uint32_t &chunk_size, void *data)
@@ -423,6 +414,13 @@ void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size,
 {
 	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
+	// At this point, we should cancel all file chunks that are asked to the
+	// peer and which this peer actually doesn't possesses. Otherwise, the transfer may get stuck. 
+	// This should be done by:
+	// 	- first setting the peer availability map
+	// 	- then asking the chunkmap which chunks are being downloaded, but actually shouldn't
+	// 	- cancelling them in the ftFileCreator, so that they can be re-asked later to another peer.
+	//
 	chunkMap.setPeerAvailabilityMap(peer_id,chunk_size,nb_chunks,compressed_map) ;
 }
 

libretroshare/src/ft/ftserver.cc

@@ -367,24 +367,19 @@ bool ftServer::FileUploads(std::list<std::string> &hashs)
 
 bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info)
 {
-	bool found = false;
 	if (hintflags & RS_FILE_HINTS_DOWNLOAD)
-	{
+		if(mFtController->FileDetails(hash, info))
-		//found = mFtDataplex->FileDetails(hash, hintflags, info);
+			return true ;
-		//
-		// Use Controller for download searches.
-		found = mFtController->FileDetails(hash, info);
-	}
-	else if (hintflags & RS_FILE_HINTS_UPLOAD)
-	{
-		found = mFtDataplex->FileDetails(hash, hintflags, info);
-	}
 
-	if (!found)
+	if(hintflags & RS_FILE_HINTS_UPLOAD)
-	{
+		if(mFtDataplex->FileDetails(hash, hintflags, info))
-		found = mFtSearch->search(hash, 0, hintflags, info);
+			return true ;
-	}
+
-	return found;
+	if(hintflags & ~(RS_FILE_HINTS_UPLOAD | RS_FILE_HINTS_DOWNLOAD)) 
+		if(mFtSearch->search(hash, 0, hintflags, info))
+			return true ;
+
+	return false;
 }
 
 	/***************************************************************/

libretroshare/src/turtle/p3turtle.cc

@@ -515,11 +515,6 @@ uint32_t p3turtle::generatePersonalFilePrint(const TurtleFileHash& hash,bool b)
 //
 int p3turtle::handleIncoming()
 {
-#ifdef P3TURTLE_DEBUG
-//	std::cerr << "p3turtle::handleIncoming()";
-//	std::cerr << std::endl;
-#endif
-
 	int nhandled = 0;
 	// While messages read
 	//
@@ -529,6 +524,15 @@ int p3turtle::handleIncoming()
 	{
 		nhandled++;
 
+		RsTurtleGenericTunnelItem *gti = dynamic_cast<RsTurtleGenericTunnelItem *>(item) ;
+
+		if(gti != NULL)
+			routeGenericTunnelItem(gti) ;	/// Generic packets, that travel through established tunnels. 
+		else			 							/// These packets should be destroyed by the client.
+		{
+			/// Special packets that require specific treatment, because tunnels do not exist for these packets.
+			/// These packets are destroyed here, after treatment.
+			//
 			switch(item->PacketSubType())
 			{
 				case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST:
@@ -543,19 +547,12 @@ int p3turtle::handleIncoming()
 
 				case RS_TURTLE_SUBTYPE_TUNNEL_OK     : handleTunnelResult(dynamic_cast<RsTurtleTunnelOkItem *>(item)) ;
 																	break ;
-
-			case RS_TURTLE_SUBTYPE_FILE_REQUEST  : handleRecvFileRequest(dynamic_cast<RsTurtleFileRequestItem *>(item)) ;
-																break ;
-
-			case RS_TURTLE_SUBTYPE_FILE_DATA     : handleRecvFileData(dynamic_cast<RsTurtleFileDataItem *>(item)) ;
-																break ;
-
-			// Here will also come handling of file transfer requests, tunnel digging/closing, etc.
 				default:
 																	std::cerr << "p3turtle::handleIncoming: Unknown packet subtype " << item->PacketSubType() << std::endl ;
 			}
 			delete item;
 		}
+	}
 
 	return nhandled;
 }
@@ -725,6 +722,88 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 // ---------------------------------  File Transfer. -------------------------------- //
 // -----------------------------------------------------------------------------------//
 
+// Routing of turtle tunnel items in a generic manner. Most tunnel packets will use this function, except packets designed for 
+// contructing the tunnels and searching, namely TurtleSearchRequests/Results and OpenTunnel/TunnelOkItems
+//
+void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
+{
+#ifdef P3TURTLE_DEBUG
+	std::cerr << "p3Turtle: treating generic tunnel item:" << std::endl ;
+	item->print(std::cerr,1) ;
+#endif
+	RsTurtleGenericTunnelItem::Direction direction = item->travelingDirection() ;
+
+	{
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+
+		// look for the tunnel id.
+		//
+		std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(item->tunnelId())) ;
+
+		if(it == _local_tunnels.end())
+		{
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "p3turtle: got file map with unknown tunnel id " << (void*)item->tunnelId() << std::endl ;
+#endif
+			return ;
+		}
+
+		TurtleTunnel& tunnel(it->second) ;
+
+		// Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete.
+		if(item->shouldStampTunnel())
+			tunnel.time_stamp = time(NULL) ;
+
+		// Let's figure out whether this packet is for us or not.
+
+		if(direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT && tunnel.local_src != mConnMgr->getOwnId())
+		{														 	
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "  Forwarding generic item to peer " << tunnel.local_src << std::endl ;
+#endif
+			item->PeerId(tunnel.local_src) ;
+
+			sendItem(item) ;
+			return ;
+		}
+
+		if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER && tunnel.local_dst != mConnMgr->getOwnId())
+		{
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "  Forwarding generic item to peer " << tunnel.local_dst << std::endl ;
+#endif
+			item->PeerId(tunnel.local_dst) ;
+
+			sendItem(item) ;
+			return ;
+		}
+	}
+
+	// The packet was not forwarded, so it is for us. Let's treat it.
+	// This is done off-mutex, to avoid various deadlocks
+	//
+	if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER)
+		switch(item->PacketSubType())
+		{
+			case RS_TURTLE_SUBTYPE_FILE_REQUEST: handleRecvFileRequest(dynamic_cast<RsTurtleFileRequestItem *>(item)) ;
+															 break ;
+			default:
+															 std::cerr << "Unknown server packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ;
+															 exit(-1) ;
+		}
+
+	if(direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT)
+		switch(item->PacketSubType())
+		{
+			case RS_TURTLE_SUBTYPE_FILE_DATA : handleRecvFileData(dynamic_cast<RsTurtleFileDataItem *>(item)) ;
+														  break ;
+			default:
+														  std::cerr << "Unknown client packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ;
+														  exit(-1) ;
+		}
+
+	delete item ;
+}
 
 void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item)
 {
@@ -742,9 +821,9 @@ void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item)
 	{
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 
-		std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(item->tunnel_id)) ;
+		std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
 
-		if(it == _local_tunnels.end())
+		if(it2 == _local_tunnels.end())
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "p3turtle: got file request with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
@@ -752,12 +831,7 @@ void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item)
 			return ;
 		}
 
-		TurtleTunnel& tunnel(it->second) ;
+		TurtleTunnel& tunnel(it2->second) ;
-
-		// Let's figure out whether this reuqest is for us or not.
-
-		if(tunnel.local_dst == mConnMgr->getOwnId()) // Yes, we have to pass on the request to the data multiplexer
-		{
 		std::map<TurtleFileHash,FileInfo>::const_iterator it(_outgoing_file_hashes.find(tunnel.hash)) ;
 #ifdef P3TURTLE_DEBUG
 		assert(!tunnel.hash.empty()) ;
@@ -767,22 +841,10 @@ void p3turtle::handleRecvFileRequest(RsTurtleFileRequestItem *item)
 		std::cerr << "  Forwarding data request to the multiplexer." << std::endl ;
 		std::cerr << "  using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
 #endif
-			// _ft_server->getMultiplexer()->recvDataRequest(tunnel.vpid,tunnel.hash,it->second.size,item->chunk_offset,item->chunk_size) ;
-			//
 		size = it->second.size ;
 		vpid = tunnel.vpid ;
 		hash = tunnel.hash ;
 	}
-		else	// No, it's a request we should forward down the pipe.
-		{
-			RsTurtleFileRequestItem *res_item = new RsTurtleFileRequestItem(*item) ;
-
-			res_item->PeerId(tunnel.local_dst) ;
-
-			sendItem(res_item) ;
-			return ;
-		}
-	}
 
 	// This call is voluntarily off-mutex gards because it can cause cross mutex locks with the multiplexer.
 	// (Yeah, this bug was a shity hard one to catch).
@@ -796,18 +858,16 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
 	std::cerr << "p3Turtle: received file data item:" << std::endl ;
 	item->print(std::cerr,1) ;
 #endif
-	// This is a new request. Let's add it to the request map, and forward it to
-	// open peers.
-
 	TurtleVirtualPeerId vpid ;
 	uint64_t size ;
 	TurtleFileHash hash ;
+
 	{
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 
-		std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(item->tunnel_id)) ;
+		std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
 
-		if(it == _local_tunnels.end())
+		if(it2 == _local_tunnels.end())
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
@@ -815,15 +875,7 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
 			return ;
 		}
 
-		TurtleTunnel& tunnel(it->second) ;
+		TurtleTunnel& tunnel(it2->second) ;
-
-		// Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete.
-		tunnel.time_stamp = time(NULL) ;
-
-		// Let's figure out whether this reuqest is for us or not.
-
-		if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer
-		{
 		std::map<TurtleFileHash,TurtleFileHashInfo>::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
 #ifdef P3TURTLE_DEBUG
 		assert(!tunnel.hash.empty()) ;
@@ -850,33 +902,13 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
 		// also update the hash time stamp to show that it's actually being downloaded.
 		it->second.time_stamp = time(NULL) ;
 	}
-		else	// No, it's a request we should forward down the pipe.
-		{
-#ifdef P3TURTLE_DEBUG
-			std::cerr << "  Forwarding data chunk to peer " << tunnel.local_src << std::endl ;
-#endif
-			RsTurtleFileDataItem *res_item = new RsTurtleFileDataItem(*item) ;
 
-			res_item->chunk_data = malloc(res_item->chunk_size) ;
-
-			if(res_item->chunk_data == NULL)
-			{
-				std::cerr << "p3turtle: Warning: failed malloc of " << res_item->chunk_size << " bytes for received data packet." << std::endl ;
-				return ;
-			}
-			memcpy(res_item->chunk_data,item->chunk_data,res_item->chunk_size) ;
-
-			res_item->PeerId(tunnel.local_src) ;
-
-			sendItem(res_item) ;
-			return ;
-		}
-	}
 	_ft_server->getMultiplexer()->recvData(vpid,hash,size,item->chunk_offset,item->chunk_size,item->chunk_data) ;
 	item->chunk_data = NULL ;	// this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
 										// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
 }
 
+
 void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
 {
 #ifdef P3TURTLE_DEBUG
@@ -886,25 +918,18 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
 	{
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 
-		std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(item->tunnel_id)) ;
+		std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
 
-		if(it == _local_tunnels.end())
+		if(it2 == _local_tunnels.end())
 		{
 #ifdef P3TURTLE_DEBUG
-			std::cerr << "p3turtle: got file map with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
+			std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
 #endif
 			return ;
 		}
 
-		TurtleTunnel& tunnel(it->second) ;
+		TurtleTunnel& tunnel(it2->second) ;
 
-		// Only file data transfer updates tunnels time_stamp field, to avoid maintaining tunnel that are incomplete.
-		tunnel.time_stamp = time(NULL) ;
-
-		// Let's figure out whether this reuqest is for us or not.
-
-		if(tunnel.local_src == mConnMgr->getOwnId()) // Yes, we have to pass on the data to the multiplexer
-		{
 		std::map<TurtleFileHash,TurtleFileHashInfo>::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
 #ifdef P3TURTLE_DEBUG
 		assert(!tunnel.hash.empty()) ;
@@ -930,18 +955,6 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
 		//
 		_ft_server->getMultiplexer()->recvFileMap(tunnel.vpid,tunnel.hash,item->chunk_size,item->nb_chunks,item->compressed_map) ;
 	}
-		else	// No, it's a request we should forward down the pipe.
-		{
-#ifdef P3TURTLE_DEBUG
-			std::cerr << "  Forwarding file map to peer " << tunnel.local_src << std::endl ;
-#endif
-			RsTurtleFileMapItem *res_item = new RsTurtleFileMapItem(*item) ;
-			res_item->PeerId(tunnel.local_src) ;
-
-			sendItem(res_item) ;
-			return ;
-		}
-	}
 }
 
 // Send a data request into the correct tunnel for the given file hash
@@ -962,8 +975,6 @@ void p3turtle::sendDataRequest(const std::string& peerId, const std::string& has
 	TurtleTunnelId tunnel_id = it->second ;
 	TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
 
-//	tunnel.time_stamp = time(NULL) ;
-
 #ifdef P3TURTLE_DEBUG
 	assert(hash == tunnel.hash) ;
 #endif
@@ -1157,7 +1168,9 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 	item->print(std::cerr,0) ;
 #endif
 	// If the item contains an already handled tunnel request, give up.  This
-	// happens when the same tunnel request gets relayed by different peers
+	// happens when the same tunnel request gets relayed by different peers.  We
+	// have to be very careful here, not to call ftController while mTurtleMtx is
+	// locked.
 	//
 	{
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
@@ -1169,23 +1182,33 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 #endif
 			return ;
 		}
-
+		// This is a new request. Let's add it to the request map, and forward
-		// This is a new request. Let's add it to the request map, and forward it to
+		// it to open peers, while the mutex is locked, so no-one can trigger the
-		// open peers.
+		// lock before the data is consistent.
 
 		TurtleRequestInfo& req( _tunnel_requests_origins[item->request_id] ) ;
 		req.origin = item->PeerId() ;
 		req.time_stamp = time(NULL) ;
+	}
 
 	// If it's not for us, perform a local search. If something found, forward the search result back.
+	// We're off-mutex here.
+
+	bool found = false ;
+	FileInfo info ;
 
 	if(item->PeerId() != mConnMgr->getOwnId())
 	{
-			FileInfo info ;
 #ifdef P3TURTLE_DEBUG
 		std::cerr << "  Request not from us. Performing local search" << std::endl ;
 #endif
-			if((_sharing_strategy != SHARE_FRIENDS_ONLY || item->depth < 2) && performLocalHashSearch(item->file_hash,info))
+		found = (_sharing_strategy != SHARE_FRIENDS_ONLY || item->depth < 2) && performLocalHashSearch(item->file_hash,info) ;
+	}
+
+	{
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+
+		if(found)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Local hash found. Sending tunnel ok to origin (" << item->PeerId() << ")." << std::endl ;
@@ -1225,7 +1248,6 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 			std::cerr << "  No hash found locally, or local file not allowed for distant peers. Forwarding. " << std::endl ;
 #endif
 	}
-	}
 
 	// If search depth not too large, also forward this search request to all other peers.
 	//
@@ -1538,6 +1560,9 @@ void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item)
 	rsicontrol->getNotify().notifyTurtleSearchResult(item->request_id,item->result) ;
 }
 
+/// Warning: this function should never be called while the turtle mutex is locked.
+/// Otherwize this is a possible source of cross-lock with the File mutex.
+//
 bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,FileInfo& info)
 {
 	return rsFiles->FileDetails(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY | RS_FILE_HINTS_DOWNLOAD, info);

libretroshare/src/turtle/p3turtle.h

@@ -229,82 +229,93 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
 		//
 		virtual void monitorFileTunnels(const std::string& name,const std::string& file_hash,uint64_t size) ;
 
-		// This should be called when canceling a file download, so that the turtle router stops
+		/// This should be called when canceling a file download, so that the turtle router stops
-		// handling tunnels for this file.
+		/// handling tunnels for this file.
-		//
+		///
 		virtual void stopMonitoringFileTunnels(const std::string& file_hash) ;
 
-		// get info about tunnels
+		/// get info about tunnels
 		virtual void getInfo(std::vector<std::vector<std::string> >&,
 									std::vector<std::vector<std::string> >&,
 									std::vector<std::vector<std::string> >&,
 									std::vector<std::vector<std::string> >&) const ;
 		
 		/************* from pqiMonitor *******************/
-		// Informs the turtle router that some peers are (dis)connected. This should initiate digging new tunnels,
+		/// Informs the turtle router that some peers are (dis)connected. This should initiate digging new tunnels,
-		// and closing other tunnels.
+		/// and closing other tunnels.
-		//
+		///
 		virtual void statusChange(const std::list<pqipeer> &plist);
 
 		/************* from pqiMonitor *******************/
 
-		// This function does many things:
+		/// This function does many things:
-		// 	- It handles incoming and outgoing packets
+		/// 	- It handles incoming and outgoing packets
-		// 	- it sorts search requests and forwards search results upward.
+		/// 	- it sorts search requests and forwards search results upward.
-		// 	- it cleans unused (tunnel+search) requests.
+		/// 	- it cleans unused (tunnel+search) requests.
-		// 	- it maintains the pool of tunnels, for each request file hash.
+		/// 	- it maintains the pool of tunnels, for each request file hash.
-		//
+		///
 		virtual int tick();
 
-//		/************* from ftSearch *******************/
-//		// Search function. This function looks into the file hashes currently handled , and sends back info.
-//		//
-//		virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const ;
-
 		/************* from p3Config *******************/
 		virtual RsSerialiser *setupSerialiser() ;
 		virtual std::list<RsItem*> saveList(bool& cleanup) ;
 		virtual bool loadList(std::list<RsItem*> load) ;
 
 		/************* Communication with ftserver *******************/
-		// Does the turtle router manages tunnels to this peer ? (this is not a
+		/// Does the turtle router manages tunnels to this peer ? (this is not a
-		// real id, but a fake one, that the turtle router is capable of connecting with a tunnel id).
+		/// real id, but a fake one, that the turtle router is capable of connecting with a tunnel id).
 		bool isTurtlePeer(const std::string& peer_id) const ;
 
-		// Examines the peer id, finds the turtle tunnel in it, and respond yes if the tunnel is ok and operational.
+		/// Examines the peer id, finds the turtle tunnel in it, and respond yes if the tunnel is ok and operational.
 		bool isOnline(const std::string& peer_id) const ;
 
-		// Returns a unique peer id, corresponding to the given tunnel.
+		/// Returns a unique peer id, corresponding to the given tunnel.
 		std::string getTurtlePeerId(TurtleTunnelId tid) const ;
 	
-		// returns the list of virtual peers for all tunnels.
+		/// returns the list of virtual peers for all tunnels.
 		void getVirtualPeersList(std::list<pqipeer>& list) ;
 
-		// Send a data request into the correct tunnel for the given file hash
+		/// Send a data request into the correct tunnel for the given file hash
 		void sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) ;
 
-		// Send file data into the correct tunnel for the given file hash
+		/// Send file data into the correct tunnel for the given file hash
 		void sendFileData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) ;
 	private:
 		//--------------------------- Admin/Helper functions -------------------------//
 		
-		uint32_t generatePersonalFilePrint(const TurtleFileHash&,bool) ;	/// Generates a cyphered combination of ownId() and file hash
+		/// Generates a cyphered combination of ownId() and file hash
-		uint32_t generateRandomRequestId() ;								/// Generates a random uint32_t number.
+		uint32_t generatePersonalFilePrint(const TurtleFileHash&,bool) ;	
 
-		void autoWash() ;															/// Auto cleaning of unused tunnels, search requests and tunnel requests.
+		/// Generates a random uint32_t number.
+		uint32_t generateRandomRequestId() ;								
+
+		/// Auto cleaning of unused tunnels, search requests and tunnel requests.
+		void autoWash() ;															
 
 		//------------------------------ Tunnel handling -----------------------------//
 
-		TurtleRequestId diggTunnel(const TurtleFileHash& hash) ;	/// initiates tunnels from here to any peers having the given file hash
+		/// initiates tunnels from here to any peers having the given file hash
-		void addDistantPeer(const TurtleFileHash&, TurtleTunnelId) ;	/// adds info related to a new virtual peer.
+		TurtleRequestId diggTunnel(const TurtleFileHash& hash) ;	
+
+		/// adds info related to a new virtual peer.
+		void addDistantPeer(const TurtleFileHash&, TurtleTunnelId) ;	
 
 		//----------------------------- Routing functions ----------------------------//
 		
-		void manageTunnels() ;									/// Handle tunnel digging for current file hashes
+		/// Handle tunnel digging for current file hashes
-		void locked_closeTunnel(TurtleTunnelId tid) ;	/// closes a given tunnel. Should be called with mutex set.
+		void manageTunnels() ;									
-		int handleIncoming(); 									/// Main routing function
 
-		void handleSearchRequest(RsTurtleSearchRequestItem *item);		/// specific routing functions for handling particular packets.
+		/// closes a given tunnel. Should be called with mutex set.
+		void locked_closeTunnel(TurtleTunnelId tid) ;	
+
+		/// Main routing function
+		int handleIncoming(); 									
+
+		/// Generic routing function for all tunnel packets that derive from RsTurtleGenericTunnelItem
+		void routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) ;
+
+		/// specific routing functions for handling particular packets.
+		void handleSearchRequest(RsTurtleSearchRequestItem *item);		
 		void handleSearchResult(RsTurtleSearchResultItem *item);
 		void handleTunnelRequest(RsTurtleOpenTunnelItem *item);		
 		void handleTunnelResult(RsTurtleTunnelOkItem *item);		
@@ -314,13 +325,13 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
 
 		//------ Functions connecting the turtle router to other components.----------//
 		
-		// Performs a search calling local cache and search structure.
+		/// Performs a search calling local cache and search structure.
 		void performLocalSearch(const std::string& match_string,std::list<TurtleFileInfo>& result) ;
 
-		// Returns a search result upwards (possibly to the gui)
+		/// Returns a search result upwards (possibly to the gui)
 		void returnSearchResult(RsTurtleSearchResultItem *item) ;
 
-		// Returns true if the file with given hash is hosted locally.
+		/// Returns true if the file with given hash is hosted locally.
 		bool performLocalHashSearch(const TurtleFileHash& hash,FileInfo& info) ;
 
 		//--------------------------- Local variables --------------------------------//
@@ -332,21 +343,38 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
 
 		mutable RsMutex mTurtleMtx;
 
-		std::map<TurtleSearchRequestId,TurtleRequestInfo> 	_search_requests_origins ; /// keeps trace of who emmitted a given search request
+		/// keeps trace of who emmitted a given search request
-		std::map<TurtleTunnelRequestId,TurtleRequestInfo> 	_tunnel_requests_origins ; /// keeps trace of who emmitted a tunnel request
+		std::map<TurtleSearchRequestId,TurtleRequestInfo> 	_search_requests_origins ; 
-		std::map<TurtleFileHash,TurtleFileHashInfo>			_incoming_file_hashes ;		/// stores adequate tunnels for each file hash locally managed
+
-		std::map<TurtleFileHash,FileInfo>						_outgoing_file_hashes ;		/// stores file info for each file we provide.
+		/// keeps trace of who emmitted a tunnel request
-		std::map<TurtleTunnelId,TurtleTunnel > 				_local_tunnels ;				/// local tunnels, stored by ids (Either transiting or ending).
+		std::map<TurtleTunnelRequestId,TurtleRequestInfo> 	_tunnel_requests_origins ; 
-		std::map<TurtleVirtualPeerId,TurtleTunnelId>			_virtual_peers ;				/// Peers corresponding to each tunnel.
+
-		std::vector<TurtleFileHash>								_hashes_to_remove ;			/// Hashes marked to be deleted.
+		/// stores adequate tunnels for each file hash locally managed
+		std::map<TurtleFileHash,TurtleFileHashInfo>			_incoming_file_hashes ;		
+
+		/// stores file info for each file we provide.
+		std::map<TurtleFileHash,FileInfo>						_outgoing_file_hashes ;		
+
+		/// local tunnels, stored by ids (Either transiting or ending).
+		std::map<TurtleTunnelId,TurtleTunnel > 				_local_tunnels ;				
+
+		/// Peers corresponding to each tunnel.
+		std::map<TurtleVirtualPeerId,TurtleTunnelId>			_virtual_peers ;				
+
+		/// Hashes marked to be deleted.
+		std::vector<TurtleFileHash>								_hashes_to_remove ;			
 
 		time_t _last_clean_time ;
 		time_t _last_tunnel_management_time ;
 		time_t _last_tunnel_campaign_time ;
 
 		std::list<pqipeer> _online_peers;
-		bool _force_digg_new_tunnels ;			/// used to force digging new tunnels
+
+		/// used to force digging new tunnels
+		bool _force_digg_new_tunnels ;			
+
 #ifdef P3TURTLE_DEBUG
+		// debug function
 		void dumpState() ;
 #endif
 };

libretroshare/src/turtle/rsturtleitem.cc

@@ -406,7 +406,7 @@ bool RsTurtleSearchResultItem::serialize(void *data,uint32_t& pktsize)
 }
 
 RsTurtleFileMapItem::RsTurtleFileMapItem(void *data,uint32_t pktsize)
-	: RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP)
+	: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP)
 {
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  type = search result" << std::endl ;
@@ -630,7 +630,7 @@ bool RsTurtleFileRequestItem::serialize(void *data,uint32_t& pktsize)
 }
 
 RsTurtleFileRequestItem::RsTurtleFileRequestItem(void *data,uint32_t pktsize)
-	: RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_REQUEST)
+	: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_REQUEST)
 {
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  type = file request" << std::endl ;
@@ -662,7 +662,7 @@ RsTurtleFileDataItem::~RsTurtleFileDataItem()
 	free(chunk_data) ;
 }
 RsTurtleFileDataItem::RsTurtleFileDataItem(void *data,uint32_t pktsize)
-	: RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_DATA)
+	: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_DATA)
 {
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  type = file request" << std::endl ;

libretroshare/src/turtle/rsturtleitem.h

@@ -36,6 +36,7 @@ class RsTurtleItem: public RsItem
 
 /***********************************************************************************/
 /*                           Turtle Search Item classes                            */
+/*                                Specific packets                                 */
 /***********************************************************************************/
 
 class RsTurtleSearchResultItem: public RsTurtleItem
@@ -143,16 +144,48 @@ class RsTurtleTunnelOkItem: public RsTurtleItem
 		virtual uint32_t serial_size() ; 
 };
 
+/***********************************************************************************/
+/*                           Generic turtle packets for tunnels                    */
+/***********************************************************************************/
+
+class RsTurtleGenericTunnelItem: public RsTurtleItem
+{
+	public:
+		RsTurtleGenericTunnelItem(uint8_t sub_packet_id) : RsTurtleItem(sub_packet_id) {}
+
+		typedef enum { DIRECTION_CLIENT, DIRECTION_SERVER } Direction ;
+
+		/// Does this packet stamps tunnels when it passes through ?
+		/// This is used for keeping trace weither tunnels are active or not.
+		
+		virtual bool shouldStampTunnel() const = 0 ;
+
+		/// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to 
+		/// indicate which tunnel they are travelling through.
+		
+		virtual TurtleTunnelId tunnelId() const = 0 ;
+
+		/// Indicate weither the packet is a client packet (goign back to the
+		/// client) or a server packet (going to the server. Typically file
+		/// requests are server packets, whereas file data are client packets.
+		
+		virtual Direction travelingDirection() const = 0 ;
+};
+
 /***********************************************************************************/
 /*                           Turtle File Transfer item classes                     */
 /***********************************************************************************/
 
-class RsTurtleFileRequestItem: public RsTurtleItem
+class RsTurtleFileRequestItem: public RsTurtleGenericTunnelItem
 {
 	public:
-		RsTurtleFileRequestItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) {}
+		RsTurtleFileRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_REQUEST) {}
 		RsTurtleFileRequestItem(void *data,uint32_t size) ;		// deserialization
 
+		virtual bool shouldStampTunnel() const { return false ; }
+		virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
+		virtual Direction travelingDirection() const { return DIRECTION_SERVER ; }
+
 		uint32_t tunnel_id ;		// id of the tunnel to travel through
 		uint64_t chunk_offset ;
 		uint32_t chunk_size ;
@@ -163,13 +196,17 @@ class RsTurtleFileRequestItem: public RsTurtleItem
 		virtual uint32_t serial_size() ; 
 };
 
-class RsTurtleFileDataItem: public RsTurtleItem
+class RsTurtleFileDataItem: public RsTurtleGenericTunnelItem
 {
 	public:
-		RsTurtleFileDataItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_DATA) {}
+		RsTurtleFileDataItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_DATA) {}
 		~RsTurtleFileDataItem() ;
 		RsTurtleFileDataItem(void *data,uint32_t size) ;		// deserialization
 
+		virtual bool shouldStampTunnel() const { return true ; }
+		virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
+		virtual Direction travelingDirection() const { return DIRECTION_CLIENT ; }
+
 		uint32_t tunnel_id ;		// id of the tunnel to travel through
 		uint64_t chunk_offset ;	// offset in the file
 		uint32_t chunk_size ;	// size of the file chunk
@@ -181,12 +218,17 @@ class RsTurtleFileDataItem: public RsTurtleItem
 		virtual uint32_t serial_size() ; 
 };
 
-class RsTurtleFileMapItem: public RsTurtleItem			
+class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem			
 {
 	public:
-		RsTurtleFileMapItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_FILE_MAP) {}
+		RsTurtleFileMapItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP) {}
 		RsTurtleFileMapItem(void *data,uint32_t size) ;		// deserialization
 
+		virtual bool shouldStampTunnel() const { return false ; }
+		virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
+		virtual Direction travelingDirection() const { return direction ; }
+
+		Direction direction ;	// travel direction for this packet (server/client)
 		uint32_t tunnel_id ;		// id of the tunnel to travel through. Also used for identifying the file source
 		uint32_t chunk_size ;	// fixed size of chunks, as seen from the source, for the given map.
 		uint32_t nb_chunks ;		// number of chunks in the file.	The last two infos are redundant, as we can recompute