added grouter as a msg sending service. Not enable yet. Added serialisation for grouter items and test methods in tests/serialiser/. Added saveList/loadList for grouter. set connectToTurtleRouter() and connectToGRouter() to be mandatory methods

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6970 b45a01b8-16f6-495d-af2f-9b41ad6348cc
2025-05-02 06:06:10 -04:00 · 2013-12-27 20:06:47 +00:00 · 2013-12-27 20:06:47 +00:00 · 5c52890ad5
commit 5c52890ad5
parent da1b6ac845
17 changed files with 784 additions and 80 deletions
--- a/libretroshare/src/grouter/grouteritems.cc
+++ b/libretroshare/src/grouter/grouteritems.cc
@ -16,8 +16,6 @@ bool RsGRouterItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvs

 	pktsize = tlvsize;

-	bool ok = true;
-
 	if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize))
 	{
 		std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl;
@ -86,6 +84,9 @@ RsItem *RsGRouterSerialiser::deserialise(void *data, uint32_t *pktsize)
 		case RS_PKT_SUBTYPE_GROUTER_PUBLISH_KEY:  return deserialise_RsGRouterPublishKeyItem(data, *pktsize);
 		case RS_PKT_SUBTYPE_GROUTER_DATA:   		return deserialise_RsGRouterGenericDataItem(data, *pktsize);
 		case RS_PKT_SUBTYPE_GROUTER_ACK:    		return deserialise_RsGRouterACKItem(data, *pktsize);
+		case RS_PKT_SUBTYPE_GROUTER_MATRIX_CLUES:	return deserialise_RsGRouterMatrixCluesItem(data, *pktsize);
+		case RS_PKT_SUBTYPE_GROUTER_FRIENDS_LIST:	return deserialise_RsGRouterMatrixFriendListItem(data, *pktsize);
+		case RS_PKT_SUBTYPE_GROUTER_ROUTING_INFO:	return deserialise_RsGRouterRoutingInfoItem(data, *pktsize);
 		default:
 				std::cerr << "RsGRouterSerialiser::deserialise(): Could not de-serialise item. SubPacket id = " << std::hex << getRsItemSubType(rstype) << " id = " << rstype << std::dec << std::endl;
 			return NULL;
@ -166,10 +167,116 @@ RsGRouterItem *RsGRouterSerialiser::deserialise_RsGRouterACKItem(void *data, uin
 	return item;
 }

+RsGRouterItem *RsGRouterSerialiser::deserialise_RsGRouterRoutingInfoItem(void *data, uint32_t pktsize) const
+{
+	uint32_t offset = 8; // skip the header 
+	uint32_t rssize = getRsItemSize(data);
+	bool ok = true ;
+
+	RsGRouterRoutingInfoItem *item = new RsGRouterRoutingInfoItem() ;
+
+	ok &= getRawUInt32(data, pktsize, &offset, &item->status_flags); 	
+	ok &= getRawSSLId(data, pktsize, &offset, item->origin); 	
+	ok &= getRawTimeT(data, pktsize, &offset, item->received_time); 	
+
+	uint32_t s = 0 ;
+	ok &= getRawUInt32(data, pktsize, &offset, &s) ;
+
+	for(uint32_t i=0;i<s;++i)
+	{
+		FriendTrialRecord ftr ;
+
+		ok &= getRawSSLId(data, pktsize, &offset, ftr.friend_id); 	
+		ok &= getRawTimeT(data, pktsize, &offset, ftr.time_stamp) ;
+
+		item->tried_friends.push_back(ftr) ;
+	}
+
+	item->data_item = new RsGRouterGenericDataItem ;
+
+	ok &= getRawUInt32(data, pktsize, &offset, &item->data_item->routing_id); 	
+	ok &= getRawSha1(data, pktsize, &offset, item->data_item->destination_key) ;
+	ok &= getRawUInt32(data, pktsize, &offset, &item->data_item->data_size) ;
+
+	item->data_item->data_bytes = (uint8_t*)malloc(item->data_item->data_size) ;
+	memcpy(item->data_item->data_bytes,&((uint8_t*)data)[offset],item->data_item->data_size) ;
+	offset += item->data_item->data_size ;
+
+	if (offset != rssize || !ok)
+	{
+		std::cerr << __PRETTY_FUNCTION__ << ": error while deserialising! Item will be dropped." << std::endl;
+		return NULL ;
+	}
+
+	return item;
+}
+RsGRouterItem *RsGRouterSerialiser::deserialise_RsGRouterMatrixFriendListItem(void *data, uint32_t pktsize) const
+{
+	uint32_t offset = 8; // skip the header 
+	uint32_t rssize = getRsItemSize(data);
+	bool ok = true ;
+
+	RsGRouterMatrixFriendListItem *item = new RsGRouterMatrixFriendListItem() ;
+
+	uint32_t nb_friends = 0 ;
+	ok &= getRawUInt32(data, pktsize, &offset, &nb_friends); 	// file hash
+
+	item->reverse_friend_indices.resize(nb_friends) ;
+
+	for(uint32_t i=0;ok && i<nb_friends;++i)
+		ok &= getRawSSLId(data, pktsize, &offset, item->reverse_friend_indices[i]) ;
+
+	if (offset != rssize || !ok)
+	{
+		std::cerr << __PRETTY_FUNCTION__ << ": error while deserialising! Item will be dropped." << std::endl;
+		return NULL ;
+	}
+
+	return item;
+}
+RsGRouterItem *RsGRouterSerialiser::deserialise_RsGRouterMatrixCluesItem(void *data, uint32_t pktsize) const
+{
+	uint32_t offset = 8; // skip the header 
+	uint32_t rssize = getRsItemSize(data);
+	bool ok = true ;
+
+	RsGRouterMatrixCluesItem *item = new RsGRouterMatrixCluesItem() ;
+
+	ok &= getRawSha1(data,pktsize,&offset,item->destination_key) ;
+		
+	uint32_t nb_clues = 0 ;
+	ok &= getRawUInt32(data, pktsize, &offset, &nb_clues); 	
+
+	item->clues.clear() ;
+
+	for(uint32_t j=0;j<nb_clues;++j)
+	{
+		RoutingMatrixHitEntry HitE ;
+
+		ok &= getRawUInt32(data, pktsize, &offset, &HitE.friend_id); 	
+		ok &= getRawUFloat32(data, pktsize, &offset, HitE.weight); 	
+		ok &= getRawTimeT(data, pktsize, &offset, HitE.time_stamp); 	
+
+		item->clues.push_back(HitE) ;
+	}
+
+	if (offset != rssize || !ok)
+	{
+		std::cerr << __PRETTY_FUNCTION__ << ": error while deserialising! Item will be dropped." << std::endl;
+		return NULL ;
+	}
+
+	return item;
+}
+
+
 RsGRouterGenericDataItem *RsGRouterGenericDataItem::duplicate() const
 {
 	RsGRouterGenericDataItem *item = new RsGRouterGenericDataItem ;
-	*item = *this ; // copies everything.
+
+	item->routing_id = routing_id ;
+	item->destination_key = destination_key ;
+	item->data_size = data_size ;

 	// then duplicate the memory chunk

@ -242,6 +349,127 @@ bool RsGRouterACKItem::serialise(void *data,uint32_t& size) const
 	return ok;
 }

+/* serialise the data to the buffer */
+uint32_t RsGRouterMatrixCluesItem::serial_size() const
+{
+	uint32_t s = 8 ; 									// header
+
+	s += 20 ;				// Key size
+	s += 4 ; 				// list<RoutingMatrixHitEntry>::size()
+	s += (4+4+8) * clues.size() ;
+
+	return s ;
+}
+uint32_t RsGRouterMatrixFriendListItem::serial_size() const
+{
+	uint32_t s = 8 ; 									// header
+	s += 4  ; 											// reverse_friend_indices.size()
+	s += 16 * reverse_friend_indices.size() ; // sha1 for published_key
+
+	return s ;
+}
+uint32_t RsGRouterRoutingInfoItem::serial_size() const
+{
+	uint32_t s = 8 ; 									// header
+	s += 4  ; 											// status_flags
+	s += 16  ; 											// origin
+	s += 8  ; 											// received_time
+	s += 4  ; 											// tried_friends.size() ;
+
+	s += tried_friends.size() * ( 16 + 8 ) ; 	// FriendTrialRecord
+
+	s += 4;												// data_item->routing_id
+	s += 20;												// data_item->destination_key
+	s += 4;												// data_item->data_size
+	s += data_item->data_size;						// data_item->data_bytes 
+
+	return s ;
+}
+
+bool RsGRouterMatrixFriendListItem::serialise(void *data,uint32_t& size) const
+{
+	uint32_t tlvsize,offset=0;
+	bool ok = true;
+	
+	if(!serialise_header(data,size,tlvsize,offset))
+		return false ;
+
+	/* add mandatory parts first */
+	ok &= setRawUInt32(data, tlvsize, &offset, reverse_friend_indices.size());
+
+	for(uint32_t i=0;ok && i<reverse_friend_indices.size();++i)
+		ok &= setRawSSLId(data,tlvsize,&offset,reverse_friend_indices[i]) ;
+
+	if (offset != tlvsize)
+	{
+		ok = false;
+		std::cerr << "rsfileitemserialiser::serialisedata() size error! " << std::endl;
+	}
+
+	return ok;
+}
+bool RsGRouterMatrixCluesItem::serialise(void *data,uint32_t& size) const
+{
+	uint32_t tlvsize,offset=0;
+	bool ok = true;
+	
+	if(!serialise_header(data,size,tlvsize,offset))
+		return false ;
+
+	/* add mandatory parts first */
+	ok &= setRawSha1(data,tlvsize,&offset,destination_key) ;
+	ok &= setRawUInt32(data, tlvsize, &offset, clues.size());
+
+	for(std::list<RoutingMatrixHitEntry>::const_iterator it2(clues.begin());it2!=clues.end();++it2)
+	{
+		ok &= setRawUInt32(data, tlvsize, &offset,  (*it2).friend_id) ;
+		ok &= setRawUFloat32(data, tlvsize, &offset, (*it2).weight) ;
+		ok &= setRawTimeT(data, tlvsize, &offset,  (*it2).time_stamp) ;
+	}
+
+	if (offset != tlvsize)
+	{
+		ok = false;
+		std::cerr << "rsfileitemserialiser::serialisedata() size error! " << std::endl;
+	}
+
+	return ok;
+}
+bool RsGRouterRoutingInfoItem::serialise(void *data,uint32_t& size) const
+{
+	uint32_t tlvsize,offset=0;
+	bool ok = true;
+	
+	if(!serialise_header(data,size,tlvsize,offset))
+		return false ;
+
+	ok &= setRawUInt32(data, tlvsize, &offset, status_flags) ;
+	ok &= setRawSSLId(data, tlvsize, &offset, origin) ;
+	ok &= setRawTimeT(data, tlvsize, &offset, received_time) ;
+	ok &= setRawUInt32(data, tlvsize, &offset, tried_friends.size()) ;
+
+	for(std::list<FriendTrialRecord>::const_iterator it(tried_friends.begin());it!=tried_friends.end();++it)
+	{
+		ok &= setRawSSLId(data, tlvsize, &offset, (*it).friend_id) ;
+		ok &= setRawTimeT(data, tlvsize, &offset, (*it).time_stamp) ;
+	}
+
+	ok &= setRawUInt32(data, tlvsize, &offset, data_item->routing_id) ;
+	ok &= setRawSha1(data, tlvsize, &offset, data_item->destination_key) ;
+	ok &= setRawUInt32(data, tlvsize, &offset, data_item->data_size) ;
+
+	memcpy(&((uint8_t*)data)[offset],data_item->data_bytes,data_item->data_size) ;
+	offset += data_item->data_size ;
+
+	if (offset != tlvsize)
+	{
+		ok = false;
+		std::cerr << "rsfileitemserialiser::serialisedata() size error! " << std::endl;
+	}
+
+	return ok;
+}
+
 // -----------------------------------------------------------------------------------//
 // -------------------------------------  IO  --------------------------------------- // 
 // -----------------------------------------------------------------------------------//
@ -277,3 +505,35 @@ std::ostream& RsGRouterGenericDataItem::print(std::ostream& o, uint16_t)
 	return o ;
 }

+std::ostream& RsGRouterRoutingInfoItem::print(std::ostream& o, uint16_t)
+{
+	o << "RsGRouterRoutingInfoItem:" << std::endl ;
+	o << "  direct origin: \""<< PeerId() << "\"" << std::endl ;
+	o << "  origin:          "<< origin.toStdString() << std::endl ;
+	o << "  recv time:       "<< received_time << std::endl ;
+	o << "  flags:           "<< std::hex << status_flags << std::dec << std::endl ;
+	o << "  Key:             "<< data_item->destination_key.toStdString() << std::endl ;
+	o << "  Data size:       "<< data_item->data_size << std::endl ;
+	o << "  Tried friends:   "<< tried_friends.size() << std::endl;
+
+	return o ;
+}
+
+std::ostream& RsGRouterMatrixCluesItem::print(std::ostream& o, uint16_t)
+{
+	o << "RsGRouterMatrixCluesItem:" << std::endl ;
+	o << "  destination k:  " << destination_key.toStdString()  << std::endl;
+	o << "  routing clues:  " << clues.size() << std::endl;
+
+	for(std::list<RoutingMatrixHitEntry>::const_iterator it(clues.begin());it!=clues.end();++it)
+		o << "    " << (*it).friend_id << " " << (*it).time_stamp << " " << (*it).weight << std::endl;
+
+	return o ;
+}
+std::ostream& RsGRouterMatrixFriendListItem::print(std::ostream& o, uint16_t)
+{
+	o << "RsGRouterMatrixCluesItem:" << std::endl ;
+	o << "  friends:  " << reverse_friend_indices.size() << std::endl;
+
+	return o ;
+}
--- a/libretroshare/src/grouter/grouteritems.h
+++ b/libretroshare/src/grouter/grouteritems.h
@ -27,6 +27,7 @@

 #include "serialiser/rsserial.h"
 #include "rsgrouter.h"
+#include "p3grouter.h"

 // To be put in serialiser/rsserviceids.h
 static const uint8_t RS_SERVICE_TYPE_GROUTER = 0x0016 ;
@ -35,6 +36,10 @@ const uint8_t RS_PKT_SUBTYPE_GROUTER_PUBLISH_KEY	= 0x01 ;			// used to publish a
 const uint8_t RS_PKT_SUBTYPE_GROUTER_DATA          = 0x02 ;			// used to send data to a destination
 const uint8_t RS_PKT_SUBTYPE_GROUTER_ACK           = 0x03 ;			// acknowledgement of data received

+const uint8_t RS_PKT_SUBTYPE_GROUTER_MATRIX_CLUES  = 0x80 ;			// item to save matrix clues
+const uint8_t RS_PKT_SUBTYPE_GROUTER_ROUTING_INFO  = 0x81 ;			// item to save routing info
+const uint8_t RS_PKT_SUBTYPE_GROUTER_FRIENDS_LIST  = 0x82 ;			// item to save friend lists
+
 const uint8_t QOS_PRIORITY_RS_GROUTER_PUBLISH_KEY = 3 ;				// slow items. No need to congest the network with this.
 const uint8_t QOS_PRIORITY_RS_GROUTER_ACK         = 3 ;
 const uint8_t QOS_PRIORITY_RS_GROUTER_DATA        = 3 ;
@ -88,16 +93,29 @@ class RsGRouterPublishKeyItem: public RsGRouterItem
 		std::string  description_string ;
 };

-class RsGRouterGenericDataItem: public RsGRouterItem
+class RsGRouterNonCopyableObject
+{
+	public:
+		RsGRouterNonCopyableObject() {}
+	private:
+		RsGRouterNonCopyableObject(const RsGRouterNonCopyableObject&) {}
+		RsGRouterNonCopyableObject operator=(const RsGRouterNonCopyableObject&) { return *this ;}
+};
+
+class RsGRouterGenericDataItem: public RsGRouterItem, public RsGRouterNonCopyableObject
 {
 	public:
 		RsGRouterGenericDataItem() : RsGRouterItem(RS_PKT_SUBTYPE_GROUTER_DATA) { setPriorityLevel(QOS_PRIORITY_RS_GROUTER_DATA) ; }
-		virtual ~RsGRouterGenericDataItem() { free(data_bytes); data_bytes=NULL;}
+		virtual ~RsGRouterGenericDataItem() { clear() ; }

 		virtual bool serialise(void *data,uint32_t& size) const ;	
 		virtual uint32_t serial_size() const ; 						

-		virtual void clear() {} 
+		virtual void clear() 
+		{
+			free(data_bytes); 
+			data_bytes=NULL;
+		} 
 		virtual std::ostream& print(std::ostream &out, uint16_t indent = 0) ;

 		RsGRouterGenericDataItem *duplicate() const ;
@ -128,6 +146,64 @@ class RsGRouterACKItem: public RsGRouterItem
 		uint32_t state ; 							// packet was delivered, not delivered, bounced, etc
 };

+// Items for saving the routing matrix information.
+
+class RsGRouterMatrixCluesItem: public RsGRouterItem
+{
+	public:
+		RsGRouterMatrixCluesItem() : RsGRouterItem(RS_PKT_SUBTYPE_GROUTER_MATRIX_CLUES) 
+		{ setPriorityLevel(0) ; }	// this item is never sent through the network
+
+		virtual bool serialise(void *data,uint32_t& size) const ;	
+		virtual uint32_t serial_size() const ; 						
+
+		virtual void clear() {} 
+		virtual std::ostream& print(std::ostream &out, uint16_t indent = 0) ;
+
+		// packet data
+		//
+		GRouterKeyId destination_key ;
+		std::list<RoutingMatrixHitEntry> clues ;
+};
+
+class RsGRouterMatrixFriendListItem: public RsGRouterItem
+{
+	public:
+		RsGRouterMatrixFriendListItem() : RsGRouterItem(RS_PKT_SUBTYPE_GROUTER_FRIENDS_LIST) 
+		{ setPriorityLevel(0) ; }	// this item is never sent through the network
+
+		virtual bool serialise(void *data,uint32_t& size) const ;	
+		virtual uint32_t serial_size() const ; 						
+
+		virtual void clear() {} 
+		virtual std::ostream& print(std::ostream &out, uint16_t indent = 0) ;
+
+		// packet data
+		//
+		std::vector<SSLIdType> reverse_friend_indices ;
+};
+
+class RsGRouterRoutingInfoItem: public RsGRouterItem, public GRouterRoutingInfo, public RsGRouterNonCopyableObject
+{
+	public:
+		RsGRouterRoutingInfoItem() : RsGRouterItem(RS_PKT_SUBTYPE_GROUTER_ROUTING_INFO) 
+		{ setPriorityLevel(0) ; }	// this item is never sent through the network
+
+		virtual ~RsGRouterRoutingInfoItem() { clear() ; }
+		
+		virtual bool serialise(void *data,uint32_t& size) const ;	
+		virtual uint32_t serial_size() const ; 						
+
+		virtual void clear() 
+		{
+			if(data_item != NULL)
+				delete data_item ;
+			data_item = NULL ;
+			tried_friends.clear() ;
+		}
+		virtual std::ostream& print(std::ostream &out, uint16_t indent = 0) ;
+};
+
 /***********************************************************************************/
 /*                                Serialisation                                    */
 /***********************************************************************************/
@ -151,6 +227,9 @@ class RsGRouterSerialiser: public RsSerialType
 		RsGRouterItem *deserialise_RsGRouterPublishKeyItem(void *data,uint32_t size) const ;
 		RsGRouterItem *deserialise_RsGRouterGenericDataItem(void *data,uint32_t size) const ;
 		RsGRouterItem *deserialise_RsGRouterACKItem(void *data,uint32_t size) const ;
+		RsGRouterItem *deserialise_RsGRouterMatrixCluesItem(void *data,uint32_t size) const ;
+		RsGRouterItem *deserialise_RsGRouterMatrixFriendListItem(void *data,uint32_t size) const ;
+		RsGRouterItem *deserialise_RsGRouterRoutingInfoItem(void *data,uint32_t size) const ;
 };


--- a/libretroshare/src/grouter/groutermatrix.cc
+++ b/libretroshare/src/grouter/groutermatrix.cc
@ -25,6 +25,7 @@

 #include "groutertypes.h"
 #include "groutermatrix.h"
+#include "grouteritems.h"

 GRouterMatrix::GRouterMatrix()
 {
@ -177,8 +178,13 @@ bool GRouterMatrix::computeRoutingProbabilities(const GRouterKeyId& key_id, cons

 bool GRouterMatrix::updateRoutingProbabilities()
 {
+	std::cerr << "Updating routing probabilities..." << std::endl;
+
 	if(!_proba_need_updating)
+	{
+		std::cerr << "  not needed." << std::endl;
 		return false ;
+	}

 	time_t now = time(NULL) ;

@ -196,11 +202,61 @@ bool GRouterMatrix::updateRoutingProbabilities()
 			v[(*it2).friend_id] += (*it2).weight / (time_difference_in_days*time_difference_in_days) ;
 		}
 	}
+	std::cerr << "  done." << std::endl;
+
 	_proba_need_updating = false ;
 	return true ;
 }

+bool GRouterMatrix::saveList(std::list<RsItem*>& items) 
+{
+	std::cerr << "  GRoutingMatrix::saveList()" << std::endl;

+	RsGRouterMatrixFriendListItem *item = new RsGRouterMatrixFriendListItem ;

+	item->reverse_friend_indices = _reverse_friend_indices ;
+	items.push_back(item) ;

+	for(std::map<GRouterKeyId,std::list<RoutingMatrixHitEntry> >::const_iterator it(_routing_clues.begin());it!=_routing_clues.end();++it)
+	{
+		RsGRouterMatrixCluesItem *item = new RsGRouterMatrixCluesItem ;
+
+		item->destination_key = it->first ;
+		item->clues = it->second ;
+
+		items.push_back(item) ;
+	}
+
+	return true ;
+}
+bool GRouterMatrix::loadList(std::list<RsItem*>& items) 
+{
+	RsGRouterMatrixFriendListItem *itm1 = NULL ;
+	RsGRouterMatrixCluesItem      *itm2 = NULL ;
+
+	std::cerr << "  GRoutingMatrix::loadList()" << std::endl;
+
+	for(std::list<RsItem*>::const_iterator it(items.begin());it!=items.end();++it)
+	{
+		if(NULL != (itm2 = dynamic_cast<RsGRouterMatrixCluesItem*>(*it)))
+		{
+			std::cerr << "    initing routing clues." << std::endl;
+
+			_routing_clues[itm2->destination_key] = itm2->clues ;
+			_proba_need_updating = true ;	// notifies to re-compute all the info.
+		}
+		if(NULL != (itm1 = dynamic_cast<RsGRouterMatrixFriendListItem*>(*it)))
+		{
+			_reverse_friend_indices = itm1->reverse_friend_indices ;
+			_friend_indices.clear() ;
+
+			for(uint32_t i=0;i<_reverse_friend_indices.size();++i)
+				_friend_indices[_reverse_friend_indices[i]] = i ;
+
+			_proba_need_updating = true ;	// notifies to re-compute all the info.
+		}
+	}
+
+	return true ;
+}

--- a/libretroshare/src/grouter/groutermatrix.h
+++ b/libretroshare/src/grouter/groutermatrix.h
@ -31,6 +31,8 @@
 #include "groutertypes.h"
 #include "rsgrouter.h"

+class RsItem ;
+
 // The routing matrix records the event clues received from each friend
 //
 struct RoutingMatrixHitEntry
@ -64,6 +66,9 @@ class GRouterMatrix
 		//
 		void debugDump() const ;

+		bool saveList(std::list<RsItem*>& items) ;
+		bool loadList(std::list<RsItem*>& items) ;
+
 	private:
 		// returns the friend id, possibly creating a new id.
 		//
@ -75,8 +80,8 @@ class GRouterMatrix

 		// List of events received and computed routing probabilities
 		//
-		std::map<GRouterKeyId, std::list<RoutingMatrixHitEntry> > _routing_clues ;
-		std::map<GRouterKeyId, std::vector<float> > 					 _time_combined_hits ; // hit matrix after time-convolution filter
+		std::map<GRouterKeyId, std::list<RoutingMatrixHitEntry> > _routing_clues ;			// received routing clues. Should be saved.
+		std::map<GRouterKeyId, std::vector<float> > 					 _time_combined_hits ; 	// hit matrix after time-convolution filter

 		// This is used to avoid re-computing probas when new events have been received.
 		//
--- a/libretroshare/src/grouter/groutertypes.h
+++ b/libretroshare/src/grouter/groutertypes.h
@ -27,6 +27,10 @@

 #include <stdint.h>
 #include <time.h>
+#include <list>
+#include "pgp/rscertificate.h"
+
+class RsGRouterGenericDataItem ;

 typedef uint32_t GRouterServiceId ;
 typedef uint32_t GRouterKeyPropagationId ;
@ -35,14 +39,43 @@ typedef uint32_t GRouterMsgPropagationId ;
 static const uint32_t RS_GROUTER_MATRIX_MAX_HIT_ENTRIES       = 5;
 static const uint32_t RS_GROUTER_MATRIX_MIN_TIME_BETWEEN_HITS = 60;	// can be set to up to half the publish time interval. Prevents flooding routes.

-static const time_t RS_GROUTER_DEBUG_OUTPUT_PERIOD       =       20 ; // Output everything
-static const time_t RS_GROUTER_AUTOWASH_PERIOD           =       60 ; // Autowash every minute. Not a costly operation.
+static const time_t RS_GROUTER_DEBUG_OUTPUT_PERIOD         =       20 ; // Output everything
+static const time_t RS_GROUTER_AUTOWASH_PERIOD             =       60 ; // Autowash every minute. Not a costly operation.
 //static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD   =    10*60 ; // Check for key advertising every 10 minutes
 //static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL = 24*60*60 ; // Advertise each key once a day at most.
-static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD   =    1 *60 ; // Check for key advertising every 10 minutes
-static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL =    2 *60 ; // Advertise each key once a day at most.
+static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD     =    1 *60 ; // Check for key advertising every 10 minutes
+static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL   =    2 *60 ; // Advertise each key once a day at most.
+static const time_t RS_GROUTER_ROUTING_WAITING_TIME        =     3600 ; // time between two trial of sending a given message
+static const time_t RS_GROUTER_KEY_DIFFUSION_MAX_KEEP      =     7200 ; // time to keep key diffusion items in cache, to avoid multiple diffusion.

 static const uint32_t RS_GROUTER_ROUTING_STATE_UNKN = 0x0000 ;		// unknown. Unused.
 static const uint32_t RS_GROUTER_ROUTING_STATE_PEND = 0x0001 ;		// item is pending. Should be sent asap. 
 static const uint32_t RS_GROUTER_ROUTING_STATE_SENT = 0x0002 ;		// item is sent. Waiting for answer
 static const uint32_t RS_GROUTER_ROUTING_STATE_ARVD = 0x0003 ;		// item is at destination. The cache only holds it to avoid duplication.
+
+class GRouterPublishedKeyInfo
+{
+	public:
+		GRouterServiceId service_id ;
+		std::string description_string ;
+		time_t last_published_time ;
+		time_t validity_time ;
+};
+
+struct FriendTrialRecord
+{
+	SSLIdType friend_id ;			// id of the friend
+	time_t    time_stamp ;			// time of the last tried
+};
+
+class GRouterRoutingInfo
+{
+	public:
+		RsGRouterGenericDataItem *data_item ;
+
+		uint32_t status_flags ;									// pending, waiting, etc.
+		std::list<FriendTrialRecord> tried_friends ; 	// list of friends to which the item was sent ordered with time.
+		SSLIdType origin ;										// which friend sent us that item
+		time_t received_time ;									// time at which the item was received
+};
+
--- a/libretroshare/src/grouter/p3grouter.cc
+++ b/libretroshare/src/grouter/p3grouter.cc
@ -32,8 +32,6 @@
 #include "groutertypes.h"
 #include "grouterclientservice.h"

-static const uint32_t RS_GROUTER_ROUTING_WAITING_TIME = 3600 ; // time between two trial of sending a message
-
 p3GRouter::p3GRouter(p3LinkMgr *lm)
 	: p3Service(RS_SERVICE_TYPE_GROUTER), p3Config(CONFIG_TYPE_GROUTER), mLinkMgr(lm), grMtx("GRouter")
 {
@ -75,6 +73,7 @@ int p3GRouter::tick()
 	if(now > last_publish_campaign_time + RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD)
 	{
 		last_publish_campaign_time = now ;
+
 		publishKeys() ;
 		_routing_matrix.updateRoutingProbabilities() ;
 	}
@ -103,13 +102,34 @@ void p3GRouter::autoWash()
 {
 	RsStackMutex mtx(grMtx) ;

-	std::cerr << "p3GRouter::autoWash() Unimplemented !!" << std::endl;
+	std::cerr << "p3GRouter::autoWash(): cleaning old entried." << std::endl;

 	// cleanup cache
+
+	time_t now = time(NULL) ;
+
+	for(std::map<GRouterKeyPropagationId,time_t>::iterator it(_key_diffusion_time_stamps.begin());it!=_key_diffusion_time_stamps.end();)
+		if(it->second + RS_GROUTER_KEY_DIFFUSION_MAX_KEEP < now)
+		{
+			std::cerr << "  Removing key diffusion time stamp " << it->second << " for diffusion id " << std::hex << it->first << std::dec << std::endl;
+			std::map<GRouterKeyPropagationId,time_t>::iterator tmp(it) ;
+			++tmp ;
+			_key_diffusion_time_stamps.erase(it) ;
+			it = tmp ;
+		}
+		else
+			++it ;
+
+	// look into pending items.
+	
+	std::cerr << "  Pending key diffusion items: " << _key_diffusion_items.size() << std::endl;
+	std::cerr << "  Pending messages to route  : " << _pending_messages.size() << std::endl;
 }

 void p3GRouter::routePendingObjects()
 {
+	RsStackMutex mtx(grMtx) ;
+
 	// Go through list of published keys
 	// broadcast a publishKeyItem for each of them.
 	//
@ -157,7 +177,7 @@ void p3GRouter::routePendingObjects()
 		lst.push_back(SSLIdType(*it)) ;

 	for(std::map<GRouterMsgPropagationId, GRouterRoutingInfo>::iterator it(_pending_messages.begin());it!=_pending_messages.end();)
-		if((it->second.status_flags & RS_GROUTER_ROUTING_STATE_PEND) || it->second.status_flags == RS_GROUTER_ROUTING_STATE_SENT && it->second.tried_friends.front().time_stamp+RS_GROUTER_ROUTING_WAITING_TIME < now)
+		if((it->second.status_flags & RS_GROUTER_ROUTING_STATE_PEND) || (it->second.status_flags == RS_GROUTER_ROUTING_STATE_SENT && it->second.tried_friends.front().time_stamp+RS_GROUTER_ROUTING_WAITING_TIME < now))
 		{
 			std::cerr << "  Msg id: " << std::hex << it->first << std::dec << std::endl;
 			std::cerr << "  Origin: " << it->second.origin.toStdString() << std::endl;
@ -265,12 +285,12 @@ void p3GRouter::routePendingObjects()

 void p3GRouter::publishKeys()
 {
+	RsStackMutex mtx(grMtx) ;
+
 	// Go through list of published keys
 	// broadcast a publishKeyItem for each of them.

 	time_t now = time(NULL) ;
-	std::list<std::string> connected_peers ;
-	mLinkMgr->getOnlineList(connected_peers) ;

 	for(std::map<GRouterKeyId, GRouterPublishedKeyInfo>::iterator it(_owned_key_ids.begin());it!=_owned_key_ids.end();++it)
 	{
@ -291,25 +311,41 @@ void p3GRouter::publishKeys()
 			item.service_id = info.service_id ;
 			item.randomized_distance = drand48() ;
 			item.description_string = info.description_string ;
+			item.PeerId("") ;	// no peer id => key is forwarded to all friends.

-			// get list of connected friends, and broadcast to all of them
-			//
-			for(std::list<std::string>::const_iterator it(connected_peers.begin());it!=connected_peers.end();++it)
-			{
-				std::cerr << "    sending to " << (*it) << std::endl; 
+			locked_forwardKey(item) ;

-				RsGRouterPublishKeyItem *itm = new RsGRouterPublishKeyItem(item) ;
-				itm->PeerId(*it) ;

-				// we should randomise the depth
-
-				sendItem(itm) ;
-			}
 			info.last_published_time = now ;
 		}
 	}
 }

+void p3GRouter::locked_forwardKey(const RsGRouterPublishKeyItem& item) 
+{
+	std::list<std::string> connected_peers ;
+	mLinkMgr->getOnlineList(connected_peers) ;
+
+	std::cerr << "  Forwarding key item to all available friends..." << std::endl;
+
+	// get list of connected friends, and broadcast to all of them
+	//
+	for(std::list<std::string>::const_iterator it(connected_peers.begin());it!=connected_peers.end();++it)
+		if(item.PeerId() != *it)
+		{
+			std::cerr << "    sending to " << (*it) << std::endl; 
+
+			RsGRouterPublishKeyItem *itm = new RsGRouterPublishKeyItem(item) ;
+			itm->PeerId(*it) ;
+
+			// we should randomise the depth
+
+			sendItem(itm) ;
+		}
+		else
+			std::cerr << "    Not forwarding to source id " << item.PeerId() << std::endl;
+}
+
 bool p3GRouter::registerKey(const GRouterKeyId& key,const GRouterServiceId& client_id,const std::string& description) 
 {
 	RsStackMutex mtx(grMtx) ;
@ -355,6 +391,8 @@ void p3GRouter::handleIncoming()

 void p3GRouter::handleRecvPublishKeyItem(RsGRouterPublishKeyItem *item)
 {
+	RsStackMutex mtx(grMtx) ;
+
 	std::cerr << "Received key publish item for key :" << std::endl ;
 	std::cerr << "     diffusion  = " << std::hex << item->diffusion_id << std::dec << std::endl ;
 	std::cerr << "     key id     = " << item->published_key.toStdString() << std::endl ;
@ -369,25 +407,27 @@ void p3GRouter::handleRecvPublishKeyItem(RsGRouterPublishKeyItem *item)
 	// forward the key to other peers according to key forwarding cache
 	
 	std::map<GRouterKeyPropagationId,time_t>::iterator it = _key_diffusion_time_stamps.find(item->diffusion_id) ;
-	bool found = false ;

-	if(it != _key_diffusion_time_stamps.end())	// found. We don't propagate further
-		found = true ;
+	bool found = (it != _key_diffusion_time_stamps.end());	// found. We don't propagate further

 	_key_diffusion_time_stamps[item->diffusion_id] = time(NULL) ;	// always stamp

 	if(found)
+	{
+		std::cerr << "  Key diffusion item already in cache. Not forwardign further." << std::endl;
 		return ;
+	}

 	// Propagate the item to all other online friends. We don't do this right now, but push items in a queue.
 	// Doing this we can control the amount of key propagation and avoid flooding.

-	_key_diffusion_items.push(item) ;
+	locked_forwardKey(*item) ;
 }

 void p3GRouter::handleRecvACKItem(RsGRouterACKItem *item)
 {
-	std::cerr << "Received ACK item, mid=" << (void*)item->mid << std::endl;
+	RsStackMutex mtx(grMtx) ;
+	std::cerr << "Received ACK item, mid=" << std::hex << item->mid << std::dec << std::endl;

 	// find the item in the pendign list, 
 	// 	- if not found, drop.
@ -438,6 +478,7 @@ void p3GRouter::handleRecvACKItem(RsGRouterACKItem *item)

 void p3GRouter::handleRecvDataItem(RsGRouterGenericDataItem *item)
 {
+	RsStackMutex mtx(grMtx) ;
 	std::cerr << "Received data item for key " << item->destination_key.toStdString() << std::endl;

 	// Do we have this item in the cache already?
@ -497,8 +538,16 @@ void p3GRouter::handleRecvDataItem(RsGRouterGenericDataItem *item)
 		std::cerr << "  Item is not for us. Leaving in pending msgs to be routed later." << std::endl;
 }

+bool p3GRouter::registerClientService(const GRouterServiceId& id,GRouterClientService *service)
+{
+	RsStackMutex mtx(grMtx) ;
+	_registered_services[id] = service ;
+	return true ;
+}
+
 void p3GRouter::sendData(const GRouterKeyId& destination, RsGRouterGenericDataItem *item)
 {
+	RsStackMutex mtx(grMtx) ;
 	// push the item into pending messages.
 	//
 	GRouterRoutingInfo info ;
@ -511,7 +560,7 @@ void p3GRouter::sendData(const GRouterKeyId& destination, RsGRouterGenericDataIt
 	// Make sure we have a unique id (at least locally).
 	//
 	GRouterMsgPropagationId propagation_id ;
-	do propagation_id = RSRandom::random_u32() ; while(_pending_messages.find(propagation_id) != _pending_messages.end()) ;
+	do { propagation_id = RSRandom::random_u32(); } while(_pending_messages.find(propagation_id) != _pending_messages.end()) ;

 	std::cerr << "p3GRouter::sendGRouterData(): pushing the followign item in the msg pending list:" << std::endl;
 	std::cerr << "  data_item.size = " << info.data_item->data_size << std::endl;
@ -536,24 +585,71 @@ void p3GRouter::sendACK(const SSLIdType& peer, GRouterMsgPropagationId mid, uint

 bool p3GRouter::loadList(std::list<RsItem*>& items) 
 {
-	std::cerr << "(WW) " << __PRETTY_FUNCTION__ << ": not implemented." << std::endl;
-	return false ;
-}
-bool p3GRouter::saveList(bool&,std::list<RsItem*>& items) 
-{
-	std::cerr << "(WW) " << __PRETTY_FUNCTION__ << ": not implemented." << std::endl;
+	RsStackMutex mtx(grMtx) ;

+	std::cerr << "p3GRouter::loadList() : " << std::endl;
+
+	_routing_matrix.loadList(items) ;
+
+	// remove all existing objects.
+	//
+	std::cerr << "  removing all existing items (" << _pending_messages.size() << " items to delete)." << std::endl;
+
+	for(std::map<GRouterMsgPropagationId,GRouterRoutingInfo>::iterator it(_pending_messages.begin());it!=_pending_messages.end();++it)
+		delete it->second.data_item ;
+	_pending_messages.clear() ;
+
+	for(std::list<RsItem*>::const_iterator it(items.begin());it!=items.end();++it)
+	{
+		RsGRouterRoutingInfoItem *itm1 = NULL ;
+
+		if(NULL != (itm1 = dynamic_cast<RsGRouterRoutingInfoItem*>(*it)))
+		{
+			_pending_messages[itm1->data_item->routing_id] = *itm1 ;	
+			_pending_messages[itm1->data_item->routing_id].data_item = itm1->data_item ;	// avoids duplication.
+
+			itm1->data_item = NULL ;	// prevents deletion.
+		}
+
+		delete *it ;
+	}
+	return true ;
+}
+bool p3GRouter::saveList(bool& cleanup,std::list<RsItem*>& items) 
+{
 	// We save
 	// 	- the routing clues
 	// 	- the pending items

-	return false ;
+	cleanup = true ; // the client should delete the items.
+
+	std::cerr << "p3GRouter::saveList()..." << std::endl;
+	std::cerr << "  saving routing clues." << std::endl;
+
+	_routing_matrix.saveList(items) ;
+
+	std::cerr << "  saving pending items." << std::endl;
+
+	for(std::map<GRouterMsgPropagationId,GRouterRoutingInfo>::const_iterator it(_pending_messages.begin());it!=_pending_messages.end();++it)
+	{
+		RsGRouterRoutingInfoItem *item = new RsGRouterRoutingInfoItem ;
+		
+		*(GRouterRoutingInfo*)item = it->second ;	// copy all members
+
+		item->data_item = it->second.data_item->duplicate() ;	// deep copy, because we call delete on the object, and the item might be removed before we handle it in the client.
+
+		items.push_back(item) ;
+	}
+
+	return true ;
 }

 // Dump everything
 //
 void p3GRouter::debugDump()
 {
+	RsStackMutex mtx(grMtx) ;
+
 	time_t now = time(NULL) ;

 	std::cerr << "Full dump of Global Router state: " << std::endl; 
--- a/libretroshare/src/grouter/p3grouter.h
+++ b/libretroshare/src/grouter/p3grouter.h
@ -43,32 +43,8 @@
 static const uint32_t CONFIG_TYPE_GROUTER = 0x0016 ;

 class p3LinkMgr ;
-
-class GRouterPublishedKeyInfo
-{
-	public:
-		GRouterServiceId service_id ;
-		std::string description_string ;
-		time_t last_published_time ;
-		time_t validity_time ;
-};
-
-struct FriendTrialRecord
-{
-	SSLIdType friend_id ;			// id of the friend
-	time_t    time_stamp ;			// time of the last tried
-};
-
-class GRouterRoutingInfo
-{
-	public:
-		RsGRouterGenericDataItem *data_item ;
-
-		uint32_t status_flags ;									// pending, waiting, etc.
-		std::list<FriendTrialRecord> tried_friends ; 	// list of friends to which the item was sent ordered with time.
-		SSLIdType origin ;										// which friend sent us that item
-		time_t received_time ;									// time at which the item was received
-};
+class RsGRouterPublishKeyItem ;
+class RsGRouterACKItem ;

 class p3GRouter: public RsGRouter, public p3Service, public p3Config
 {
@ -149,6 +125,7 @@ class p3GRouter: public RsGRouter, public p3Service, public p3Config
 		void handleIncoming() ;
 		void publishKeys() ;
 		void debugDump() ;
+		void locked_forwardKey(const RsGRouterPublishKeyItem&) ;

 		//===================================================//
 		//                  p3Config methods                 //
@ -157,7 +134,7 @@ class p3GRouter: public RsGRouter, public p3Service, public p3Config
 		// Load/save the routing info, the pending items in transit, and the config variables.
 		//
 		virtual bool loadList(std::list<RsItem*>& items) ;
-		virtual bool saveList(bool&,std::list<RsItem*>& items) ;
+		virtual bool saveList(bool& cleanup,std::list<RsItem*>& items) ;

 		virtual RsSerialiser *setupSerialiser() ;