added basic probability computation to global routing matrix, and debug output. Fixed bug in Float serialisation (this should be improved btw)

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6890 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/grouter/groutermatrix.cc

@@ -25,9 +25,117 @@
 
 #include "groutermatrix.h"
 
-bool GRouterMatrix::addRoutingClue(const GRouterKeyId& id,const GRouterServiceId& sid,float distance,const std::string& desc_string,const SSLIdType& source_friend) 
+GRouterMatrix::GRouterMatrix()
 {
-	std::cerr << "(WW) " << __PRETTY_FUNCTION__ << ": not implemented" << std::endl;
+	_proba_need_updating = true ;
+}
+
+bool GRouterMatrix::addRoutingClue(	const GRouterKeyId& key_id,const GRouterServiceId& sid,float distance,
+												const std::string& desc_string,const SSLIdType& source_friend) 
+{
+	// 1 - get the friend index.
+	//
+	uint32_t fid = getFriendId(source_friend) ;
+
+	// 2 - get the Key map, and add the routing clue.
+	//
+	time_t now = time(NULL) ;
+
+	RoutingMatrixHitEntry rc ;
+	rc.weight = 1.0f / (1.0f + distance) ;
+	rc.time_stamp = now ;
+	rc.friend_id = fid ;
+
+	_routing_clues[key_id].push_front(rc) ;	// create it if necessary
+	_proba_need_updating = true ; 				// always, since we added new clues.
+
+	return true ;
+}
+
+uint32_t GRouterMatrix::getFriendId(const SSLIdType& source_friend)
+{
+	std::map<SSLIdType,uint32_t>::const_iterator it = _friend_indices.find(source_friend) ;
+
+	if(it == _friend_indices.end())
+	{
+		// add a new friend
+
+		uint32_t new_id = _reverse_friend_indices.size() ;
+		_reverse_friend_indices.push_back(source_friend) ;
+		_friend_indices[source_friend] = new_id ;
+
+		return new_id ;
+	}
+	else
+		return it->second ;
+}
+
+void GRouterMatrix::debugDump() const
+{
+	std::cerr << "    Proba needs up: " << _proba_need_updating << std::endl;
+	std::cerr << "    Routing events: " << std::endl;
+	time_t now = time(NULL) ;
+
+	for(std::map<GRouterKeyId, std::list<RoutingMatrixHitEntry> >::const_iterator it(_routing_clues.begin());it!=_routing_clues.end();++it)
+	{
+		std::cerr << "      " << it->first.toStdString() << " : " ;
+		for(std::list<RoutingMatrixHitEntry>::const_iterator it2(it->second.begin());it2!=it->second.end();++it2)
+			std::cerr << now - (*it2).time_stamp << " (" << (*it2).friend_id << "," << (*it2).weight << ") " ;
+
+		std::cerr << std::endl;
+	}
+	std::cerr << "    Routing probabilities: " << std::endl;
+
+	for(std::map<GRouterKeyId, std::vector<float> >::const_iterator it(_time_combined_hits.begin());it!=_time_combined_hits.end();++it)
+	{
+		std::cerr << it->first.toStdString() << "  :  " ;
+
+		for(uint32_t i=0;i<it->second.size();++i)
+			std::cerr << it->second[i] << "   " ;
+		std::cerr << std::endl;
+	}
+}
+
+bool GRouterMatrix::computeRoutingProbabilities(const GRouterKeyId& id, const std::vector<SSLIdType>& friends, std::vector<float>& probas) const
+{
+	// Routing probabilities are computed according to routing clues
+	//
+	// For a given key, each friend has a known set of routing clues (time_t, weight)
+	//	We combine these to compute a static weight for each friend/key pair. 
+	//	This is performed in updateRoutingProbabilities()
+	//
+	//	Then for a given list of online friends, the weights are computed into probabilities, 
+	//	that always sum up to 1.
+	//
+	return true ;
+}
+
+bool GRouterMatrix::updateRoutingProbabilities()
+{
+	if(!_proba_need_updating)
+		return false ;
+
+	time_t now = time(NULL) ;
+
+	for(std::map<GRouterKeyId, std::list<RoutingMatrixHitEntry> >::const_iterator it(_routing_clues.begin());it!=_routing_clues.end();++it)
+	{
+		std::cerr << "      " << it->first.toStdString() << " : " ;
+
+		std::vector<float>& v(_time_combined_hits[it->first]) ;
+		v.clear() ;
+		v.resize(_friend_indices.size(),0.0f) ;
+
+		for(std::list<RoutingMatrixHitEntry>::const_iterator it2(it->second.begin());it2!=it->second.end();++it2)
+		{
+			float time_difference_in_days = 1 + (now - (*it2).time_stamp ) / 86400.0f ;
+			v[(*it2).friend_id] += (*it2).weight / (time_difference_in_days*time_difference_in_days) ;
+		}
+	}
+	_proba_need_updating = false ;
+	return true ;
 }
 
 
+
+
+

libretroshare/src/grouter/groutermatrix.h

@@ -35,21 +35,16 @@
 //
 struct RoutingMatrixHitEntry
 {
+	uint32_t friend_id ;			// not the full key. Gets too big otherwise!
 	float weight ;
 	time_t time_stamp ;
 };
 
-// The map indexes for each friend the list of recent routing clues received.
-//
-struct RoutingMatrixFriendKeyData
-{
-	std::list<RoutingMatrixHitEntry> routing_clues ;
-	float probability ;
-};
-
 class GRouterMatrix
 {
 	public:
+		GRouterMatrix() ;
+
 		// Computes the routing probabilities for this id  for the given list of friends.
 		// the computation accounts for the time at which the info was received and the
 		// weight of each routing hit record.
@@ -69,13 +64,27 @@ class GRouterMatrix
 		//
 		bool addRoutingClue(const GRouterKeyId& id,const GRouterServiceId& sid,float distance,const std::string& desc_string,const SSLIdType& source_friend) ;
 
+		// Dump info in terminal.
+		//
+		void debugDump() const ;
+
 	private:
+		uint32_t getFriendId(const SSLIdType& id) ;
+
 		// List of events received and computed routing probabilities
 		//
-		std::map<GRouterKeyId, RoutingMatrixFriendKeyData> _routing_info ;
+		std::map<GRouterKeyId, std::list<RoutingMatrixHitEntry> > _routing_clues ;
+		std::map<GRouterKeyId, std::vector<float> > _time_combined_hits ; // hit matrix after time-convolution filter
 
 		// This is used to avoid re-computing probas when new events have been received.
 		//
 		bool _proba_need_updating ;
+
+		// Routing weights. These are the result of a time convolution of the routing clues and weights
+		// recorded in _routing_clues.
+		//
+		std::map<SSLIdType,uint32_t> _friend_indices ;	// index for each friend to lookup in the routing matrix Not saved.
+		std::vector<SSLIdType> _reverse_friend_indices ;// SSLid corresponding to each friend index. Saved.
+
 };
 

libretroshare/src/grouter/p3grouter.cc

@@ -30,44 +30,65 @@
 #include "p3grouter.h"
 #include "grouteritems.h"
 
+static const time_t RS_GROUTER_DEBUG_OUTPUT_PERIOD       =       20 ; // Output everything
 static const time_t RS_GROUTER_AUTOWASH_PERIOD           =       60 ; // Autowash every minute. Not a costly operation.
-static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD   =    10*60 ; // Check for key advertising every 10 minutes
+//static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD   =    10*60 ; // Check for key advertising every 10 minutes
-static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL = 24*60*60 ; // Advertise each key once a day at most.
+//static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL = 24*60*60 ; // Advertise each key once a day at most.
+static const time_t RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD   =    1 *60 ; // Check for key advertising every 10 minutes
+static const time_t RS_GROUTER_PUBLISH_KEY_TIME_INTERVAL =    2 *60 ; // Advertise each key once a day at most.
 
 p3GRouter::p3GRouter(p3LinkMgr *lm)
 	: p3Service(RS_SERVICE_TYPE_GROUTER), p3Config(CONFIG_TYPE_GROUTER), mLinkMgr(lm), grMtx("GRouter")
 {
 	addSerialType(new RsGRouterSerialiser()) ;
+
+	// Debug stuff. Create a random key and register it.
+	      uint8_t random_hash_buff[20] ;
+	      RSRandom::random_bytes(random_hash_buff,20) ;
+	      GRouterKeyId key(random_hash_buff) ;
+	      static GRouterServiceId client_id = 0x0300ae15 ;
+	      static std::string description = "Test string for debug purpose" ;
+
+	      registerKey(key,client_id,description) ;
 }
 
 int p3GRouter::tick()
 {
 	static time_t last_autowash_time = 0 ;
 	static time_t last_publish_campaign_time = 0 ;
+	static time_t last_debug_output_time = 0 ;
 
 	time_t now = time(NULL) ;
 
 	if(now > last_autowash_time + RS_GROUTER_AUTOWASH_PERIOD)
 	{
+		// route pending objects
+		//
+		routeObjects() ;
+
 		last_autowash_time = now ;
 		autoWash() ;
 	}
-	// handle incoming items
+	// Handle incoming items
 	// 
 	handleIncoming() ;
 	
-	// route pending objects
+	// Advertise published keys
-	//
-	routeObjects() ;
-
-	// advertise published keys
 	//
 	if(now > last_publish_campaign_time + RS_GROUTER_PUBLISH_CAMPAIGN_PERIOD)
 	{
 		last_publish_campaign_time = now ;
 		publishKeys() ;
+		_routing_matrix.updateRoutingProbabilities() ;
 	}
 
+	// Debug dump everything
+	//
+	if(now > last_debug_output_time + RS_GROUTER_DEBUG_OUTPUT_PERIOD)
+	{
+		last_debug_output_time = now ;
+		debugDump() ;
+	}
 	return 0 ;
 }
 
@@ -114,17 +135,24 @@ void p3GRouter::publishKeys()
 		{
 			// publish this key
 
+			std::cerr << "Publishing this key: " << std::endl;
+			std::cerr << "   Key id          : " << it->first.toStdString() << std::endl;
+			std::cerr << "   Service id      : " << std::hex << info.service_id << std::dec << std::endl;
+			std::cerr << "   Description     : " << info.description_string << std::endl;
+
 			RsGRouterPublishKeyItem item ;
 			item.diffusion_id = RSRandom::random_u32() ;
 			item.published_key = it->first ;
 			item.service_id = info.service_id ;
-			item.randomized_distance = 0 ;
+			item.randomized_distance = drand48() ;
 			item.description_string = info.description_string ;
 
 			// get list of connected friends, and broadcast to all of them
 			//
 			for(std::list<std::string>::const_iterator it(connected_peers.begin());it!=connected_peers.end();++it)
 			{
+				std::cerr << "    sending to " << (*it) << std::endl; 
+
 				RsGRouterPublishKeyItem *itm = new RsGRouterPublishKeyItem(item) ;
 				itm->PeerId(*it) ;
 
@@ -132,6 +160,7 @@ void p3GRouter::publishKeys()
 
 				sendItem(itm) ;
 			}
+			info.last_published_time = now ;
 		}
 	}
 }
@@ -148,6 +177,11 @@ bool p3GRouter::registerKey(const GRouterKeyId& key,const GRouterServiceId& clie
 
 	_owned_key_ids[key] = info ;
 
+	std::cerr << "Registered the following key: " << std::endl;
+	std::cerr << "   Key id      : " << key.toStdString() << std::endl;
+	std::cerr << "   Client id   : " << std::hex << client_id << std::dec << std::endl;
+	std::cerr << "   Description : " << description << std::endl;
+
 	return true ;
 }
 
@@ -251,4 +285,46 @@ bool p3GRouter::saveList(bool&,std::list<RsItem*>& items)
 	return false ;
 }
 
+// Dump everything
+//
+void p3GRouter::debugDump()
+{
+	time_t now = time(NULL) ;
+
+	std::cerr << "Full dump of Global Router state: " << std::endl; 
+	std::cerr << "  Owned keys : " << std::endl;
+
+	for(std::map<GRouterKeyId, GRouterPublishedKeyInfo>::const_iterator it(_owned_key_ids.begin());it!=_owned_key_ids.end();++it)
+	{
+		std::cerr << "    Key id          : " << it->first.toStdString() << std::endl;
+		std::cerr << "      Service id    : " << std::hex << it->second.service_id << std::dec << std::endl;
+		std::cerr << "      Description   : " << it->second.description_string << std::endl;
+		std::cerr << "      Last published: " << now - it->second.last_published_time << " secs ago" << std::endl;
+	}
+
+	std::cerr << "  Registered services: " << std::endl;
+
+	for(std::map<GRouterServiceId,GRouterClientService *>::const_iterator it(_registered_services.begin() );it!=_registered_services.end();++it)
+		std::cerr << "    " << std::hex << it->first << "   " << std::dec << (void*)it->second << std::endl;
+
+	std::cerr << "  Key diffusion cache: " << std::endl;
+
+	for(std::map<GRouterKeyPropagationId,time_t>::const_iterator it(_key_diffusion_time_stamps.begin() );it!=_key_diffusion_time_stamps.end();++it)
+		std::cerr << "    " << std::hex << it->first << "   " << std::dec << now - it->second << " secs ago" << std::endl;
+
+	std::cerr << "  Key diffusion items: " << std::endl;
+	std::cerr << "    [Not shown yet]    " << std::endl;
+
+	std::cerr << "  Data items: " << std::endl;
+	std::cerr << "    [Not shown yet]    " << std::endl;
+
+	std::cerr << "  Routing matrix: " << std::endl;
+
+	_routing_matrix.debugDump() ;
+
+	std::cerr << "  Routing probabilities: " << std::endl;
+}
+
+
+
 

libretroshare/src/grouter/p3grouter.h

@@ -109,7 +109,7 @@ class p3GRouter: public RsGRouter, public p3Service, public p3Config
 		//
 		virtual bool getRoutingCacheInfo(RoutingCacheInfo& info) { return false ;} 
 
-	private:
+	protected:
 		//===================================================//
 		//         Routing method handling                   //
 		//===================================================//
@@ -121,10 +121,12 @@ class p3GRouter: public RsGRouter, public p3Service, public p3Config
 		//
 		virtual int tick() ; 
 
+	private:
 		void autoWash() ;
 		void routeObjects() ;
 		void handleIncoming() ;
 		void publishKeys() ;
+		void debugDump() ;
 
 		//===================================================//
 		//                  p3Config methods                 //

libretroshare/src/serialiser/rsbaseserial.cc

@@ -210,7 +210,7 @@ bool setRawUFloat32(void *data,uint32_t size,uint32_t *offset,float f)
 	// 0.01% and most of the time less than 1e-05% The error is well distributed
 	// over numbers also.
 	//
-	uint32_t n = (uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0))) ;
+	uint32_t n = (f < 1e-7)?(~(uint32_t)0): ((uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0)))) ;
 
 	return setRawUInt32(data, size, offset, n);
 }