added per-item type limit for turtle search results

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -36,6 +36,8 @@
 #define GXS_NET_TUNNEL_DEBUG()             std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : "
 #define GXS_NET_TUNNEL_ERROR()             std::cerr << "(EE) GXS_NET_TUNNEL ERROR : "
 
+static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA    =   1;
+static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH  = 100;
 
 RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
 {
@@ -1011,7 +1013,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size)
+bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
 {
 	GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
 
@@ -1023,6 +1025,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d
     {
 		GXS_NET_TUNNEL_DEBUG() << "  : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
 
+        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
+
         std::list<RsGxsGroupSummary> results ;
 
 		RS_STACK_MUTEX(mGxsNetTunnelMtx);
@@ -1057,6 +1061,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d
 		RS_STACK_MUTEX(mGxsNetTunnelMtx);
 		auto it = mSearchableServices.find(substring_gr->service) ;
 
+        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
+
         unsigned char *encrypted_group_data = NULL ;
         uint32_t encrypted_group_data_len = 0 ;
 

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -244,7 +244,7 @@ public:
        * \brief receiveSearchRequest
        * 			See RsTurtleClientService::@
        */
-	  virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len);
+	  virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
 	  virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
 
 	  // Overloaded from RsTickingThread

libretroshare/src/turtle/p3turtle.cc

@@ -83,17 +83,18 @@ void TS_dumpState() ;
 //    - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5
 //    - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels.
 //
-static const time_t TUNNEL_REQUESTS_LIFE_TIME 	         = 240 ;		/// life time for tunnel requests in the cache.
+static const time_t TUNNEL_REQUESTS_LIFE_TIME 	           = 240 ;		/// life time for tunnel requests in the cache.
-static const time_t SEARCH_REQUESTS_LIFE_TIME 	         = 240 ;		/// life time for search requests in the cache
+static const time_t SEARCH_REQUESTS_LIFE_TIME 	           = 240 ;		/// life time for search requests in the cache
-static const time_t REGULAR_TUNNEL_DIGGING_TIME          = 300 ;		/// maximum interval between two tunnel digging campaigns.
+static const time_t REGULAR_TUNNEL_DIGGING_TIME            = 300 ;		/// maximum interval between two tunnel digging campaigns.
-static const time_t MAXIMUM_TUNNEL_IDLE_TIME 	         =  60 ;		/// maximum life time of an unused tunnel.
+static const time_t MAXIMUM_TUNNEL_IDLE_TIME 	           =  60 ;		/// maximum life time of an unused tunnel.
-static const time_t EMPTY_TUNNELS_DIGGING_TIME 	         =  50 ;		/// look into tunnels regularly every 50 sec.
+static const time_t EMPTY_TUNNELS_DIGGING_TIME 	           =  50 ;		/// look into tunnels regularly every 50 sec.
-static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE	         =   5 ;		/// estimate tunnel speed every 5 seconds
+static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE	           =   5 ;		/// estimate tunnel speed every 5 seconds
-static const time_t TUNNEL_CLEANING_LAPS_TIME  	         =  10 ;		/// clean tunnels every 10 secs
+static const time_t TUNNEL_CLEANING_LAPS_TIME  	           =  10 ;		/// clean tunnels every 10 secs
-static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS =   2 ;                /// Tunnel management calls every 2 secs.
+static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS   =   2 ;        /// Tunnel management calls every 2 secs.
-static const uint32_t MAX_TUNNEL_REQS_PER_SECOND         =   1 ;		/// maximum number of tunnel requests issued per second. Was 0.5 before
+static const uint32_t MAX_TUNNEL_REQS_PER_SECOND           =   1 ;		/// maximum number of tunnel requests issued per second. Was 0.5 before
-static const uint32_t MAX_ALLOWED_SR_IN_CACHE            = 120 ;		/// maximum number of search requests allowed in cache. That makes 2 per sec.
+static const uint32_t MAX_ALLOWED_SR_IN_CACHE              = 120 ;		/// maximum number of search requests allowed in cache. That makes 2 per sec.
-static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS      =5000 ;		/// maximum number of search results forwarded back to the source.
+static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES  =5000 ;		/// maximum number of search results forwarded back to the source.
+static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ;		/// default maximum number of search results forwarded back source.
 
 static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
 
@@ -916,6 +917,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
     // Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain.
 
     uint32_t search_result_count = 0;
+    uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT;
 
 	if(item->PeerId() != _own_id) // is the request not coming from us?
 	{
@@ -924,7 +926,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 #endif
         std::list<RsTurtleSearchResultItem*> search_results ;
 
-        performLocalSearch(item,search_result_count,search_results) ;
+        performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ;
 
         for(auto it(search_results.begin());it!=search_results.end();++it)
         {
@@ -948,13 +950,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 	req.result_count = search_result_count;
 	req.keywords = item->GetKeywords() ;
 	req.service_id = item->serviceId() ;
+    req.max_allowed_hits = max_allowed_hits;
 
 	// if enough has been sent back already, do not sarch further
 
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  result count = " << req.result_count << std::endl;
 #endif
-	if(req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
+	if(req.result_count >= max_allowed_hits)
 		return ;
 
 	// If search depth not too large, also forward this search request to all other peers.
@@ -1014,13 +1017,13 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 
 // This function should be removed in the future, when file search will also use generic search items.
 
-void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results)
+void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results,uint32_t& max_allowed_hits)
 {
     RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast<RsTurtleFileSearchRequestItem*>(item) ;
 
     if(ftsearch != NULL)
     {
-        performLocalSearch_files(ftsearch,req_result_count,search_results) ;
+        performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ;
         return ;
     }
 
@@ -1028,12 +1031,12 @@ void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_
 
     if(gnsearch != NULL)
     {
-        performLocalSearch_generic(gnsearch,req_result_count,search_results) ;
+        performLocalSearch_generic(gnsearch,req_result_count,search_results,max_allowed_hits) ;
         return ;
     }
 }
 
-void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result)
+void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
 {
     unsigned char *search_result_data = NULL ;
     uint32_t search_result_data_len = 0 ;
@@ -1050,7 +1053,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item
 		client = it->second ;
 	}
 
-    if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len))
+    if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len,max_allowed_hits))
     {
 		RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ;
 
@@ -1061,7 +1064,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item
     }
 }
 
-void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result)
+void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
 {
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "Performing rsFiles->search()" << std::endl ;
@@ -1078,6 +1081,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
 	uint32_t item_size = 0 ;
 
 	static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
+    max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_FILES;
 
 	for(auto it(initialResults.begin());it!=initialResults.end();++it)
 	{
@@ -1096,7 +1100,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
 
 		item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ;
 
-		if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
+		if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= max_allowed_hits)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
@@ -1151,18 +1155,18 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 
 			uint32_t n = item->count(); // not so good!
 
-			if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
+			if(it->second.result_count >= it->second.max_allowed_hits)
 			{
 				std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
 				return ;
 			}
 
-			if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS)
+			if(it->second.result_count + n > it->second.max_allowed_hits)
 			{
-				for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i)
+				for(uint32_t i=it->second.result_count + n; i>it->second.max_allowed_hits;--i)
 					item->pop() ;
 
-				it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ;
+				it->second.result_count = it->second.max_allowed_hits ;
 			}
 			else
 				it->second.result_count += n ;

libretroshare/src/turtle/p3turtle.h

@@ -174,6 +174,7 @@ class TurtleSearchRequestInfo
 		uint32_t     result_count;    // responses to this request. Useful to avoid spamming tunnel responses.
 		std::string  keywords;
         uint16_t     service_id;      // ID of the client service who issues the request. This is null if the request does not have a local origin.
+        uint32_t     max_allowed_hits;// Max number of hits allowed for this search. This actually depends on the type of search (files, GXS groups, GXS group data, etc)
 };
 class TurtleTunnelRequestInfo
 {
@@ -397,9 +398,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		//------ Functions connecting the turtle router to other components.----------//
 		
 		/// Performs a search calling local cache and search structure.
-		void performLocalSearch        (RsTurtleSearchRequestItem        *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
+		void performLocalSearch        (RsTurtleSearchRequestItem        *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits) ;
-		void performLocalSearch_files  (RsTurtleFileSearchRequestItem    *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
+		void performLocalSearch_files  (RsTurtleFileSearchRequestItem    *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
-		void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
+		void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
 
 		/// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer.
 		virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service);

libretroshare/src/turtle/rsturtleitem.cc

@@ -50,6 +50,11 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
 	return NULL ;
 }
 
+std::string RsTurtleGenericSearchRequestItem::GetKeywords()
+{
+    return std::string("Generic search : " + RsUtil::BinToHex(search_data,search_data_len,10));
+}
+
 void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process          (j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ;
@@ -67,6 +72,7 @@ void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::Seria
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
+    RsTypeSerializer::serial_process<uint8_t >(j,ctx,request_type,"request_type") ;
 
     RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ;
     RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;

libretroshare/src/turtle/rsturtleitem.h

@@ -82,7 +82,7 @@ class RsTurtleSearchRequestItem: public RsTurtleItem
 		virtual RsTurtleSearchRequestItem *clone() const = 0 ;					// used for cloning in routing methods
 
 		virtual std::string GetKeywords() = 0;
-        virtual uint16_t serviceId() = 0 ;
+        virtual uint16_t serviceId() const= 0 ;
 
 		uint32_t request_id ; 		// randomly generated request id.
 		uint16_t depth ;				// Used for limiting search depth.
@@ -94,7 +94,7 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem
         RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {}
         virtual ~RsTurtleFileSearchRequestItem() {}
 
-        virtual uint16_t serviceId() { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
+        virtual uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
 		virtual void search(std::list<TurtleFileInfo> &) const =0;
 };
 
@@ -149,12 +149,15 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem
 
         uint16_t service_id ;		// service to search
         uint32_t search_data_len ;
+        uint8_t  request_type ;     // type of request. This is used to limit the number of responses.
         unsigned char *search_data ;
 
-		std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); }
+		std::string GetKeywords() ;
-        virtual uint16_t serviceId() { return service_id ; }
+        virtual uint16_t serviceId() const { return service_id ; }
 
 		virtual RsTurtleSearchRequestItem *clone() const ;
+        virtual uint32_t requestType() const { return request_type; }
+
 		void clear() { free(search_data); search_data=NULL; search_data_len=0; }
 
 	protected:

libretroshare/src/turtle/turtleclientservice.h

@@ -101,10 +101,15 @@ class RsTurtleClientService
          * \param search_request_data_len  length of the serialized search data
          * \param search_result_data       generic serialized search result data
          * \param search_result_data_len   length of the serialized search result data
+         * \param max_allowed_hits         max number of hits allowed to be sent back and forwarded
          *
          * \return true if the search is successful.
          */
-    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,uint32_t /*search_request_data_len*/,unsigned char *& /*search_result_data*/,uint32_t& /*search_result_data_len*/)
+    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
+                                          uint32_t /*search_request_data_len*/,
+                                          unsigned char *& /*search_result_data*/,
+                                          uint32_t& /*search_result_data_len*/,
+                                          uint32_t& /* max_allows_hits */)
 		{
 			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
             return false;