Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

added per-item type limit for turtle search results

Browse Source
This commit is contained in:
csoler 2018-07-05 17:40:06 +02:00
parent 4a64ea5f1f
commit 98f8e4da0a
No known key found for this signature in database
GPG Key ID: 7BCA522266C0804C
7 changed files with 59 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -36,6 +36,8 @@
#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : "
#define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : "
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
@ -1011,7 +1013,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size)
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
@ -1023,6 +1025,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
std::list<RsGxsGroupSummary> results ;
RS_STACK_MUTEX(mGxsNetTunnelMtx);
@ -1057,6 +1061,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;

2
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -244,7 +244,7 @@ public:
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len);
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
// Overloaded from RsTickingThread

52
libretroshare/src/turtle/p3turtle.cc
View File

@ -83,17 +83,18 @@ void TS_dumpState() ;
// - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5
// - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels.
//
static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache.
static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache
static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns.
static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel.
static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec.
static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds
static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs
static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs.
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS =5000 ; /// maximum number of search results forwarded back to the source.
static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache.
static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache
static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns.
static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel.
static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec.
static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds
static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs
static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs.
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES =5000 ; /// maximum number of search results forwarded back to the source.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ; /// default maximum number of search results forwarded back source.
static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
@ -916,6 +917,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
// Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain.
uint32_t search_result_count = 0;
uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT;
if(item->PeerId() != _own_id) // is the request not coming from us?
{
@ -924,7 +926,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
#endif
std::list<RsTurtleSearchResultItem*> search_results ;
performLocalSearch(item,search_result_count,search_results) ;
performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ;
for(auto it(search_results.begin());it!=search_results.end();++it)
{
@ -948,13 +950,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
req.result_count = search_result_count;
req.keywords = item->GetKeywords() ;
req.service_id = item->serviceId() ;
req.max_allowed_hits = max_allowed_hits;
// if enough has been sent back already, do not sarch further
#ifdef P3TURTLE_DEBUG
std::cerr << " result count = " << req.result_count << std::endl;
#endif
if(req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
if(req.result_count >= max_allowed_hits)
return ;
// If search depth not too large, also forward this search request to all other peers.
@ -1014,13 +1017,13 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
// This function should be removed in the future, when file search will also use generic search items.
void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results)
void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results,uint32_t& max_allowed_hits)
{
RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast<RsTurtleFileSearchRequestItem*>(item) ;
if(ftsearch != NULL)
{
performLocalSearch_files(ftsearch,req_result_count,search_results) ;
performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ;
return ;
}
@ -1028,12 +1031,12 @@ void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_
if(gnsearch != NULL)
{
performLocalSearch_generic(gnsearch,req_result_count,search_results) ;
performLocalSearch_generic(gnsearch,req_result_count,search_results,max_allowed_hits) ;
return ;
}
}
void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result)
void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
{
unsigned char *search_result_data = NULL ;
uint32_t search_result_data_len = 0 ;
@ -1050,7 +1053,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item
client = it->second ;
}
if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len))
if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len,max_allowed_hits))
{
RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ;
@ -1061,7 +1064,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item
}
}
void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result)
void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "Performing rsFiles->search()" << std::endl ;
@ -1078,6 +1081,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
uint32_t item_size = 0 ;
static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_FILES;
for(auto it(initialResults.begin());it!=initialResults.end();++it)
{
@ -1096,7 +1100,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ;
if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= max_allowed_hits)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
@ -1151,18 +1155,18 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
uint32_t n = item->count(); // not so good!
if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
if(it->second.result_count >= it->second.max_allowed_hits)
{
std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
return ;
}
if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS)
if(it->second.result_count + n > it->second.max_allowed_hits)
{
for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i)
for(uint32_t i=it->second.result_count + n; i>it->second.max_allowed_hits;--i)
item->pop() ;
it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ;
it->second.result_count = it->second.max_allowed_hits ;
}
else
it->second.result_count += n ;

7
libretroshare/src/turtle/p3turtle.h
View File

@ -174,6 +174,7 @@ class TurtleSearchRequestInfo
uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses.
std::string keywords;
uint16_t service_id; // ID of the client service who issues the request. This is null if the request does not have a local origin.
uint32_t max_allowed_hits;// Max number of hits allowed for this search. This actually depends on the type of search (files, GXS groups, GXS group data, etc)
};
class TurtleTunnelRequestInfo
{
@ -397,9 +398,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
//------ Functions connecting the turtle router to other components.----------//
/// Performs a search calling local cache and search structure.
void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result) ;
void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits) ;
void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
/// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer.
virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service);

6
libretroshare/src/turtle/rsturtleitem.cc
View File

@ -50,6 +50,11 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
return NULL ;
}
std::string RsTurtleGenericSearchRequestItem::GetKeywords()
{
return std::string("Generic search : " + RsUtil::BinToHex(search_data,search_data_len,10));
}
void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ;
@ -67,6 +72,7 @@ void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::Seria
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
RsTypeSerializer::serial_process<uint8_t >(j,ctx,request_type,"request_type") ;
RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ;
RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;

11
libretroshare/src/turtle/rsturtleitem.h
View File

@ -82,7 +82,7 @@ class RsTurtleSearchRequestItem: public RsTurtleItem
virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods
virtual std::string GetKeywords() = 0;
virtual uint16_t serviceId() = 0 ;
virtual uint16_t serviceId() const= 0 ;
uint32_t request_id ; // randomly generated request id.
uint16_t depth ; // Used for limiting search depth.
@ -94,7 +94,7 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem
RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {}
virtual ~RsTurtleFileSearchRequestItem() {}
virtual uint16_t serviceId() { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual void search(std::list<TurtleFileInfo> &) const =0;
};
@ -149,12 +149,15 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem
uint16_t service_id ; // service to search
uint32_t search_data_len ;
uint8_t request_type ; // type of request. This is used to limit the number of responses.
unsigned char *search_data ;
std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); }
virtual uint16_t serviceId() { return service_id ; }
std::string GetKeywords() ;
virtual uint16_t serviceId() const { return service_id ; }
virtual RsTurtleSearchRequestItem *clone() const ;
virtual uint32_t requestType() const { return request_type; }
void clear() { free(search_data); search_data=NULL; search_data_len=0; }
protected:

7
libretroshare/src/turtle/turtleclientservice.h
View File

@ -101,10 +101,15 @@ class RsTurtleClientService
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and forwarded
*
* \return true if the search is successful.
*/
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,uint32_t /*search_request_data_len*/,unsigned char *& /*search_result_data*/,uint32_t& /*search_result_data_len*/)
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
uint32_t /*search_request_data_len*/,
unsigned char *& /*search_result_data*/,
uint32_t& /*search_result_data_len*/,
uint32_t& /* max_allows_hits */)
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
return false;