From 84194b6234c9156f03fb43b193a9aa4b94ef4a84 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 16 Jun 2018 22:39:35 +0200 Subject: [PATCH] sereral fixes to GXS distant search --- libretroshare/src/ft/ftserver.h | 2 + libretroshare/src/grouter/p3grouter.h | 1 + libretroshare/src/gxs/rsgxsnetservice.cc | 9 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 30 +++- libretroshare/src/gxs/rsgxsnettunnel.h | 8 + libretroshare/src/gxstunnel/p3gxstunnel.h | 2 + libretroshare/src/rsserver/rsinit.cc | 5 + libretroshare/src/services/p3idservice.cc | 1 - libretroshare/src/turtle/p3turtle.cc | 176 +++++++++++++--------- libretroshare/src/turtle/p3turtle.h | 8 +- libretroshare/src/turtle/rsturtleitem.cc | 4 +- libretroshare/src/turtle/rsturtleitem.h | 33 ++-- 12 files changed, 172 insertions(+), 107 deletions(-) diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 0833127cf..cd0101933 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -95,6 +95,8 @@ public: // Implements RsTurtleClientService // + + uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; } virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; //virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO diff --git a/libretroshare/src/grouter/p3grouter.h b/libretroshare/src/grouter/p3grouter.h index 8a3a54000..a4bc280ff 100644 --- a/libretroshare/src/grouter/p3grouter.h +++ b/libretroshare/src/grouter/p3grouter.h @@ -208,6 +208,7 @@ protected: // Interaction with turtle router // //===================================================// + uint16_t serviceId() const { return RS_SERVICE_TYPE_GROUTER; } virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ; virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/); virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index da94cb002..3a5c9ecb2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -272,12 +272,12 @@ NXS_NET_DEBUG_8 gxs distant sync ***/ -#define NXS_NET_DEBUG_0 1 -#define NXS_NET_DEBUG_1 1 +//#define NXS_NET_DEBUG_0 1 +//#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 -#define NXS_NET_DEBUG_5 1 +//#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 #define NXS_NET_DEBUG_8 1 @@ -5143,5 +5143,8 @@ bool RsGxsNetService::search(const std::string& substring,std::listturtleSearch(mem,size,this) ; } bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size) { - GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; + GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; @@ -985,14 +997,20 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d if(substring_sr != NULL) { - auto it = mSearchableServices.find(substring_sr->service) ; + GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl; std::list results ; + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + auto it = mSearchableServices.find(substring_sr->service) ; + if(it != mSearchableServices.end() && it->second->search(substring_sr->substring_match,results)) { RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ; + GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl; + search_result_item.service = substring_sr->service ; search_result_item.group_infos = results ; @@ -1044,11 +1062,13 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id { RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); + GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl; + RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; if(result_gs != NULL) { - std::cerr << "Received group summary result for search request " << std::hex << request_id << " for service " << result_gs->service << std::dec << ": " << std::endl; + GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 22917f35b..ade780fe8 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -187,6 +187,14 @@ public: */ bool getVirtualPeers(std::list& peers) ; // returns the virtual peers for this service + /*! + * \brief serviceId + * Overloads the method in RsTurtleClientService. + * \return + * The service id for RsGxsNetTunnel. + */ + uint16_t serviceId() const ; + /*! * \brief sendData * send data to this virtual peer, and takes memory ownership (deletes the item) diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.h b/libretroshare/src/gxstunnel/p3gxstunnel.h index cff0fd126..a5a53b437 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.h +++ b/libretroshare/src/gxstunnel/p3gxstunnel.h @@ -127,6 +127,8 @@ public: explicit p3GxsTunnelService(RsGixs *pids) ; virtual void connectToTurtleRouter(p3turtle *) ; + uint16_t serviceId() const { return RS_SERVICE_TYPE_GXS_TUNNEL ; } + // Creates the invite if the public key of the distant peer is available. // Om success, stores the invite in the map above, so that we can respond to tunnel requests. // diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 7fd67fa27..34b68ef11 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1621,6 +1621,11 @@ int RsServer::StartupRetroShare() serviceCtrl->registerServiceMonitor(mBwCtrl, mBwCtrl->getServiceInfo().mServiceType); /**************************************************************************/ + // Turtle search for GXS services + + mGxsNetTunnel->registerSearchableService(gxschannels_ns) ; + + /**************************************************************************/ //mConfigMgr->addConfiguration("ftserver.cfg", ftserver); // diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 5618568fa..eff0f5cb5 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -47,7 +47,6 @@ #include #include -#define DEBUG_IDS 1 /**** * #define DEBUG_IDS 1 * #define DEBUG_RECOGN 1 diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 1232b950c..233703492 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -930,7 +930,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #endif std::list search_results ; - item->performLocalSearch(req,search_results) ; + locked_performLocalSearch(item,req,search_results) ; for(auto it(search_results.begin());it!=search_results.end();++it) sendItem(*it) ; @@ -999,6 +999,97 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #endif } +// This function should be removed in the future, when file search will also use generic search items. + +void p3turtle::locked_performLocalSearch(RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& search_results) +{ + RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast(item) ; + + if(ftsearch != NULL) + { + locked_performLocalSearch_files(ftsearch,req,search_results) ; + return ; + } + + RsTurtleGenericSearchRequestItem *gnsearch = dynamic_cast(item) ; + + if(gnsearch != NULL) + { + locked_performLocalSearch_generic(gnsearch,req,search_results) ; + return ; + } +} + +void p3turtle::locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +{ + unsigned char *search_result_data = NULL ; + uint32_t search_result_data_len = 0 ; + + auto it = _registered_services.find(item->service_id) ; + + if(it == _registered_services.end()) + return ; + + if(it->second->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len)) + { + RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ; + + result_item->result_data = search_result_data ; + result_item->result_data_len = search_result_data_len ; + + result.push_back(result_item) ; + } +} + +void p3turtle::locked_performLocalSearch_files(RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +{ +#ifdef P3TURTLE_DEBUG + std::cerr << "Performing rsFiles->search()" << std::endl ; +#endif + // now, search! + std::list initialResults ; + item->search(initialResults) ; + +#ifdef P3TURTLE_DEBUG + std::cerr << initialResults.size() << " matches found." << std::endl ; +#endif + result.clear() ; + RsTurtleFTSearchResultItem *res_item = NULL ; + uint32_t item_size = 0 ; + + static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; + + for(auto it(initialResults.begin());it!=initialResults.end();++it) + { + if(res_item == NULL) + { + res_item = new RsTurtleFTSearchResultItem ; + item_size = 0 ; + + res_item->depth = 0 ; + res_item->request_id = item->request_id ; + res_item->PeerId(item->PeerId()) ; // send back to the same guy + + result.push_back(res_item) ; + } + res_item->result.push_back(*it); + + // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. + // + ++req.result_count ; // increase hit number for this particular search request. + + item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; + + if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; +#endif + res_item = NULL ; // forces creation of a new item. + } + } +} + void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) { std::list > results_to_notify_off_mutex ; @@ -1771,59 +1862,7 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item) // ------------------------------ IO with libretroshare ----------------------------// // -----------------------------------------------------------------------------------// // -void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const -{ -#ifdef P3TURTLE_DEBUG - std::cerr << "Performing rsFiles->search()" << std::endl ; -#endif - // now, search! - std::list initialResults ; - search(initialResults) ; -#ifdef P3TURTLE_DEBUG - std::cerr << initialResults.size() << " matches found." << std::endl ; -#endif - result.clear() ; - RsTurtleFTSearchResultItem *res_item = NULL ; - uint32_t item_size = 0 ; - - static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; - - for(auto it(initialResults.begin());it!=initialResults.end();++it) - { - if(res_item == NULL) - { - res_item = new RsTurtleFTSearchResultItem ; - item_size = 0 ; - - res_item->depth = 0 ; - res_item->request_id = request_id ; - res_item->PeerId(PeerId()) ; // send back to the same guy - - result.push_back(res_item) ; - } - res_item->result.push_back(*it); - - // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. - // - ++req.result_count ; // increase hit number for this particular search request. - - item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; - - if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) - { -#ifdef P3TURTLE_DEBUG - std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; -#endif - res_item = NULL ; // forces creation of a new item. - } - } -} - -void RsTurtleGenericSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const -{ - std::cerr << "(EE) p3turtle: Missing code to perform actual GXS search" << std::endl; -} void RsTurtleStringSearchRequestItem::search(std::list& result) const { @@ -1967,22 +2006,22 @@ TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t s // Form a request packet that simulates a request from us. // - RsTurtleGenericSearchRequestItem *item = new RsTurtleGenericSearchRequestItem ; + RsTurtleGenericSearchRequestItem item ; #ifdef P3TURTLE_DEBUG std::cerr << "performing search. OwnId = " << _own_id << std::endl ; #endif - item->PeerId(_own_id) ; - item->service_id = client_service->serviceId(); - item->search_data = search_bin_data ; - item->search_data_len = search_bin_data_len ; - item->request_id = id ; - item->depth = 0 ; + item.PeerId(_own_id) ; + item.service_id = client_service->serviceId(); + item.search_data = search_bin_data ; + item.search_data_len = search_bin_data_len ; + item.request_id = id ; + item.depth = 0 ; // send it - handleSearchRequest(item) ; + handleSearchRequest(&item) ; return id ; } @@ -2056,10 +2095,10 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& if(_registered_services.empty()) std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl; - for(std::list::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it) - if( (*it)->handleTunnelRequest(hash,peer_id)) + for(auto it(_registered_services.begin());it!=_registered_services.end();++it) + if( (*it).second->handleTunnelRequest(hash,peer_id)) { - service = *it ; + service = it->second ; return true ; } @@ -2069,14 +2108,9 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& void p3turtle::registerTunnelService(RsTurtleClientService *service) { -#ifdef P3TURTLE_DEBUG - for(std::list::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it) - if(service == *it) - throw std::runtime_error("p3turtle::registerTunnelService(): Cannot register the same service twice. Please fix the code!") ; -#endif - std::cerr << "p3turtle: registered new tunnel service " << (void*)service << std::endl; + std::cerr << "p3turtle: registered new tunnel service with ID=" << std::hex << service->serviceId() << std::dec << " and pointer " << (void*)service << std::endl; - _registered_services.push_back(service) ; + _registered_services[service->serviceId()] = service ; _serialiser->registerClientService(service) ; } diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 04d90c812..0c30653f3 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -397,7 +397,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config //------ Functions connecting the turtle router to other components.----------// /// Performs a search calling local cache and search structure. - void performLocalSearch(const std::string& match_string,std::list& result) ; + void locked_performLocalSearch (RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; + void locked_performLocalSearch_files (RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; + void locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; /// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer. virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service); @@ -419,7 +421,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::map _tunnel_requests_origins ; /// stores adequate tunnels for each file hash locally managed - std::map _incoming_file_hashes ; + std::map _incoming_file_hashes ; /// stores file info for each file we provide. std::map _outgoing_tunnel_client_services ; @@ -434,7 +436,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::set _hashes_to_remove ; /// List of client services that have regitered. - std::list _registered_services ; + std::map _registered_services ; time_t _last_clean_time ; time_t _last_tunnel_management_time ; diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index ce3a4450a..d600b4877 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -75,9 +75,11 @@ RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const { RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ; + memcpy(sr,this,sizeof(RsTurtleGenericSearchRequestItem)) ; + sr->search_data = (unsigned char*)rs_malloc(search_data_len) ; memcpy(sr->search_data,search_data,search_data_len) ; - sr->search_data_len = search_data_len ; + return sr ; } template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r) diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 035490bc1..c5e3bc727 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -57,26 +57,18 @@ class RsTurtleItem: public RsItem // +---- RsTurtleSearchRequestItem // | | // | +---- RsTurtleFileSearchRequestItem -// | | | -// | | +---- RsTurtleFileSearchRequestItem -// | | | -// | | +---- RsTurtleStringSearchRequestItem -// | | | -// | | +---- RsTurtleReqExpSearchRequestItem +// | | | +// | | +---- RsTurtleStringSearchRequestItem +// | | | +// | | +---- RsTurtleReqExpSearchRequestItem // | | -// | +---- RsTurtleGxsSearchRequestItem -// | | -// | +---- RsTurtleGxsGroupRequestItem +// | +---- RsTurtleGenericSearchRequestItem // | // +---- RsTurtleSearchResultItem // | // +---- RsTurtleFTSearchResultItem // | -// +---- RsTurtleGxsSearchResultItem -// | -// +---- RsTurtleGxsSearchResultGroupSummaryItem -// | -// +---- RsTurtleGxsSearchResultGroupDataItem +// +---- RsTurtleGenericSearchResultItem // class RsTurtleSearchResultItem ; @@ -89,8 +81,6 @@ class RsTurtleSearchRequestItem: public RsTurtleItem virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const = 0 ; // abstracts the search method - virtual std::string GetKeywords() = 0; uint32_t request_id ; // randomly generated request id. @@ -103,9 +93,6 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {} virtual ~RsTurtleFileSearchRequestItem() {} - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method - - protected: virtual void search(std::list &) const =0; }; @@ -115,8 +102,9 @@ class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem RsTurtleStringSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {} virtual ~RsTurtleStringSearchRequestItem() {} - std::string match_string ; // string to match + virtual void search(std::list &) const ; + std::string match_string ; // string to match std::string GetKeywords() { return match_string; } virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleStringSearchRequestItem(*this) ; } @@ -124,7 +112,6 @@ class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem void clear() { match_string.clear() ; } protected: - virtual void search(std::list &) const ; void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; @@ -144,10 +131,11 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem return exs; } + virtual void search(std::list &) const ; + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } void clear() { expr = RsRegularExpression::LinearizedExpression(); } protected: - virtual void search(std::list &) const ; void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; @@ -166,7 +154,6 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem virtual RsTurtleSearchRequestItem *clone() const ; void clear() { free(search_data); search_data=NULL; search_data_len=0; } - virtual void performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const; protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);