From 6fb459ce642cd04377ee3975560902b9d7f65577 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Jun 2018 22:34:17 +0200 Subject: [PATCH] added logic for GXS search in RsGxsNetTunnel and Gxs client net service --- libretroshare/src/ft/ftserver.cc | 2 +- libretroshare/src/gxs/rsgxsnetservice.cc | 10 +- libretroshare/src/gxs/rsgxsnetservice.h | 8 + libretroshare/src/gxs/rsgxsnettunnel.cc | 230 ++++++++++++++++-- libretroshare/src/gxs/rsgxsnettunnel.h | 18 +- libretroshare/src/gxs/rsnxs.h | 24 ++ libretroshare/src/turtle/p3turtle.cc | 2 +- .../src/turtle/turtleclientservice.h | 2 +- 8 files changed, 273 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 3172cfd89..a454be33f 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -1462,7 +1462,7 @@ TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::Lineariz return mTurtleRouter->turtleSearch(expr) ; } -#warning we should do this here, but for now it's done by turtle router. +#warning we should do this here, but for now it is done by turtle router. // // Dont delete the item. The client (p3turtle) is doing it after calling this. // // // void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index b0d1743d6..d83ceed54 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5105,9 +5105,15 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) void RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { - mGxsNetTunnel->turtleGroupRequest(group_id) ; + mGxsNetTunnel->turtleGroupRequest(group_id,this) ; } void RsGxsNetService::turtleSearchRequest(const std::string& match_string) { - mGxsNetTunnel->turtleSearchRequest(match_string) ; + mGxsNetTunnel->turtleSearchRequest(match_string,this) ; +} + +bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) +{ +#warning MISSING CODE HERE! + return true ; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index aa8f8288c..9643f49bf 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -122,8 +122,16 @@ public: virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; } virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; } + /*! + * \brief Search methods. + * These four methods are used to request distant search and receive the results. + * \param group_id + */ virtual void turtleGroupRequest(const RsGxsGroupId& group_id); virtual void turtleSearchRequest(const std::string& match_string); + + virtual bool search(const std::string& substring,std::list& group_infos) ; + /*! * pauses synchronisation of subscribed groups and request for group id * from peers diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 964f15781..7cd657a55 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -27,6 +27,7 @@ #include "util/rstime.h" #include "retroshare/rspeers.h" #include "serialiser/rstypeserializer.h" +#include "gxs/rsnxs.h" #include "rsgxsnettunnel.h" #define DEBUG_RSGXSNETTUNNEL 1 @@ -52,9 +53,13 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ; class RsGxsNetTunnelItem: public RsItem { @@ -107,6 +112,57 @@ public: Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. }; +class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {} + virtual ~RsGxsNetTunnelTurtleSearchSubstringItem() {} + + uint16_t service ; + std::string substring_match ; + + virtual void clear() { substring_match.clear() ; } + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_KEY,substring_match,"substring_match") ; + } +}; + +class RsGxsNetTunnelTurtleSearchGroupRequestItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchGroupRequestItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST) {} + virtual ~RsGxsNetTunnelTurtleSearchGroupRequestItem() {} + + uint16_t service ; + Sha1CheckSum hashed_group_id ; + + virtual void clear() { hashed_group_id.clear() ; } + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,hashed_group_id,"hashed_group_id") ; + } +}; + +class RsGxsNetTunnelTurtleSearchGroupSummaryItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchGroupSummaryItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY) {} + virtual ~RsGxsNetTunnelTurtleSearchGroupSummaryItem() {} + + uint16_t service ; + std::list group_infos; + + virtual void clear() { group_infos.clear() ; } + + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ; + } +}; class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: @@ -132,10 +188,32 @@ public: } }; +template<> +void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, RsGxsGroupSummary& gs, const std::string& member_name ) +{ + RsTypeSerializer::serial_process(j,ctx,gs.group_id,member_name+"-group_id") ; // RsGxsGroupId group_id ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_COMMENT,gs.group_description,member_name+"-group_description") ; // std::string group_description ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ; + RsTypeSerializer::serial_process(j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ; + RsTypeSerializer::serial_process(j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ; + RsTypeSerializer::serial_process(j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ; + RsTypeSerializer::serial_process(j,ctx,gs.last_message_ts,member_name+"-last_message_ts") ; // time_t last_message_ts ; +} + //===========================================================================================================================================// // Interface with rest of the software // //===========================================================================================================================================// +bool RsGxsNetTunnelService::registerSearchableService(RsNetworkExchangeService *gxs_service) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + mSearchableServices[gxs_service->serviceType()] = gxs_service ; + + return true; +} + class DataAutoDelete { public: @@ -853,22 +931,144 @@ RsSerialiser *RsGxsNetTunnelService::setupSerialiser() return ser ; } -bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len) +//===========================================================================================================================================// +// Turtle Search system // +//===========================================================================================================================================// + +TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id,RsNetworkExchangeService *client_service) { - std::cerr << __PRETTY_FUNCTION__ << ": received a request. Code needed to handle it" << std::endl; + Sha1CheckSum hashed_group_id = RsDirUtil::sha1sum(group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; + + GXS_NET_TUNNEL_DEBUG() << ": starting a turtle group request for grp \"" << group_id << "\" hashed to \"" << hashed_group_id << "\"" << std::endl; + + RsGxsNetTunnelTurtleSearchGroupRequestItem search_item ; + search_item.hashed_group_id = hashed_group_id ; + search_item.service = client_service->serviceType() ; + + uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; + unsigned char *mem = (unsigned char*)rs_malloc(size) ; + + if(mem == NULL) + return 0 ; + + RsGxsNetTunnelSerializer().serialise(&search_item,mem,&size); + + return mTurtle->turtleSearch(mem,size,this) ; +} + +TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) +{ + GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string\"" << match_string << "\"" << std::endl; + + RsGxsNetTunnelTurtleSearchSubstringItem search_item ; + search_item.substring_match = match_string ; + search_item.service = client_service->serviceType() ; + + uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; + unsigned char *mem = (unsigned char*)rs_malloc(size) ; + + if(mem == NULL) + return 0 ; + + RsGxsNetTunnelSerializer().serialise(&search_item,mem,&size); + + return mTurtle->turtleSearch(mem,size,this) ; +} + +bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size) +{ + GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; + + RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; + + RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast(item) ; + + if(substring_sr != NULL) + { + auto it = mSearchableServices.find(substring_sr->service) ; + + std::list results ; + + if(it != mSearchableServices.end() && it->second->search(substring_sr->substring_match,results)) + { + RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ; + + search_result_item.service = substring_sr->service ; + search_result_item.group_infos = results ; + + search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; + search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ; + + if(search_result_data == NULL) + return false ; + + RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); + + return true ; + } + } + + RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast(item) ; + + if(substring_gr != NULL) + { +#ifdef TODO + auto it = mSearchableGxsServices.find(substring_sr->service) ; + + RsNxsGrp *grp = NULL ; + + if(it != mSearchableGxsServices.end() && it->second.search(substring_sr->group_id,grp)) + { + RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ; + + search_result_item.service = substring_sr->service ; + search_result_item.group_infos = results ; + + search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; + search_result_data = (unsigned char*)rs_malloc(size) ; + + if(search_result_data == NULL) + return false ; + + RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); + + return true ; + } +#endif + } + return false ; } -void RsGxsNetTunnelService::receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) + +void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len) { - std::cerr << __PRETTY_FUNCTION__ << ": received a search result. Code needed to handle it" << std::endl; + RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); + + RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; + + if(result_gs != NULL) + { + + } } -void RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id) -{ - std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle group request not implemented yet" << std::endl; -} -void RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string) -{ - std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle search request not implemented yet" << std::endl; -} + + + + + + + + + + + + + + + + + + + diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index f85b997f5..22917f35b 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -105,6 +105,7 @@ typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; class RsGxsNetTunnelItem ; +class RsNetworkExchangeService ; struct RsGxsNetTunnelVirtualPeerInfo { @@ -159,6 +160,15 @@ public: RsGxsNetTunnelService() ; virtual ~RsGxsNetTunnelService() ; + /*! + * \brief registerSearchableService + * Adds the network exchange service as a possible search source. This is used to allow distant search on the corresponding + * GXS service. + * \return + * always returns true. + */ + bool registerSearchableService(RsNetworkExchangeService *) ; + /*! * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released @@ -219,15 +229,15 @@ public: */ virtual void connectToTurtleRouter(p3turtle *tr) ; - void turtleGroupRequest(const RsGxsGroupId& group_id) ; - void turtleSearchRequest(const std::string& match_string) ; + TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ; + TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ; /*! * \brief receiveSearchRequest * See RsTurtleClientService::@ */ virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len); - virtual void receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) ; + virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len); // Overloaded from RsTickingThread @@ -270,6 +280,8 @@ private: std::map > > mIncomingData; // list of incoming data items + std::map mSearchableServices ; + /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to * hide the real group id. diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 59d2bf209..5dfcde9d6 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -58,6 +58,26 @@ * 2 transfers only between group * - the also group matrix settings which is by default everyone can transfer to each other */ + +/*! + * \brief The RsGxsGroupSymmary struct + * This structure is used to transport group summary information when a GXS service is searched. It contains the group information + * as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make + * search responses as light as possible. + */ +struct RsGxsGroupSummary +{ + RsGxsGroupId group_id ; + + std::string group_name ; + std::string group_description ; + std::string search_context ; + RsGxsId author_id ; + time_t publish_ts ; + uint32_t number_of_messages ; + time_t last_message_ts ; +}; + class RsNetworkExchangeService { public: @@ -65,6 +85,7 @@ public: RsNetworkExchangeService(){ return;} virtual ~RsNetworkExchangeService() {} + virtual uint16_t serviceType() const =0; /*! * Use this to set how far back synchronisation of messages should take place * @param age in seconds the max age a sync/store item can to be allowed in a synchronisation @@ -83,6 +104,9 @@ public: virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; virtual void turtleSearchRequest(const std::string& match_string)=0; + + virtual bool search(const std::string& substring,std::list& group_infos) =0; + /*! * Initiates a search through the network * This returns messages which contains the search terms set in RsGxsSearch diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 9914d510c..1232b950c 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -1085,7 +1085,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) RsTurtleGenericSearchResultItem *gnsr = dynamic_cast(it->first) ; if(gnsr!=NULL) - (*it).second->receiveSearchResult(gnsr->result_data,gnsr->result_data_len) ; + (*it).second->receiveSearchResult(gnsr->request_id,gnsr->result_data,gnsr->result_data_len) ; } } diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index 277c8d5c3..9bdd15ad4 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -117,7 +117,7 @@ class RsTurtleClientService * \param search_result_data result data. Memory ownership is owned by the turtle router. So do not delete! * \param search_result_data length of result data */ - virtual void receiveSearchResult(unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/) + virtual void receiveSearchResult(TurtleSearchRequestId /* request_id */,unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/) { std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; }