added structures for generic turtle search and access functions in Gxs objects

libresapi/src/api/FileSearchHandler.cpp

@@ -190,9 +190,9 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp)
         // i have no idea what the reasons for two different search modes are
         // rs-gui does it, so do we
         if(words.size() == 1)
-            search_id = mTurtle->turtleSearch(words.front());
+            search_id = rsFiles->turtleSearch(words.front());
         else
-            search_id = mTurtle->turtleSearch(lin_exp);
+            search_id = rsFiles->turtleSearch(lin_exp);
     }
 
     std::list<FileDetail> results;

libretroshare/src/ft/ftserver.cc

@@ -1317,7 +1317,7 @@ bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHas
 
 // Decrypts the given item using aead-chacha20-poly1305
 
-bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
+bool ftServer::decryptItem(const RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
 {
 #ifndef USE_NEW_METHOD
 	unsigned char *data = NULL ;
@@ -1453,9 +1453,34 @@ bool ftServer::findRealHash(const RsFileHash& hash, RsFileHash& real_hash)
 		return false ;
 }
 
+TurtleSearchRequestId ftServer::turtleSearch(const std::string& string_to_match)
+{
+    return mTurtleRouter->turtleSearch(string_to_match) ;
+}
+TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::LinearizedExpression& expr)
+{
+    return mTurtleRouter->turtleSearch(expr) ;
+}
+
+#warning we should do this here, but for now it's done by turtle router.
+//   // Dont delete the item. The client (p3turtle) is doing it after calling this.
+//   //
+//   void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item)
+//   {
+//       RsTurtleFTSearchResultItem *ft_sr = dynamic_cast<RsTurtleFTSearchResultItem*>(item) ;
+//
+//       if(ft_sr == NULL)
+//       {
+//   		FTSERVER_ERROR() << "(EE) ftServer::receiveSearchResult(): item cannot be cast to a RsTurtleFTSearchResultItem" << std::endl;
+//           return ;
+//       }
+//
+//   	RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ;
+//   }
+
 // Dont delete the item. The client (p3turtle) is doing it after calling this.
 //
-void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
+void ftServer::receiveTurtleData(const RsTurtleGenericTunnelItem *i,
                                  const RsFileHash& hash,
                                  const RsPeerId& virtual_peer_id,
                                  RsTurtleGenericTunnelItem::Direction direction)
@@ -1475,7 +1500,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 		}
 
 		RsTurtleGenericTunnelItem *decrypted_item ;
-		if(!decryptItem(dynamic_cast<RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
+		if(!decryptItem(dynamic_cast<const RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
 		{
 			FTSERVER_ERROR() << "(EE) decryption error." << std::endl;
 			return ;
@@ -1491,7 +1516,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 	{
 	case RS_TURTLE_SUBTYPE_FILE_REQUEST:
 	{
-		RsTurtleFileRequestItem *item = dynamic_cast<RsTurtleFileRequestItem *>(i) ;
+		const RsTurtleFileRequestItem *item = dynamic_cast<const RsTurtleFileRequestItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1504,7 +1529,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_FILE_DATA :
 	{
-		RsTurtleFileDataItem *item = dynamic_cast<RsTurtleFileDataItem *>(i) ;
+		const RsTurtleFileDataItem *item = dynamic_cast<const RsTurtleFileDataItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1512,7 +1537,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 #endif
 			getMultiplexer()->recvData(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size,item->chunk_data) ;
 
-			item->chunk_data = NULL ;	// this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
+			const_cast<RsTurtleFileDataItem*>(item)->chunk_data = NULL ;	// this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
 			// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
 		}
 	}
@@ -1520,7 +1545,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_FILE_MAP :
 	{
-		RsTurtleFileMapItem *item = dynamic_cast<RsTurtleFileMapItem *>(i) ;
+		const RsTurtleFileMapItem *item = dynamic_cast<const RsTurtleFileMapItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1543,7 +1568,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_CHUNK_CRC :
 	{
-		RsTurtleChunkCrcItem *item = dynamic_cast<RsTurtleChunkCrcItem *>(i) ;
+		const RsTurtleChunkCrcItem *item = dynamic_cast<const RsTurtleChunkCrcItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1556,7 +1581,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST:
 	{
-		RsTurtleChunkCrcRequestItem *item = dynamic_cast<RsTurtleChunkCrcRequestItem *>(i) ;
+		const RsTurtleChunkCrcRequestItem *item = dynamic_cast<const RsTurtleChunkCrcRequestItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG

libretroshare/src/ft/ftserver.h

@@ -96,7 +96,8 @@ public:
     // Implements RsTurtleClientService
     //
     virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+	//virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO
     virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ;
 	virtual RsServiceSerializer *serializer() { return this ; }
 
@@ -143,6 +144,9 @@ public:
 	virtual void setFilePermDirectDL(uint32_t perm) ;
 	virtual uint32_t filePermDirectDL() ;
 
+	virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
+	virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+
     /***
          * Control of Downloads Priority.
          ***/
@@ -250,7 +254,7 @@ public:
     static void deriveEncryptionKey(const RsFileHash& hash, uint8_t *key);
 
     bool encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item);
-    bool decryptItem(RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
+    bool decryptItem(const RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
 
     /*************** Internal Transfer Fns *************************/
     virtual int tick();

libretroshare/src/grouter/p3grouter.cc

@@ -482,7 +482,7 @@ void p3GRouter::handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem *t
 #endif
 }
 
-void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
+void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
 {
 #ifdef GROUTER_DEBUG
     std::cerr << "p3GRouter::receiveTurtleData() " << std::endl;
@@ -496,7 +496,7 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
     //  - possibly packs multi-item blocks back together
     // 	- converts it into a grouter generic item (by deserialising it)
 
-    RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
+    const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
 
     if(item == NULL)
     {
@@ -510,7 +510,8 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
 
     // Items come out of the pipe in order. We need to recover all chunks before we de-serialise the content and have it handled by handleIncoming()
 
-    RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&item->data_size) ;
+    uint32_t size = item->data_size ;
+    RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&size);
 
 if(itm == NULL)
 {

libretroshare/src/grouter/p3grouter.h

@@ -209,7 +209,7 @@ protected:
     //===================================================//
 
     virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
     virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ;
     virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) ;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -3384,3 +3384,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list<RsNxsMsg*>& msgs, Gx
 	}
 }
 
+void RsGenExchange::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+    mNetService->turtleGroupRequest(group_id) ;
+}
+void RsGenExchange::turtleSearchRequest(const std::string& match_string)
+{
+    mNetService->turtleSearchRequest(match_string) ;
+}

libretroshare/src/gxs/rsgenexchange.h

@@ -287,6 +287,15 @@ public:
 	 */
 	bool getGroupStatistic(const uint32_t& token, GxsGroupStatistic& stats);
 
+    /*!
+     * \brief turtleGroupRequest
+     * 			Issues a browadcast group request using the turtle router generic search system. The request is obviously asynchroneous and will be
+     * 			handled in RsGenExchange when received.
+     * \param group_id
+     */
+    void turtleGroupRequest(const RsGxsGroupId& group_id);
+    void turtleSearchRequest(const std::string& match_string);
+
 protected:
 
 	bool messagePublicationTest(const RsGxsMsgMetaData&) ;

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -5102,3 +5102,12 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid)
 
     return true;
 }
+
+void RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+    mGxsNetTunnel->turtleGroupRequest(group_id) ;
+}
+void RsGxsNetService::turtleSearchRequest(const std::string& match_string)
+{
+    mGxsNetTunnel->turtleSearchRequest(match_string) ;
+}

libretroshare/src/gxs/rsgxsnetservice.h

@@ -122,6 +122,8 @@ public:
 	virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; }
 	virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; }
 
+    virtual void turtleGroupRequest(const RsGxsGroupId& group_id);
+    virtual void turtleSearchRequest(const std::string& match_string);
     /*!
      * pauses synchronisation of subscribed groups and request for group id
      * from peers

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -389,7 +389,7 @@ bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsP
 	return mHandledHashes.find(hash) != mHandledHashes.end();
 }
 
-void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& turtle_virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction)
+void RsGxsNetTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *item, const RsFileHash& hash, const RsPeerId& turtle_virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction)
 {
 	RS_STACK_MUTEX(mGxsNetTunnelMtx);
 
@@ -424,7 +424,7 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co
 
 	generateEncryptionKey(group_id,turtle_virtual_peer_id,encryption_master_key);
 
-	if(!p3turtle::decryptItem(static_cast<RsTurtleGenericDataItem*>(item),encryption_master_key,data,data_size))
+	if(!p3turtle::decryptItem(static_cast<const RsTurtleGenericDataItem*>(item),encryption_master_key,data,data_size))
 	{
 		GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl;
 
@@ -852,3 +852,23 @@ RsSerialiser *RsGxsNetTunnelService::setupSerialiser()
 
 	return ser ;
 }
+
+bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len)
+{
+    std::cerr << __PRETTY_FUNCTION__ << ": received a request. Code needed to handle it" << std::endl;
+    return false ;
+}
+void RsGxsNetTunnelService::receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len)
+{
+    std::cerr << __PRETTY_FUNCTION__ << ": received a search result. Code needed to handle it" << std::endl;
+}
+
+void RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+    std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle group request not implemented yet" << std::endl;
+}
+void RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string)
+{
+    std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle search request not implemented yet" << std::endl;
+}
+

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -212,13 +212,23 @@ public:
 	   */
 	  void dump() const;
 
-	  // other methods are still missing.
+      /*!
-	  //  - derived from p3Config, to load/save data
+       * \brief connectToTurtleRouter
-	  //  - method to respond to tunnel requests, probably using RsGxsNetService
+       * 			Should be called after allocating a RsGxsNetTunnelService
-	  //  - method to encrypt/decrypt data and send/receive to/from turtle.
+       * \param tr turtle router object
-
+       */
 	  virtual void connectToTurtleRouter(p3turtle *tr) ;
 
+      void turtleGroupRequest(const RsGxsGroupId& group_id) ;
+      void turtleSearchRequest(const std::string& match_string) ;
+
+      /*!
+       * \brief receiveSearchRequest
+       * 			See RsTurtleClientService::@
+       */
+	  virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len);
+	  virtual void receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) ;
+
 	  // Overloaded from RsTickingThread
 
 	  void data_tick() ;
@@ -233,7 +243,7 @@ protected:
 	  // interaction with turtle router
 
 	  virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
-	  virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+	  virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
 	  void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
 	  void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
 	  const Bias20Bytes& locked_randomBias() ;

libretroshare/src/gxs/rsnxs.h

@@ -81,6 +81,8 @@ public:
     virtual uint32_t getDefaultSyncAge() =0;
     virtual uint32_t getDefaultKeepAge() =0;
 
+    virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0;
+    virtual void turtleSearchRequest(const std::string& match_string)=0;
     /*!
      * Initiates a search through the network
      * This returns messages which contains the search terms set in RsGxsSearch

libretroshare/src/gxstunnel/p3gxstunnel.cc

@@ -685,7 +685,7 @@ void p3GxsTunnelService::removeVirtualPeer(const TurtleFileHash& hash,const Turt
     }
 }
 
-void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,const RsFileHash& hash, const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction)
+void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash& hash, const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction)
 {
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "GxsTunnelService::receiveTurtleData(): Received turtle data. " << std::endl;
@@ -697,7 +697,7 @@ void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,cons
     (void) direction;
 #endif
 
-    RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
+    const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
 
     if(item == NULL)
     {

libretroshare/src/gxstunnel/p3gxstunnel.h

@@ -211,7 +211,7 @@ private:
     // Overloaded from RsTurtleClientService
 
     virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
     void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
     void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
     

libretroshare/src/pqi/p3notify.cc

@@ -231,7 +231,8 @@ void p3Notify::notifyChatLobbyTimeShift     (int                time_shift)
 void p3Notify::notifyCustomState            (const std::string& peer_id   , const std::string&               status_string )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState       (peer_id,status_string) ; }
 void p3Notify::notifyHashingInfo            (uint32_t           type      , const std::string&               fileinfo      )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo       (type,fileinfo) ; } 
 void p3Notify::notifyTurtleSearchResult     (uint32_t           search_id , const std::list<TurtleFileInfo>& files         )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; } 
-void p3Notify::notifyTurtleSearchResult     (uint32_t           search_id , const std::list<TurtleGxsInfo>&  groups        )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; }
+#warning MISSING CODE HERE
+//void p3Notify::notifyTurtleSearchResult     (uint32_t           search_id , const std::list<TurtleGxsInfo>&  groups        )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; }
 void p3Notify::notifyPeerHasNewAvatar       (std::string        peer_id   )                                                     { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; }
 void p3Notify::notifyOwnAvatarChanged       ()                                                                                  { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnAvatarChanged() ; } 
 void p3Notify::notifyOwnStatusMessageChanged()                                                                                  { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnStatusMessageChanged() ; } 

libretroshare/src/pqi/p3notify.h

@@ -106,7 +106,8 @@ class p3Notify: public RsNotify
 		void notifyCustomState                (const std::string& /* peer_id   */, const std::string&               /* status_string */) ;
 		void notifyHashingInfo                (uint32_t           /* type      */, const std::string&               /* fileinfo      */) ;
 		void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleFileInfo>& /* files         */) ;
-		void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) ;
+#warning MISSING CODE HERE
+//		void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) ;
 		void notifyPeerHasNewAvatar           (std::string        /* peer_id   */) ;
 		void notifyOwnAvatarChanged           () ;
 		void notifyOwnStatusMessageChanged    () ;

libretroshare/src/retroshare/rsfiles.h

@@ -32,6 +32,7 @@
 #include <string>
 
 #include "rstypes.h"
+#include "rsturtle.h"
 
 class RsFiles;
 extern RsFiles  *rsFiles;
@@ -213,6 +214,8 @@ class RsFiles
 		virtual uint32_t getMaxUploadSlotsPerFriend()=0;
 		virtual void setFilePermDirectDL(uint32_t perm)=0;
 		virtual uint32_t filePermDirectDL()=0;
+		virtual TurtleRequestId turtleSearch(const std::string& string_to_match) =0;
+        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) =0;
 
 		/***
 		 * Control of Downloads Priority.

libretroshare/src/retroshare/rsgxschannels.h

@@ -96,6 +96,9 @@ virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &p
 //virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0;
 //virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0;
 
+    virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0;
+    virtual void turtleSearchRequest(const std::string& match_string)=0;
+
         //////////////////////////////////////////////////////////////////////////////
 virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
 

libretroshare/src/retroshare/rsnotify.h

@@ -228,7 +228,8 @@ class NotifyClient
 		virtual void notifyCustomState                (const std::string& /* peer_id   */, const std::string&               /* status_string */) {}
 		virtual void notifyHashingInfo                (uint32_t           /* type      */, const std::string&               /* fileinfo      */) {}
 		virtual void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleFileInfo>& /* files         */) {}
-		virtual void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) {}
+#warning MISSING CODE HERE
+		// virtual void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) {}
 		virtual void notifyPeerHasNewAvatar           (std::string        /* peer_id   */) {}
 		virtual void notifyOwnAvatarChanged           () {}
 		virtual void notifyOwnStatusMessageChanged    () {}

libretroshare/src/retroshare/rsturtle.h

@@ -54,13 +54,7 @@ struct TurtleFileInfo
 	std::string name ;
 	uint64_t size ;
 };
-struct TurtleGxsInfo
+
-{
-	RsGxsGroupId group_id ;
-	uint16_t service_id ;
-	std::string name ;
-	//RsTlvBinaryData meta ;// is that actually needed? Not sure. Better if it's not.
-};
 struct TurtleTunnelRequestDisplayInfo
 {
 	uint32_t request_id ;     // Id of the request
@@ -118,8 +112,7 @@ class RsTurtle
 		// the request id, which will be further used by the gui to store results
 		// as they come back.
 		//
-		virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ;
+        virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) =0;
-        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ;
 
 		// Initiates tunnel handling for the given file hash.  tunnels.  Launches
 		// an exception if an error occurs during the initialization process. The

libretroshare/src/rsitems/rsgxsitems.cc

@@ -6,6 +6,8 @@
  */
 
 
+#include "serialiser/rstypeserializer.h"
+#include "serialiser/rsbaseserial.h"
 #include "rsgxsitems.h"
 #include "gxs/rsgxsdata.h"
 #include <iostream>
@@ -70,4 +72,58 @@ std::ostream &operator<<(std::ostream &out, const RsMsgMetaData &meta)
         return out;
 }
 
+template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i)
+{
+    uint32_t s = 0 ;
 
+    s += 2 ; // service_id
+    s += i.group_id.SIZE_IN_BYTES ;
+    s += GetTlvStringSize(i.name) ;
+
+    return s;
+}
+
+template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i)
+{
+    uint32_t saved_offset = offset ;
+    bool ok = true ;
+
+	ok &= getRawUInt16(data, size, &offset, &i.service_id); 			 // service_id
+	ok &= i.group_id.deserialise(data, size, offset);                    // group_id
+	ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
+
+    if(!ok)
+        offset = saved_offset ;
+
+    return ok;
+}
+
+template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i)
+{
+	uint32_t saved_offset = offset ;
+    bool ok = true ;
+
+	ok &= setRawUInt16(data, size, &offset, i.service_id); 					 // service_id
+	ok &= i.group_id.serialise(data, size, offset);					         // group_id
+	ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
+
+	if(!ok)
+		offset = saved_offset ;
+
+	return ok;
+}
+
+template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i)
+{
+    std::cerr << "  [GXS Info  ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl;
+}
+
+void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process(j,ctx,result,"result") ;
+}
+void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::TlvMemBlock_proxy prox(encrypted_nxs_group_data,encrypted_nxs_group_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"encrypted_nxs_data") ;
+}

libretroshare/src/rsitems/rsgxsitems.h

@@ -57,6 +57,82 @@ public:
     RsMsgMetaData meta;
 };
 
+// We should make these items templates or generic classes so that each GXS service will handle them on its own.
+
+static const uint8_t  RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM = 0x20 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM     = 0x21 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM    = 0x22 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM       = 0x23 ;
+
+class RsGxsTurtleSubStringSearchItem: public RsItem
+{
+	public:
+		RsGxsTurtleSubStringSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM) {}
+        virtual ~RsGxsTurtleSubStringSearchItem() {}
+
+		std::string match_string ;	// string to match
+
+		std::string GetKeywords() { return match_string; }
+        void clear() { match_string.clear() ; }
+
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+class RsGxsTurtleGroupSearchItem: public RsItem
+{
+	public:
+		RsGxsTurtleGroupSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM) {}
+        virtual ~RsGxsTurtleGroupSearchItem() {}
+
+        uint16_t service_id ;		// searvice to search
+		Sha1CheckSum hashed_group_id ;	// the group ID is hashed in order to keep it private.
+
+		std::string GetKeywords() { return std::string("Group request for [hashed] ")+hashed_group_id.toStdString() ; }
+        void clear() { hashed_group_id.clear() ; }
+
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+struct TurtleGxsInfo
+{
+	uint16_t     service_id ;
+	RsGxsGroupId group_id ;
+	RsGxsId      author;
+	std::string  name ;
+	std::string  description ;
+	time_t       last_post ;
+	uint32_t     number_of_posts ;
+};
+
+class RsTurtleGxsSearchResultGroupSummaryItem: public RsItem
+{
+	public:
+        RsTurtleGxsSearchResultGroupSummaryItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM){}
+        virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {}
+
+		std::list<TurtleGxsInfo> result ;
+
+        void clear() { result.clear() ; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+class RsTurtleGxsSearchResultGroupDataItem: public RsItem
+{
+	public:
+        RsTurtleGxsSearchResultGroupDataItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM){}
+        virtual ~RsTurtleGxsSearchResultGroupDataItem() {}
+
+        unsigned char *encrypted_nxs_group_data;	// data is encrypted with group ID. Only the requester, or anyone who already know the group id can decrypt.
+        uint32_t encrypted_nxs_group_data_len ;
+
+        void clear() { free(encrypted_nxs_group_data); encrypted_nxs_group_data=NULL; encrypted_nxs_group_data_len=0; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
 
 
 

libretroshare/src/services/p3gxschannels.cc

@@ -1695,3 +1695,10 @@ void p3GxsChannels::handle_event(uint32_t event_type, const std::string &elabel)
 	}
 }
 
+void p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+}
+void p3GxsChannels::turtleSearchRequest(const std::string& match_string)
+{
+}
+

libretroshare/src/services/p3gxschannels.h

@@ -72,6 +72,9 @@ virtual void service_tick();
 	virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList); // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
 	virtual bool loadList(std::list<RsItem *>& loadList);               // @see p3Config::loadList(std::list<RsItem *>&)
 
+    virtual void turtleGroupRequest(const RsGxsGroupId& group_id);
+    virtual void turtleSearchRequest(const std::string& match_string);
+
 	// Overloaded to cache new groups.
 virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
 

libretroshare/src/turtle/p3turtle.cc

@@ -156,10 +156,9 @@ void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
 	names.clear();
 
 	names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST	] = "Filename substring search request";
-	names[RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST	    ] = "GXS search request";
+	names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST  ] = "Generic search request";
 	names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT		] = "File search result";
-	names[RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY		] = "GXS group summary";
+	names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT   ] = "Generic search result";
-	names[RS_TURTLE_SUBTYPE_GXS_GROUP_DATA   	 	] = "GXS group data";
 	names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL    			] = "Tunnel request";
 	names[RS_TURTLE_SUBTYPE_TUNNEL_OK      			] = "Tunnel response";
 	names[RS_TURTLE_SUBTYPE_FILE_REQUEST   			] = "Data request";
@@ -840,11 +839,11 @@ int p3turtle::handleIncoming()
 				switch(item->PacketSubType())
 				{
 					case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST:
+					case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST:
 					case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast<RsTurtleSearchRequestItem *>(item)) ;
 																				 break ;
 
-					case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA :
+					case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT :
-					case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY :
 					case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
 																		break ;
 
@@ -1001,11 +1000,15 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 }
 
 void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
+{
+    std::list<std::pair<RsTurtleSearchResultItem*,RsTurtleClientService*> > results_to_notify_off_mutex ;
+
 	{
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 		// Find who actually sent the corresponding request.
 		//
 		std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
+
 #ifdef P3TURTLE_DEBUG
 		std::cerr << "Received search result:" << std::endl ;
 		item->print(std::cerr,0) ;
@@ -1024,7 +1027,8 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 		if(it->second.origin == _own_id)
 		{
 			it->second.result_count += item->count() ;
-		returnSearchResult(item) ;		// Yes, so send upward.
+
+            results_to_notify_off_mutex.push_back(std::make_pair(item,it->second.client)) ;
 		}
 		else
 		{									// Nope, so forward it back.
@@ -1063,6 +1067,28 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 		}
 	}
 
+    // now we notify clients off-mutex.
+
+    for(auto it(results_to_notify_off_mutex.begin());it!=results_to_notify_off_mutex.end();++it)
+    {
+        // Hack to use the old search result handling in ftServer. Normally ftServer should use the new method with serialized result.
+
+#warning make sure memory is correctly deleted here
+        RsTurtleFTSearchResultItem *ftsr = dynamic_cast<RsTurtleFTSearchResultItem*>(it->first) ;
+
+        if(ftsr!=NULL)
+        {
+			RsServer::notify()->notifyTurtleSearchResult(ftsr->request_id,ftsr->result) ;
+            continue ;
+        }
+
+        RsTurtleGenericSearchResultItem *gnsr = dynamic_cast<RsTurtleGenericSearchResultItem*>(it->first) ;
+
+        if(gnsr!=NULL)
+			(*it).second->receiveSearchResult(gnsr->result_data,gnsr->result_data_len) ;
+    }
+}
+
 // -----------------------------------------------------------------------------------//
 // ---------------------------------  File Transfer. -------------------------------- //
 // -----------------------------------------------------------------------------------//
@@ -1794,17 +1820,11 @@ void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &
 	}
 }
 
-void RsTurtleGxsSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list<RsTurtleSearchResultItem*>& result) const
+void RsTurtleGenericSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list<RsTurtleSearchResultItem*>& result) const
 {
     std::cerr << "(EE) p3turtle: Missing code to perform actual GXS search" << std::endl;
 }
 
-void RsTurtleGxsGroupRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list<RsTurtleSearchResultItem*>& result) const
-{
-    std::cerr << "(EE) p3turtle: Missing code to perform actual retrieval of GXS group" << std::endl;
-}
-
-
 void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
 {
 	/* call to core */
@@ -1939,6 +1959,34 @@ TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpr
 	return id ;
 }
 
+TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service)
+{
+	// generate a new search id.
+
+	TurtleRequestId id = generateRandomRequestId() ;
+
+	// Form a request packet that simulates a request from us.
+	//
+	RsTurtleGenericSearchRequestItem *item = new RsTurtleGenericSearchRequestItem ;
+
+#ifdef P3TURTLE_DEBUG
+	std::cerr << "performing search. OwnId = " << _own_id << std::endl ;
+#endif
+
+	item->PeerId(_own_id) ;
+    item->service_id = client_service->serviceId();
+	item->search_data = search_bin_data ;
+	item->search_data_len = search_bin_data_len ;
+	item->request_id = id ;
+	item->depth = 0 ;
+
+	// send it
+
+	handleSearchRequest(item) ;
+
+	return id ;
+}
+
 void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *client_service,bool allow_multi_tunnels)
 {
 	{
@@ -1979,36 +2027,26 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie
 	IndicateConfigChanged() ;	// initiates saving of handled hashes.
 }
 
-void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item)
-{
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  Returning result for search request " << HEX_PRINT(item->request_id) << " upwards." << std::endl ;
 #endif
 
-    RsTurtleFTSearchResultItem *ft_sr = dynamic_cast<RsTurtleFTSearchResultItem*>(item) ;
 
-    if(ft_sr != NULL)
+//    RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast<RsTurtleGxsSearchResultGroupSummaryItem*>(item) ;
-    {
+//
-		RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ;
+//    if(gxs_sr_gs != NULL)
-        return ;
+//    {
-    }
+//		RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ;
-
+//        return ;
-    RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast<RsTurtleGxsSearchResultGroupSummaryItem*>(item) ;
+//    }
-
+//    RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast<RsTurtleGxsSearchResultGroupDataItem*>(item) ;
-    if(gxs_sr_gs != NULL)
+//
-    {
+//    if(gxs_sr_gd != NULL)
-		RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ;
+//    {
-        return ;
+//#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA.
-    }
+//		//RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ;
-    RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast<RsTurtleGxsSearchResultGroupDataItem*>(item) ;
+//        return ;
-
+//    }
-    if(gxs_sr_gd != NULL)
-    {
-#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA.
-		//RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ;
-        return ;
-    }
-}
 
 /// Warning: this function should never be called while the turtle mutex is locked.
 /// Otherwize this is a possible source of cross-lock with the File mutex.

libretroshare/src/turtle/p3turtle.h

@@ -173,6 +173,7 @@ class TurtleSearchRequestInfo
 		int depth ;                   // depth of the request. Used to optimize tunnel length.
 		uint32_t    result_count;     // responses to this request. Useful to avoid spamming tunnel responses.
 		std::string keywords;
+        RsTurtleClientService *client;// client who issues the request. This is null if the request does not have a local origin.
 };
 class TurtleTunnelRequestInfo
 {
@@ -244,11 +245,16 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		// the request id, which will be further used by the gui to store results
 		// as they come back.
 		//
-		// Eventually, search requests should be handled by client services. We will therefore
+        // The first two methods are old style search requests for FT, while the 3rd one is using a generic search data type, that is only to
-		// remove the specific file search packets from the turtle router.
+        // be deserialized by the service. The memory ownership is kept by the calling function. Similarly, the search response will be a
+        // generic data type that is to be deserialized by the client service.
         //
-		virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
+        // Eventually, search requests will use the generic system
-        virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+        // even for FT. We need to keep the old method for a while for backward compatibility.
+        //
+        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+        virtual TurtleRequestId turtleSearch(const std::string& string_to_match) ;
+        virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) ;
 
 		// Initiates tunnel handling for the given file hash.  tunnels.  Launches
 		// an exception if an error occurs during the initialization process. The
@@ -393,9 +399,6 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		/// Performs a search calling local cache and search structure.
 		void performLocalSearch(const std::string& match_string,std::list<TurtleFileInfo>& result) ;
 
-		/// Returns a search result upwards (possibly to the gui)
-		void returnSearchResult(RsTurtleSearchResultItem *item) ;
-
 		/// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer.
 		virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service);
 

libretroshare/src/turtle/rsturtleitem.cc

@@ -32,11 +32,8 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
 	case RS_TURTLE_SUBTYPE_OPEN_TUNNEL  			:	return new RsTurtleOpenTunnelItem();
 	case RS_TURTLE_SUBTYPE_TUNNEL_OK    			:	return new RsTurtleTunnelOkItem();
 	case RS_TURTLE_SUBTYPE_GENERIC_DATA 			:	return new RsTurtleGenericDataItem();
-
+	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST	:	return new RsTurtleGenericSearchRequestItem();
-	case RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST	    :	return new RsTurtleGxsSearchRequestItem();
+	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT	:	return new RsTurtleGenericSearchResultItem();
-	case RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST	    :	return new RsTurtleGxsGroupRequestItem();
-	case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY		:	return new RsTurtleGxsSearchResultGroupSummaryItem();
-	case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA   	 	:	return new RsTurtleGxsSearchResultGroupDataItem();
 
 	default:
 		break ;
@@ -65,21 +62,24 @@ void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::Serial
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
     RsTypeSerializer::serial_process(j,ctx,expr,"expr") ;
 }
-void RsTurtleGxsSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
-    RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ;
-}
-void RsTurtleGxsGroupRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
-{
-    RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
-    RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
-    RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
-    RsTypeSerializer::serial_process(j,ctx,hashed_group_id,"hashed_group_id") ;
-}
 
+    RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
+}
+RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const
+{
+    RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ;
+
+    sr->search_data = (unsigned char*)rs_malloc(search_data_len) ;
+    memcpy(sr->search_data,search_data,search_data_len) ;
+    sr->search_data_len = search_data_len ;
+    return sr ;
+}
 template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r)
 {
     uint32_t s = 0 ;
@@ -164,18 +164,24 @@ void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJo
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth     ,"depth") ;
     RsTypeSerializer::serial_process          (j,ctx,result    ,"result") ;
 }
-void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+void RsTurtleGenericSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth     ,"depth") ;
-    RsTypeSerializer::serial_process          (j,ctx,result    ,"result") ;
+
+    RsTypeSerializer::TlvMemBlock_proxy prox(result_data,result_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
 }
-void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+RsTurtleSearchResultItem *RsTurtleGenericSearchResultItem::duplicate() const
 {
-    RsTypeSerializer::serial_process<uint32_t> (j,ctx,request_id,"request_id") ;
+    RsTurtleGenericSearchResultItem *sr = new RsTurtleGenericSearchResultItem ;
-    RsTypeSerializer::serial_process<uint16_t> (j,ctx,depth     ,"depth") ;
+
-    RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,encrypted_nxs_group,"encrypted_nxs_group") ;
+    sr->result_data = (unsigned char*)rs_malloc(result_data_len) ;
+    memcpy(sr->result_data,result_data,result_data_len) ;
+    sr->result_data_len = result_data_len ;
+    return sr ;
 }
+
 template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i)
 {
     uint32_t s = 0 ;
@@ -222,52 +228,6 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleF
     std::cerr << "  [FileInfo  ] " << n << " size=" << i.size << " hash=" << i.hash << ", name=" << i.name << std::endl;
 }
 
-template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i)
-{
-    uint32_t s = 0 ;
-
-    s += 2 ; // service_id
-    s += i.group_id.SIZE_IN_BYTES ;
-    s += GetTlvStringSize(i.name) ;
-
-    return s;
-}
-
-template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i)
-{
-    uint32_t saved_offset = offset ;
-    bool ok = true ;
-
-	ok &= getRawUInt16(data, size, &offset, &i.service_id); 			 // service_id
-	ok &= i.group_id.deserialise(data, size, offset);                    // group_id
-	ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
-
-    if(!ok)
-        offset = saved_offset ;
-
-    return ok;
-}
-
-template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i)
-{
-	uint32_t saved_offset = offset ;
-    bool ok = true ;
-
-	ok &= setRawUInt16(data, size, &offset, i.service_id); 					 // service_id
-	ok &= i.group_id.serialise(data, size, offset);					         // group_id
-	ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
-
-	if(!ok)
-		offset = saved_offset ;
-
-	return ok;
-}
-
-template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i)
-{
-    std::cerr << "  [GXS Info  ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl;
-}
-
 void RsTurtleOpenTunnelItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process          (j,ctx,file_hash        ,"file_hash") ;

libretroshare/src/turtle/rsturtleitem.h

@@ -23,17 +23,15 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST   			= 0x07 ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA      			= 0x08 ;
 const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST   = 0x09 ;
 const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA     		= 0x0a ;
-const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST      = 0x0b ;
+const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST  = 0x0b ;
-const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST       = 0x0c ;
+const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT   = 0x0c ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP                = 0x10 ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST        = 0x11 ;
-const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC               = 0x14 ;
-const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST       = 0x15 ;
-const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY		= 0x16 ;
-const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_DATA   		= 0x17 ;
-
 // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC                = 0x12 ; // unused
 // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST        = 0x13 ;
+const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC               = 0x14 ;
+const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST       = 0x15 ;
+
 
 class TurtleSearchRequestInfo ;
 
@@ -106,6 +104,8 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem
         virtual ~RsTurtleFileSearchRequestItem() {}
 
 		virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list<RsTurtleSearchResultItem*>&) const ;	// abstracts the search method
+
+	protected:
 		virtual void search(std::list<TurtleFileInfo> &) const =0;
 };
 
@@ -151,44 +151,29 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };
 
-class RsTurtleGxsSearchRequestItem: public RsTurtleSearchRequestItem
+class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem
 {
 	public:
-		RsTurtleGxsSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST) {}
+		RsTurtleGenericSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {}
-        virtual ~RsTurtleGxsSearchRequestItem() {}
+        virtual ~RsTurtleGenericSearchRequestItem() { clear(); }
 
-		std::string match_string ;	// string to match
+        uint16_t service_id ;		// service to search
-        uint16_t service_id ;		// searvice to search
+        uint32_t search_data_len ;
+        unsigned char *search_data ;
 
-		std::string GetKeywords() { return match_string; }
+		std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); }
 
-		virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list<RsTurtleSearchResultItem*>&) const ;	// abstracts the search method
+		virtual RsTurtleSearchRequestItem *clone() const ;
-		virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsSearchRequestItem(*this) ; }
+		void clear() { free(search_data); search_data=NULL; search_data_len=0; }
-        void clear() { match_string.clear() ; }
 
+		virtual void performLocalSearch(TurtleSearchRequestInfo &req, std::list<RsTurtleSearchResultItem*>& result) const;
 	protected:
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+
+    private:
+        RsTurtleGenericSearchRequestItem(const RsTurtleGenericSearchRequestItem&): RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {}				// make the object non copi-able.
+        RsTurtleGenericSearchRequestItem& operator=(const RsTurtleGenericSearchRequestItem&) { return *this;}
 };
-
-class RsTurtleGxsGroupRequestItem: public RsTurtleSearchRequestItem
-{
-	public:
-		RsTurtleGxsGroupRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST) {}
-        virtual ~RsTurtleGxsGroupRequestItem() {}
-
-        uint16_t service_id ;		// searvice to search
-		Sha1CheckSum hashed_group_id ;	// the group ID is hashed in order to keep it private.
-
-		virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsGroupRequestItem(*this) ; }
-
-		virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list<RsTurtleSearchResultItem*>&) const ;	// abstracts the search method
-        void clear() { hashed_group_id.clear() ; }
-		std::string GetKeywords() { return hashed_group_id.toStdString(); }
-
-	protected:
-		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
-};
-
 class RsTurtleSearchResultItem: public RsTurtleItem
 {
 	public:
@@ -199,7 +184,6 @@ class RsTurtleSearchResultItem: public RsTurtleItem
 											// 	If the actual depth is 1, this field will be 1.
 											// 	If the actual depth is > 1, this field is a larger arbitrary integer.
 
-        virtual void clear() =0;
         virtual uint32_t count() const =0;
         virtual void pop() =0;
 
@@ -222,33 +206,20 @@ class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };
 
-class RsTurtleGxsSearchResultGroupSummaryItem: public RsTurtleSearchResultItem
+class RsTurtleGenericSearchResultItem: public RsTurtleSearchResultItem
 {
 	public:
-        RsTurtleGxsSearchResultGroupSummaryItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY){}
+        RsTurtleGenericSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT){}
-        virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {}
+        virtual ~RsTurtleGenericSearchResultItem() {}
 
-		std::list<TurtleGxsInfo> result ;
+        uint32_t count() const { return result_data_len/50 ; }	// This is a blind size estimate. We should probably use the actual size to limit search results.
+        virtual void pop() {}
 
-        void clear() { result.clear() ; }
+        unsigned char *result_data ;
-        uint32_t count() const { return result.size() ; }
+        uint32_t result_data_len ;
-        virtual void pop() { result.pop_back() ;}
-        virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupSummaryItem(*this) ; }
-	protected:
-		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
-};
-class RsTurtleGxsSearchResultGroupDataItem: public RsTurtleSearchResultItem
-{
-	public:
-        RsTurtleGxsSearchResultGroupDataItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_DATA){}
-        virtual ~RsTurtleGxsSearchResultGroupDataItem() {}
 
-        RsTlvBinaryData encrypted_nxs_group;	// data is encrypted with group ID.
+        virtual RsTurtleSearchResultItem *duplicate() const ;
-
+		void clear() { free(result_data); result_data=NULL; result_data_len=0; }
-        uint32_t count() const { return 1 ; }
-        virtual void pop() { clear(); }
-        void clear() { encrypted_nxs_group.TlvClear() ; }
-        virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupDataItem(*this) ; }
 	protected:
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };

libretroshare/src/turtle/turtleclientservice.h

@@ -42,6 +42,18 @@ class p3turtle ;
 class RsTurtleClientService
 {
 	public:
+    	/*!
+         * \brief serviceId
+         * 			Returns the ID of the client service. This is used to pass the ID to search requests, from the client services
+         * \return
+         * 			The service ID.
+         */
+
+    	virtual uint16_t serviceId() const
+		{
+			std::cerr << "!!!!!! Received request for service ID in turtle router client, but the client service is not handling it !!!!!!!" << std::endl ;
+            return 0 ;
+    	}
 
 		/*!
 		 * \brief handleTunnelRequest
@@ -52,7 +64,6 @@ class RsTurtleClientService
 		 */
 		virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; }
 		
-
 	    /*!
 		 * \brief receiveTurtleData
 		 *           This method is called by the turtle router to send data that comes out of a turtle tunnel, and should
@@ -74,11 +85,43 @@ class RsTurtleClientService
 		 *           By default (if not overloaded), the method will just free the data, as any subclass should do as well.
 		 *           Note: p3turtle stays owner of the item, so the client should not delete it!
 		 */
-        virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
+        virtual void receiveTurtleData(const RsTurtleGenericTunnelItem * /* item */,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
 		{ 
 			std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ; 
 		}
 
+    	/*!
+         * \brief receiveSearchRequest
+         * 			This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
+         * 			result contains the serialised generic result returned by the client.
+         *
+         * 			The turtle router keeps the memory ownership over search_request_data
+         *
+         * \param search_request_data      generic serialized search data
+         * \param search_request_data_len  length of the serialized search data
+         * \param search_result_data       generic serialized search result data
+         * \param search_result_data_len   length of the serialized search result data
+         *
+         * \return true if the search is successful.
+         */
+    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,uint32_t /*search_request_data_len*/,unsigned char *& /*search_result_data*/,uint32_t& /*search_result_data_len*/)
+		{
+			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+            return false;
+		}
+
+    	/*!
+         * \brief receiveSearchResult
+         * 			This method is called by the turtle router to notify the client of a search result. The result is serialized for the current class to read.
+         *
+         * \param search_result_data  result data. Memory ownership is owned by the turtle router. So do not delete!
+         * \param search_result_data  length of result data
+         */
+    	virtual void receiveSearchResult(unsigned char * /*search_result_data*/,uint32_t  /*search_result_data_len*/)
+		{
+			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+		}
+
 	    /*!
 		 * \brief serializer
 		 *           Method for creating specific items of the client service. The

retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp

@@ -792,7 +792,7 @@ void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression)
     RsRegularExpression::LinearizedExpression e ;
 	expression->linearize(e) ;
 
-	TurtleRequestId req_id = rsTurtle->turtleSearch(e) ;
+	TurtleRequestId req_id = rsFiles->turtleSearch(e) ;
 
 	// This will act before turtle results come to the interface, thanks to the signals scheduling policy.
 	initSearchResult(QString::fromStdString(e.GetStrings()),req_id, ui.FileTypeComboBox->currentIndex(), true) ;
@@ -858,9 +858,9 @@ void SearchDialog::searchKeywords(const QString& keywords)
 	if(ui._anonF2Fsearch_CB->isChecked())
 	{
 		if(n==1)
-			req_id = rsTurtle->turtleSearch(words.front()) ;
+			req_id = rsFiles->turtleSearch(words.front()) ;
 		else
-			req_id = rsTurtle->turtleSearch(lin_exp) ;
+			req_id = rsFiles->turtleSearch(lin_exp) ;
 	}
 	else
 		req_id = RSRandom::random_u32() ; // generate a random 32 bits request id

retroshare-gui/src/gui/notifyqt.h

@@ -23,7 +23,9 @@ class MessengerWindow;
 class ToasterItem;
 class ToasterNotify;
 class SignatureEventData ;
+
 struct TurtleFileInfo;
+struct TurtleGxsInfo;
 
 class NotifyQt: public QObject, public NotifyClient
 {