Merge branch 'master' into deep_search

Needed to make deep search and GXS search cooperate

libresapi/src/api/FileSearchHandler.cpp

@@ -190,9 +190,9 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp)
         // i have no idea what the reasons for two different search modes are
         // rs-gui does it, so do we
         if(words.size() == 1)
-            search_id = mTurtle->turtleSearch(words.front());
+            search_id = rsFiles->turtleSearch(words.front());
         else
-            search_id = mTurtle->turtleSearch(lin_exp);
+            search_id = rsFiles->turtleSearch(lin_exp);
     }
 
     std::list<FileDetail> results;

libretroshare/src/crypto/rscrypto.cpp

@@ -0,0 +1,212 @@
+/*******************************************************************************
+ * libretroshare/src/crypto: crypto.cc                                         *
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright 2018 by Cyril Soler <csoler@users.sourceforge.net>                *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "rscrypto.h"
+#include "util/rsrandom.h"
+
+//#define CRYPTO_DEBUG 1
+
+namespace librs {
+namespace crypto {
+
+#define RSCRYPTO_DEBUG() std::cerr << time(NULL) << " : RSCRYPTO : " << __FUNCTION__ << " : "
+#define RSCRYPTO_ERROR() std::cerr << "(EE) RSCRYPTO ERROR : "
+
+static const uint32_t ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE = 12 ;
+static const uint32_t ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE    = 16 ;
+static const uint32_t ENCRYPTED_MEMORY_HEADER_SIZE                =  4 ;
+static const uint32_t ENCRYPTED_MEMORY_EDATA_SIZE                 =  4 ;
+
+static const uint8_t  ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ;
+static const uint8_t  ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256   = 0x02 ;
+
+bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_len)
+{
+	uint8_t initialization_vector[ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE] ;
+
+	RSRandom::random_bytes(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ;
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "ftServer::Encrypting ft item." << std::endl;
+	RSCRYPTO_DEBUG() << "  random nonce    : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl;
+#endif
+
+	uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ;
+	uint32_t total_data_size = ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE + item_serialized_size + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE  ;
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "  clear part size : " << size(clear_item) << std::endl;
+	RSCRYPTO_DEBUG() << "  total item size : " << total_data_size << std::endl;
+#endif
+
+	encrypted_data = (unsigned char*)rs_malloc( total_data_size ) ;
+	encrypted_data_len  = total_data_size ;
+
+	if(encrypted_data == NULL)
+		return false ;
+
+	uint8_t *edata = (uint8_t*)encrypted_data;
+	uint32_t edata_size = item_serialized_size;
+	uint32_t offset = 0;
+
+	edata[0] = 0xae ;
+	edata[1] = 0xad ;
+	edata[2] = ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256 ;       // means AEAD_chacha20_sha256
+	edata[3] = 0x01 ;
+
+	offset += ENCRYPTED_MEMORY_HEADER_SIZE;
+	uint32_t aad_offset = offset ;
+	uint32_t aad_size = ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE ;
+
+	memcpy(&edata[offset], initialization_vector, ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ;
+	offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
+
+	edata[offset+0] = (edata_size >>  0) & 0xff ;
+	edata[offset+1] = (edata_size >>  8) & 0xff ;
+	edata[offset+2] = (edata_size >> 16) & 0xff ;
+	edata[offset+3] = (edata_size >> 24) & 0xff ;
+
+	offset += ENCRYPTED_MEMORY_EDATA_SIZE ;
+
+	memcpy(&edata[offset],clear_data,clear_data_size);
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "  clear item      : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl;
+#endif
+
+	uint32_t clear_item_offset = offset ;
+	offset += edata_size ;
+
+	uint32_t authentication_tag_offset = offset ;
+	assert(ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ;
+
+	//uint8_t encryption_key[32] ;
+	//deriveEncryptionKey(hash,encryption_key) ;
+
+	if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305)
+		librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ;
+	else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256)
+		librs::crypto::AEAD_chacha20_sha256  (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ;
+	else
+		return false ;
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "  encryption key  : " << RsUtil::BinToHex(encryption_key,32) << std::endl;
+	RSCRYPTO_DEBUG() << "  authen. tag     : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl;
+	RSCRYPTO_DEBUG() << "  final item      : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl;
+#endif
+
+	return true ;
+}
+
+// Decrypts the given item using aead-chacha20-poly1305
+bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_len,uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size)
+{
+	//uint8_t encryption_key[32] ;
+	//deriveEncryptionKey(hash,encryption_key) ;
+
+	uint8_t *edata = (uint8_t*)encrypted_data;
+	uint32_t offset = 0;
+
+	if(encrypted_data_len < ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE) return false ;
+
+	if(edata[0] != 0xae) return false ;
+	if(edata[1] != 0xad) return false ;
+	if(edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256) return false ;
+	if(edata[3] != 0x01) return false ;
+
+	offset += ENCRYPTED_MEMORY_HEADER_SIZE ;
+	uint32_t aad_offset = offset ;
+	uint32_t aad_size = ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
+
+	uint8_t *initialization_vector = &edata[offset] ;
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "ftServer::decrypting ft item." << std::endl;
+	RSCRYPTO_DEBUG() << "  item data       : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_data_len) << "(...)" << std::endl;
+	RSCRYPTO_DEBUG() << "  hash            : " << hash << std::endl;
+	RSCRYPTO_DEBUG() << "  encryption key  : " << RsUtil::BinToHex(encryption_key,32) << std::endl;
+	RSCRYPTO_DEBUG() << "  random nonce    : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl;
+#endif
+
+	offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
+
+	uint32_t edata_size = 0 ;
+	edata_size += ((uint32_t)edata[offset+0]) <<  0 ;
+	edata_size += ((uint32_t)edata[offset+1]) <<  8 ;
+	edata_size += ((uint32_t)edata[offset+2]) << 16 ;
+	edata_size += ((uint32_t)edata[offset+3]) << 24 ;
+
+	if(edata_size + ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE != encrypted_data_len)
+	{
+		RSCRYPTO_ERROR() << "  ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_data_len - (ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE ) << std::endl;
+		return false ;
+	}
+
+	offset += ENCRYPTED_MEMORY_EDATA_SIZE ;
+	uint32_t clear_item_offset = offset ;
+
+	uint32_t authentication_tag_offset = offset + edata_size ;
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "  authen. tag     : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl;
+#endif
+
+	bool result ;
+
+	if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305)
+		result = librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ;
+	else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256)
+		result = librs::crypto::AEAD_chacha20_sha256  (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ;
+	else
+		return false ;
+
+#ifdef CRYPTO_DEBUG
+	RSCRYPTO_DEBUG() << "  authen. result  : " << result << std::endl;
+	RSCRYPTO_DEBUG() << "  decrypted daya  : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl;
+#endif
+
+	if(!result)
+	{
+		RSCRYPTO_ERROR() << "(EE) decryption/authentication went wrong." << std::endl;
+		return false ;
+	}
+
+	decrypted_data_size = edata_size ;
+	decrypted_data = (unsigned char*)rs_malloc(edata_size) ;
+
+	if(decrypted_data == NULL)
+	{
+		std::cerr << "Failed to allocate memory for decrypted data chunk of size " << edata_size << std::endl;
+		return false ;
+	}
+	memcpy(decrypted_data,&edata[clear_item_offset],edata_size) ;
+
+	return true ;
+}
+
+}
+}
+

libretroshare/src/crypto/rscrypto.h

@@ -0,0 +1,59 @@
+
+/*******************************************************************************
+ * libretroshare/src/crypto: crypto.h                                          *
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright 2018 by Cyril Soler <csoler@users.sourceforge.net>                *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "crypto/chacha20.h"
+
+namespace librs
+{
+namespace crypto
+{
+/*!
+ * \brief encryptAuthenticateData
+ 			Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key
+            based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
+ * \param clear_data				input data to encrypt
+ * \param clear_data_size			length of input data
+ * \param encryption_master_key		encryption master key of length 32 bytes.
+ * \param encrypted_data			encrypted data, allocated using malloc
+ * \param encrypted_data_len		length of encrypted data
+ * \return
+ * 			true if everything went well.
+ */
+bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_size);
+
+/*!
+ * \brief decryptAuthenticateData
+ 			Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key
+            based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
+ * \param encrypted_data			input encrypted data
+ * \param encrypted_data_size		input encrypted data length
+ * \param encryption_master_key		encryption master key of length 32 bytes.
+ * \param decrypted_data			decrypted data, allocated using malloc.
+ * \param decrypted_data_size		length of allocated decrypted data.
+ * \return
+ * 			true if decryption + authentication are ok.
+ */
+bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_size, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size);
+}
+}
+

libretroshare/src/ft/ftserver.cc

@@ -1234,6 +1234,21 @@ static const uint8_t  ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256   = 0x02 ;
 
 bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item)
 {
+#ifndef USE_NEW_METHOD
+    uint32_t item_serialized_size = size(clear_item) ;
+
+	RsTemporaryMemory data(item_serialized_size) ;
+
+	if(data == NULL)
+		return false ;
+
+    serialise(clear_item, data, &item_serialized_size);
+
+	uint8_t encryption_key[32] ;
+	deriveEncryptionKey(hash,encryption_key) ;
+
+	return p3turtle::encryptData(data,item_serialized_size,encryption_key,encrypted_item) ;
+#else
 	uint8_t initialization_vector[ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE] ;
 
 	RSRandom::random_bytes(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ;
@@ -1312,12 +1327,34 @@ bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHas
 #endif
 
 	return true ;
+#endif
 }
 
 // Decrypts the given item using aead-chacha20-poly1305
 
-bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
+bool ftServer::decryptItem(const RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
 {
+#ifndef USE_NEW_METHOD
+	unsigned char *data = NULL ;
+	uint32_t data_size = 0 ;
+
+	uint8_t encryption_key[32] ;
+	deriveEncryptionKey(hash,encryption_key) ;
+
+	if(!p3turtle::decryptItem(encrypted_item,encryption_key,data,data_size))
+	{
+		FTSERVER_ERROR() << "Cannot decrypt data!" << std::endl;
+
+		if(data)
+			free(data) ;
+		return false ;
+	}
+	decrypted_item = dynamic_cast<RsTurtleGenericTunnelItem*>(deserialise(data,&data_size)) ;
+	free(data);
+
+	return (decrypted_item != NULL);
+
+#else
 	uint8_t encryption_key[32] ;
 	deriveEncryptionKey(hash,encryption_key) ;
 
@@ -1393,6 +1430,7 @@ bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileH
 		return false ;
 
 	return true ;
+#endif
 }
 
 bool ftServer::encryptHash(const RsFileHash& hash, RsFileHash& hash_of_hash)
@@ -1430,9 +1468,34 @@ bool ftServer::findRealHash(const RsFileHash& hash, RsFileHash& real_hash)
 		return false ;
 }
 
+TurtleSearchRequestId ftServer::turtleSearch(const std::string& string_to_match)
+{
+    return mTurtleRouter->turtleSearch(string_to_match) ;
+}
+TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::LinearizedExpression& expr)
+{
+    return mTurtleRouter->turtleSearch(expr) ;
+}
+
+#warning we should do this here, but for now it is done by turtle router.
+//   // Dont delete the item. The client (p3turtle) is doing it after calling this.
+//   //
+//   void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item)
+//   {
+//       RsTurtleFTSearchResultItem *ft_sr = dynamic_cast<RsTurtleFTSearchResultItem*>(item) ;
+//
+//       if(ft_sr == NULL)
+//       {
+//   		FTSERVER_ERROR() << "(EE) ftServer::receiveSearchResult(): item cannot be cast to a RsTurtleFTSearchResultItem" << std::endl;
+//           return ;
+//       }
+//
+//   	RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ;
+//   }
+
 // Dont delete the item. The client (p3turtle) is doing it after calling this.
 //
-void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
+void ftServer::receiveTurtleData(const RsTurtleGenericTunnelItem *i,
                                  const RsFileHash& hash,
                                  const RsPeerId& virtual_peer_id,
                                  RsTurtleGenericTunnelItem::Direction direction)
@@ -1452,7 +1515,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 		}
 
 		RsTurtleGenericTunnelItem *decrypted_item ;
-		if(!decryptItem(dynamic_cast<RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
+		if(!decryptItem(dynamic_cast<const RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
 		{
 			FTSERVER_ERROR() << "(EE) decryption error." << std::endl;
 			return ;
@@ -1468,7 +1531,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 	{
 	case RS_TURTLE_SUBTYPE_FILE_REQUEST:
 	{
-		RsTurtleFileRequestItem *item = dynamic_cast<RsTurtleFileRequestItem *>(i) ;
+		const RsTurtleFileRequestItem *item = dynamic_cast<const RsTurtleFileRequestItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1481,7 +1544,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_FILE_DATA :
 	{
-		RsTurtleFileDataItem *item = dynamic_cast<RsTurtleFileDataItem *>(i) ;
+		const RsTurtleFileDataItem *item = dynamic_cast<const RsTurtleFileDataItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1489,7 +1552,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 #endif
 			getMultiplexer()->recvData(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size,item->chunk_data) ;
 
-			item->chunk_data = NULL ;	// this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
+			const_cast<RsTurtleFileDataItem*>(item)->chunk_data = NULL ;	// this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
 			// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
 		}
 	}
@@ -1497,7 +1560,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_FILE_MAP :
 	{
-		RsTurtleFileMapItem *item = dynamic_cast<RsTurtleFileMapItem *>(i) ;
+		const RsTurtleFileMapItem *item = dynamic_cast<const RsTurtleFileMapItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1520,7 +1583,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_CHUNK_CRC :
 	{
-		RsTurtleChunkCrcItem *item = dynamic_cast<RsTurtleChunkCrcItem *>(i) ;
+		const RsTurtleChunkCrcItem *item = dynamic_cast<const RsTurtleChunkCrcItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG
@@ -1533,7 +1596,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
 
 	case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST:
 	{
-		RsTurtleChunkCrcRequestItem *item = dynamic_cast<RsTurtleChunkCrcRequestItem *>(i) ;
+		const RsTurtleChunkCrcRequestItem *item = dynamic_cast<const RsTurtleChunkCrcRequestItem *>(i) ;
 		if (item)
 		{
 #ifdef SERVER_DEBUG

libretroshare/src/ft/ftserver.h

@@ -95,8 +95,11 @@ public:
 
     // Implements RsTurtleClientService
     //
+
+    uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
     virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+	//virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO
     virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ;
 	virtual RsServiceSerializer *serializer() { return this ; }
 
@@ -143,6 +146,9 @@ public:
 	virtual void setFilePermDirectDL(uint32_t perm) ;
 	virtual uint32_t filePermDirectDL() ;
 
+	virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
+	virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+
     /***
          * Control of Downloads Priority.
          ***/
@@ -250,7 +256,7 @@ public:
     static void deriveEncryptionKey(const RsFileHash& hash, uint8_t *key);
 
     bool encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item);
-    bool decryptItem(RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
+    bool decryptItem(const RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
 
     /*************** Internal Transfer Fns *************************/
     virtual int tick();

libretroshare/src/grouter/p3grouter.cc

@@ -482,7 +482,7 @@ void p3GRouter::handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem *t
 #endif
 }
 
-void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
+void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
 {
 #ifdef GROUTER_DEBUG
     std::cerr << "p3GRouter::receiveTurtleData() " << std::endl;
@@ -496,7 +496,7 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
     //  - possibly packs multi-item blocks back together
     // 	- converts it into a grouter generic item (by deserialising it)
 
-    RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
+    const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
 
     if(item == NULL)
     {
@@ -510,7 +510,8 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
 
     // Items come out of the pipe in order. We need to recover all chunks before we de-serialise the content and have it handled by handleIncoming()
 
-    RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&item->data_size) ;
+    uint32_t size = item->data_size ;
+    RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&size);
 
 if(itm == NULL)
 {

libretroshare/src/grouter/p3grouter.h

@@ -208,8 +208,9 @@ protected:
     //         Interaction with turtle router            //
     //===================================================//
 
+    uint16_t serviceId() const { return RS_SERVICE_TYPE_GROUTER; }
     virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
     virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ;
     virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) ;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -64,7 +64,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN        = 0x00000040; // admin key
 
 #define GXS_MASK "GXS_MASK_HACK"
 
-//#define GEN_EXCH_DEBUG	1
+#define GEN_EXCH_DEBUG	1
 
 static const uint32_t MSG_CLEANUP_PERIOD     = 60*59; // 59 minutes
 static const uint32_t INTEGRITY_CHECK_PERIOD = 60*2; // 31 minutes // TODO: Restore this line before merging deep_search
@@ -1111,6 +1111,8 @@ void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
         RsGxsNotify* n = *vit;
         RsGxsGroupChange* gc;
         RsGxsMsgChange* mc;
+        RsGxsDistantSearchResultChange *gt;
+
         if((mc = dynamic_cast<RsGxsMsgChange*>(n)) != NULL)
         {
             if (mc->metaChange())
@@ -1133,6 +1135,10 @@ void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
                 out.mGrps.splice(out.mGrps.end(), gc->mGrpIdList);
             }
         }
+        else if((gt = dynamic_cast<RsGxsDistantSearchResultChange*>(n)) != NULL)
+        {
+            out.mDistantSearchReqs.push_back(gt->mRequestId);
+        }
         else
             std::cerr << "(EE) Unknown changes type!!" << std::endl;
         
@@ -1568,7 +1574,7 @@ bool RsGenExchange::setAuthenPolicyFlag(const uint8_t &msgFlag, uint32_t& authen
     return true;
 }
 
-void RsGenExchange::notifyNewGroups(std::vector<RsNxsGrp *> &groups)
+void RsGenExchange::receiveNewGroups(std::vector<RsNxsGrp *> &groups)
 {
 	RS_STACK_MUTEX(mGenMtx) ;
 
@@ -1600,7 +1606,7 @@ void RsGenExchange::notifyNewGroups(std::vector<RsNxsGrp *> &groups)
 }
 
 
-void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *>& messages)
+void RsGenExchange::receiveNewMessages(std::vector<RsNxsMsg *>& messages)
 {
 	RS_STACK_MUTEX(mGenMtx) ;
 
@@ -1635,11 +1641,22 @@ void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *>& messages)
 		}
 	}
 }
+
+void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId)
+{
+	RS_STACK_MUTEX(mGenMtx);
+
+	RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId);
+	mNotifications.push_back(gc);
+
+    std::cerr << "RsGenExchange::receiveDistantSearchResults(): received result for request " << std::hex << id << std::dec << std::endl;
+}
+
 void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId)
 {
 	RS_STACK_MUTEX(mGenMtx);
 
-	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHKEY, true);
+	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY, true);
 	gc->mGrpIdList.push_back(grpId);
 	mNotifications.push_back(gc);
 }
@@ -2310,7 +2327,7 @@ void RsGenExchange::publishMsgs()
 
 	if(!msgChangeMap.empty())
 	{
-		RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISH, false);
+		RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISHED, false);
 		ch->msgChangeMap = msgChangeMap;
 		mNotifications.push_back(ch);
 	}
@@ -2447,7 +2464,7 @@ void RsGenExchange::processGroupDelete()
 
 	if(!grpDeleted.empty())
 	{
-		RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISH, false);
+		RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHED, false);
 		gc->mGrpIdList = grpDeleted;
 		mNotifications.push_back(gc);
 	}
@@ -2756,7 +2773,7 @@ void RsGenExchange::publishGrps()
 
 	    if(!grpChanged.empty())
 	    {
-		    RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true);
+		    RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
 		    gc->mGrpIdList = grpChanged;
 		    mNotifications.push_back(gc);
 #ifdef GEN_EXCH_DEBUG
@@ -3022,7 +3039,7 @@ void RsGenExchange::processRecvdMessages()
 #endif
 		    mDataStore->storeMessage(msgs_to_store);
 
-		    RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVE, false);
+		    RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
 		    c->msgChangeMap = msgIds;
 		    mNotifications.push_back(c);
 	    }
@@ -3155,7 +3172,7 @@ void RsGenExchange::processRecvdGroups()
 
 	if(!grpIds.empty())
 	{
-		RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, false);
+		RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
 		c->mGrpIdList = grpIds;
 		mNotifications.push_back(c);
 		mDataStore->storeGroup(grps_to_store);
@@ -3239,7 +3256,7 @@ void RsGenExchange::performUpdateValidation()
 	}
 	// notify the client
 
-	RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true);
+	RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
 
 	for(uint32_t i=0;i<mGroupUpdates.size();++i)
 		if(mGroupUpdates[i].newGrp != NULL)
@@ -3335,11 +3352,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list<RsNxsMsg*>& msgs, Gx
 		mDataStore->retrieveMsgIds(*it, msgIdReq[*it]);
 
 #ifdef GEN_EXCH_DEBUG
-		const std::vector<RsGxsMessageId>& vec(msgIdReq[*it]) ;
+		const std::set<RsGxsMessageId>& vec(msgIdReq[*it]) ;
 		std::cerr << "  retrieved " << vec.size() << " message ids for group " << *it << std::endl;
 
-		for(uint32_t i=0;i<vec.size();++i)
+		for(auto it2(vec.begin());it2!=vec.end();++it2)
-			std::cerr << "    " << vec[i] << std::endl;
+			std::cerr << "    " << *it2   << std::endl;
 #endif
 	}
 
@@ -3381,3 +3398,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list<RsNxsMsg*>& msgs, Gx
 	}
 }
 
+void RsGenExchange::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+    mNetService->turtleGroupRequest(group_id) ;
+}
+void RsGenExchange::turtleSearchRequest(const std::string& match_string)
+{
+    mNetService->turtleSearchRequest(match_string) ;
+}

libretroshare/src/gxs/rsgenexchange.h

@@ -121,9 +121,9 @@ public:
 
     virtual ~RsGenExchange();
 
-    // Convention that this is implemented here. 
+    // Convention that this is implemented here.
     // and passes to network service.
-    virtual RsServiceInfo getServiceInfo() = 0; 
+    virtual RsServiceInfo getServiceInfo() = 0;
 
     void setNetworkExchangeService(RsNetworkExchangeService *ns) ;
 
@@ -132,18 +132,24 @@ public:
     /*!
      * @param messages messages are deleted after function returns
      */
-    virtual void notifyNewMessages(std::vector<RsNxsMsg*>& messages);
+    virtual void receiveNewMessages(std::vector<RsNxsMsg*>& messages);
 
     /*!
      * @param groups groups are deleted after function returns
      */
-    virtual void notifyNewGroups(std::vector<RsNxsGrp*>& groups);
+    virtual void receiveNewGroups(std::vector<RsNxsGrp*>& groups);
 
     /*!
      * @param grpId group id
      */
     virtual void notifyReceivePublishKey(const RsGxsGroupId &grpId);
 
+    /*!
+     * \brief notifyReceiveDistantSearchResults
+     * 				Should be called when new search results arrive.
+     * \param grpId
+     */
+	virtual void receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId);
     /*!
      * @param grpId group id
      */
@@ -287,6 +293,15 @@ public:
 	 */
 	bool getGroupStatistic(const uint32_t& token, GxsGroupStatistic& stats);
 
+    /*!
+     * \brief turtleGroupRequest
+     * 			Issues a browadcast group request using the turtle router generic search system. The request is obviously asynchroneous and will be
+     * 			handled in RsGenExchange when received.
+     * \param group_id
+     */
+    void turtleGroupRequest(const RsGxsGroupId& group_id);
+    void turtleSearchRequest(const std::string& match_string);
+
 protected:
 
 	bool messagePublicationTest(const RsGxsMsgMetaData&) ;
@@ -566,7 +581,7 @@ public:
      * @param msgs
      */
     void deleteMsgs(uint32_t& token, const GxsMsgReq& msgs);
-    
+
 protected:
     /*!
      * This represents the group before its signature is calculated
@@ -650,7 +665,7 @@ public:
      */
 
     void shareGroupPublishKey(const RsGxsGroupId& grpId,const std::set<RsPeerId>& peers) ;
-    
+
     /*!
      * Returns the local TS of the group as known by the network service.
      * This is useful to allow various network services to sync their update TS
@@ -755,6 +770,8 @@ protected:
      */
     int createMessage(RsNxsMsg* msg);
 
+    RsNetworkExchangeService *netService() const { return mNetService ; }
+
 private:
     /*!
      * convenience function to create sign

libretroshare/src/gxs/rsgxsdataaccess.cc

@@ -1807,6 +1807,23 @@ bool RsGxsDataAccess::updateGroupData(RsNxsGrp* grp) {
 	return mDataStore->updateGroup(grpM);
 }
 
+bool RsGxsDataAccess::getGroupData(const RsGxsGroupId& grpId, RsNxsGrp *& grp_data)
+{
+	RsStackMutex stack(mDataMutex);
+
+    std::map<RsGxsGroupId, RsNxsGrp*> grps ;
+
+    grps[grpId] = NULL ;
+
+    if(mDataStore->retrieveNxsGrps(grps, false, true))	// the false here is very important: it removes the private key parts.
+    {
+        grp_data = grps.begin()->second;
+        return true;
+    }
+    else
+        return false ;
+}
+
 bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) {
 
 	RsStackMutex stack(mDataMutex);

libretroshare/src/gxs/rsgxsdataaccess.h

@@ -156,6 +156,14 @@ public:
      */
     bool addMsgData(RsNxsMsg* msg);
 
+    /*!
+     * This retrieves a group from the gxs data base, this is a blocking call \n
+     * @param grp the group to add, memory ownership passed to the callee
+     * @return false if group cound not be retrieved
+     */
+    bool getGroupData(const RsGxsGroupId& grpId,RsNxsGrp *& grp_data);
+
+
 public:
 
 

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -166,8 +166,6 @@
 //                                                                       |                                       |
 //                          (Only send if rand() < sendingProb())        +---comes from mClientMsgUpdateMap -----+
 //
-//
-//
 // Suggestions
 // ===========
 //    * handleRecvSyncGroup should use mit->second.mLastPost to limit the sending of already known data
@@ -194,6 +192,53 @@
 //       Problem: without msg, we cannot know the grpId!!
 //
 //    * mClientMsgUpdateMap[peerid][grpId] is only updated when new msgs are received. Up to date groups will keep asking for lists!
+//
+// Distant sync
+// ============
+//
+// Distant sync uses tunnels to sync subscribed GXS groups that are not supplied by friends. Peers can subscribe to a GXS group using a RS link
+// which GXS uses to request updates through tunnels.
+//   * The whole exchange should be kept private and anonymous between the two distant peers, so we use the same trick than for FT: encrypt the data using the group ID.
+//   * The same node shouldn't be known as a common server for different GXS groups
+//
+//   GXS net service:
+//   	* talks to virtual peers, treated like normal peers
+//   	* virtual peers only depend on the server ID, not on tunnel ID, and be kept constant accross time so that ClientGroupUpdateMap is kept consistent
+//      * does not use tunnels if friends can already supply the data (??) This causes issues with "islands".
+//
+//   Tunnels:
+//	    * a specific service named GxsSyncTunnelService handles the creation/management of sync tunnels:
+//	    * tunnel data need to be encrypted.
+//
+//              bool manageTunnels(const RsGxsGroupId&) ;													// start managing tunnels for this group
+//              bool releaseTunnels(const RsGxsGroupId&) ;													// stop managing tunnels for this group
+//              bool sendData(const unsigned char *data,uint32_t size,const RsPeerId& virtual_peer) ;		// send data to this virtual peer
+//              bool getVirtualPeers(const RsGxsGroupId&, std::list<RsPeerId>& peers) ; 					// returns the virtual peers for this group
+//
+//   Proposed protocol:
+//      * request tunnels based on H(GroupId)
+//      * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does)
+//      * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias )
+//      * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync
+//
+//      * only use a single tunnel per virtual peer ID
+//
+//              Client  ------------------ TR(H(GroupId)) -------------->  Server
+//
+//              Client  <-------------------- T OK ----------------------  Server
+//
+//           [Encrypted traffic using H(GroupId, 96bits IV, tunnel ID)]
+//
+//              Client  <--------- VPID = H( Random IV | GroupId ) ------  Server
+//                 |                                                         |
+//                 +--------------> Mark the virtual peer active <-----------+
+//
+//   Unsolved problems:
+//      * if we want to preserve anonymity, we cannot prevent GXS from duplicating the data from virtual/real peers that actually are the same peers.
+//      * ultimately we should only use tunnels to sync GXS. The mix between tunnels and real peers is not a problem but will cause unnecessary traffic.
+//
+//   Notes:
+//      * given that GXS only talks to peers once every 2 mins, it's likely that keep-alive packets will be needed
 
 
 #include <unistd.h>
@@ -209,6 +254,7 @@
 #include "retroshare/rsgxscircles.h"
 #include "retroshare/rspeers.h"
 #include "pgp/pgpauxutils.h"
+#include "crypto/rscrypto.h"
 #include "util/rsdir.h"
 #include "util/rstime.h"
 #include "util/rsmemory.h"
@@ -224,6 +270,7 @@
  	NXS_NET_DEBUG_5		summary of transactions (useful to just know what comes in/out)
     NXS_NET_DEBUG_6		group sync statistics (e.g. number of posts at nighbour nodes, etc)
  	NXS_NET_DEBUG_7		encryption/decryption of transactions
+ 	NXS_NET_DEBUG_8		gxs distant sync
 
  ***/
 //#define NXS_NET_DEBUG_0 	1
@@ -234,6 +281,7 @@
 //#define NXS_NET_DEBUG_5 	1
 //#define NXS_NET_DEBUG_6 	1
 //#define NXS_NET_DEBUG_7 	1
+#define NXS_NET_DEBUG_8 	1
 
 //#define NXS_FRAG
 
@@ -256,6 +304,7 @@ static const uint32_t REJECTED_MESSAGE_RETRY_DELAY            =      24*3600; //
 static const uint32_t GROUP_STATS_UPDATE_DELAY                =          240; // update unsubscribed group statistics every 3 mins
 static const uint32_t GROUP_STATS_UPDATE_NB_PEERS             =            2; // number of peers to which the group stats are asked
 static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE            =       199000; // 200,000 bytes including signature and headers
+static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH          =           40; // dont search same group more than every 40 secs.
 
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN             = 0x00 ;
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR            = 0x01 ;
@@ -267,11 +316,12 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING     = 0x05 ;
 // Debug system to allow to print only for some IDs (group, Peer, etc)
 
 #if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2)  || defined(NXS_NET_DEBUG_3) \
- || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6)  || defined(NXS_NET_DEBUG_7)
+ || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6)  || defined(NXS_NET_DEBUG_7) \
+ || defined(NXS_NET_DEBUG_8)
 
 static const RsPeerId     peer_to_print     = RsPeerId(std::string(""))   ;
 static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ;	// use this to allow to this group id only, or "" for all IDs
-static const uint32_t     service_to_print  = RS_SERVICE_TYPE_GXS_TRANS ;                       	// use this to allow to this service id only, or 0 for all services
+static const uint32_t     service_to_print  = RS_SERVICE_GXS_TYPE_CHANNELS ;                       	// use this to allow to this service id only, or 0 for all services
 										// warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums)
 
 class nullstream: public std::ostream {};
@@ -288,7 +338,6 @@ static std::string nice_time_stamp(time_t now,time_t TS)
     }
 }
 
-
 static std::ostream& gxsnetdebug(const RsPeerId& peer_id,const RsGxsGroupId& grp_id,uint32_t service_type)
 {
     static nullstream null ;
@@ -314,7 +363,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
                                  RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs,
                                  const RsServiceInfo serviceInfo,
                                  RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs,
-                                 PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, uint32_t default_store_period, uint32_t default_sync_period)
+                                 PgpAuxUtils *pgpUtils, RsGxsNetTunnelService *mGxsNT,
+                                 bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period)
                                  : p3ThreadedService(), p3Config(), mTransactionN(0),
                                    mObserver(nxsObs), mDataStore(gds),
                                    mServType(servType), mTransactionTimeOut(TRANSAC_TIMEOUT),
@@ -322,8 +372,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
                                    mSyncTs(0), mLastKeyPublishTs(0),
                                    mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD),
                                    mCircles(circles), mGixs(gixs),
-                                   mReputations(reputations), mPgpUtils(pgpUtils),
+                                   mReputations(reputations), mPgpUtils(pgpUtils), mGxsNetTunnel(mGxsNT),
-                                   mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),
+                                   mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync),
                                    mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period),
                                    mDefaultMsgSyncPeriod(default_sync_period)
 {
@@ -331,6 +381,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
 	mOwnId = mNetMgr->getOwnId();
     mUpdateCounter = 0;
 
+	mLastCacheReloadTS = 0;
+
 	// check the consistency
 
 	if(mDefaultMsgStorePeriod > 0 && mDefaultMsgSyncPeriod > mDefaultMsgStorePeriod)
@@ -379,8 +431,7 @@ int RsGxsNetService::tick()
 {
 	// always check for new items arriving
 	// from peers
-    if(receivedItems())
+	recvNxsItemQueue();
-        recvNxsItemQueue();
 
     bool should_notify = false;
 
@@ -403,6 +454,7 @@ int RsGxsNetService::tick()
     {
         syncWithPeers();
         syncGrpStatistics();
+		checkDistantSyncState();
 
     	mSyncTs = now;
     }
@@ -450,8 +502,8 @@ void RsGxsNetService::processObserverNotifications()
 	    mNewPublishKeysToNotify.clear() ;
     }
 
-    if(!grps_copy.empty()) mObserver->notifyNewGroups  (grps_copy);
+    if(!grps_copy.empty()) mObserver->receiveNewGroups  (grps_copy);
-    if(!msgs_copy.empty()) mObserver->notifyNewMessages(msgs_copy);
+    if(!msgs_copy.empty()) mObserver->receiveNewMessages(msgs_copy);
 
     for(std::set<RsGxsGroupId>::const_iterator it(keys_copy.begin());it!=keys_copy.end();++it)
 		mObserver->notifyReceivePublishKey(*it);
@@ -520,6 +572,18 @@ void RsGxsNetService::syncWithPeers()
 
     std::set<RsPeerId> peers;
     mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers);
+
+	if(mAllowDistSync && mGxsNetTunnel != NULL)
+	{
+		// Grab all online virtual peers of distant tunnels for the current service.
+
+		std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
+		mGxsNetTunnel->getVirtualPeers(vpids);
+
+		for(auto it(vpids.begin());it!=vpids.end();++it)
+			peers.insert(RsPeerId(*it)) ;
+	}
+
     if (peers.empty()) {
         // nothing to do
         return;
@@ -549,7 +613,7 @@ void RsGxsNetService::syncWithPeers()
 #ifdef NXS_NET_DEBUG_5
 	GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << "  sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl;
 #endif
-        sendItem(grp);
+        generic_sendItem(grp);
     }
 
     if(!mAllowMsgSync)
@@ -665,7 +729,7 @@ void RsGxsNetService::syncWithPeers()
 #ifdef NXS_NET_DEBUG_7
 	    GXSNETDEBUG_PG(*sit,grpId) << "    Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << "  sending message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself - in clear " << std::endl;
 #endif
-	    sendItem(msg);
+		generic_sendItem(msg);
 
 #ifdef NXS_NET_DEBUG_5
 		GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << "  sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl;
@@ -676,6 +740,91 @@ void RsGxsNetService::syncWithPeers()
 #endif
 }
 
+void RsGxsNetService::generic_sendItem(RsNxsItem *si)
+{
+	// check if the item is to be sent to a distant peer or not
+
+	RsGxsGroupId tmp_grpId;
+
+	if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer( static_cast<RsGxsNetTunnelVirtualPeerId>(si->PeerId()),tmp_grpId))
+	{
+		RsNxsSerialiser ser(mServType);
+
+		uint32_t size = ser.size(si);
+		unsigned char *mem = (unsigned char *)rs_malloc(size) ;
+
+		if(!mem)
+			return ;
+
+#ifdef NXS_NET_DEBUG_8
+        GXSNETDEBUG_P_(si->PeerId()) << "Sending RsGxsNetTunnelService Item:" << (void*)si << " of type: " << std::hex << si->PacketId() << std::dec
+		                               << " transaction " << si->transactionNumber << " to virtual peer " << si->PeerId() << std::endl ;
+#endif
+		ser.serialise(si,mem,&size) ;
+
+		mGxsNetTunnel->sendTunnelData(mServType,mem,size,static_cast<RsGxsNetTunnelVirtualPeerId>(si->PeerId()));
+	}
+	else
+		sendItem(si) ;
+}
+
+void RsGxsNetService::checkDistantSyncState()
+{
+	if(!mAllowDistSync || mGxsNetTunnel==NULL || !mGrpAutoSync)
+		return ;
+
+	RsGxsGrpMetaTemporaryMap grpMeta;
+    mDataStore->retrieveGxsGrpMetaData(grpMeta);
+
+    // Go through group statistics and groups without information are re-requested to random peers selected
+    // among the ones who provided the group info.
+
+#ifdef NXS_NET_DEBUG_8
+    GXSNETDEBUG___<< "Checking distant sync for all groups." << std::endl;
+#endif
+	// get the list of online peers
+
+    std::set<RsPeerId> online_peers;
+    mNetMgr->getOnlineList(mServiceInfo.mServiceType , online_peers);
+
+    RS_STACK_MUTEX(mNxsMutex) ;
+
+    for(auto it(grpMeta.begin());it!=grpMeta.end();++it)
+		if(it->second->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)	// we only consider subscribed groups here.
+	    {
+#warning (cyril) We might need to also remove peers for recently unsubscribed groups
+			const RsGxsGroupId& grpId(it->first);
+		    const RsGxsGrpConfig& rec = locked_getGrpConfig(grpId) ;
+
+#ifdef NXS_NET_DEBUG_6
+		    GXSNETDEBUG__G(it->first) << "    group " << grpId;
+#endif
+			bool at_least_one_friend_is_supplier = false ;
+
+			for(auto it2(rec.suppliers.ids.begin());it2!=rec.suppliers.ids.end() && !at_least_one_friend_is_supplier;++it2)
+				if(online_peers.find(*it2) != online_peers.end())	                // check that the peer is online
+					at_least_one_friend_is_supplier = true ;
+
+			// That strategy is likely to create islands of friends connected to each other. There's no real way
+			// to decide what to do here, except maybe checking the last message TS remotely vs. locally.
+
+			if(at_least_one_friend_is_supplier)
+			{
+				mGxsNetTunnel->releaseDistantPeers(mServType,grpId);
+#ifdef NXS_NET_DEBUG_8
+				GXSNETDEBUG___<< "  Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl;
+#endif
+			}
+			else
+			{
+				mGxsNetTunnel->requestDistantPeers(mServType,grpId);
+#ifdef NXS_NET_DEBUG_8
+				GXSNETDEBUG___<< "  Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl;
+#endif
+			}
+		}
+}
+
 void RsGxsNetService::syncGrpStatistics()
 {
     RS_STACK_MUTEX(mNxsMutex) ;
@@ -690,6 +839,17 @@ void RsGxsNetService::syncGrpStatistics()
     std::set<RsPeerId> online_peers;
     mNetMgr->getOnlineList(mServiceInfo.mServiceType, online_peers);
 
+	if(mAllowDistSync && mGxsNetTunnel != NULL)
+	{
+		// Grab all online virtual peers of distant tunnels for the current service.
+
+		std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
+		mGxsNetTunnel->getVirtualPeers(vpids);
+
+		for(auto it(vpids.begin());it!=vpids.end();++it)
+			online_peers.insert(RsPeerId(*it)) ;
+	}
+
     // Go through group statistics and groups without information are re-requested to random peers selected
     // among the ones who provided the group info.
 
@@ -704,44 +864,44 @@ void RsGxsNetService::syncGrpStatistics()
 #endif
 
 	    if(rec.statistics_update_TS + GROUP_STATS_UPDATE_DELAY < now && rec.suppliers.ids.size() > 0)
-	    {
+		{
 #ifdef NXS_NET_DEBUG_6
-		    GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl;
+			GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl;
 #endif
-            // randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group
+			// randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group
 
-		    uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ;
+			uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ;
 
-		    std::set<RsPeerId>::const_iterator rit = rec.suppliers.ids.begin();
+			std::set<RsPeerId>::const_iterator rit = rec.suppliers.ids.begin();
-		    for(uint32_t i=0;i<n;++i)
+			for(uint32_t i=0;i<n;++i)
-			    ++rit ;
+				++rit ;
 
-		    for(uint32_t i=0;i<std::min(rec.suppliers.ids.size(),(size_t)GROUP_STATS_UPDATE_NB_PEERS);++i)
+			for(uint32_t i=0;i<std::min(rec.suppliers.ids.size(),(size_t)GROUP_STATS_UPDATE_NB_PEERS);++i)
-                    {
-                // we started at a random position in the set, wrap around if the end is reached
-                if(rit == rec.suppliers.ids.end())
-                    rit = rec.suppliers.ids.begin() ;
-
-			RsPeerId peer_id = *rit ;
-			++rit ;
-
-                	if(online_peers.find(peer_id) != online_peers.end())	// check that the peer is online
 			{
+				// we started at a random position in the set, wrap around if the end is reached
+				if(rit == rec.suppliers.ids.end())
+					rit = rec.suppliers.ids.begin() ;
+
+				RsPeerId peer_id = *rit ;
+				++rit ;
+
+				if(online_peers.find(peer_id) != online_peers.end())	// check that the peer is online
+				{
 #ifdef NXS_NET_DEBUG_6
-				GXSNETDEBUG_PG(peer_id,it->first) << "  asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl;
+					GXSNETDEBUG_PG(peer_id,it->first) << "  asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl;
 #endif
 
-				RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ;
+					RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ;
 
-				grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ;
+					grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ;
 
-				grs->grpId = it->first ;
+					grs->grpId = it->first ;
-				grs->PeerId(peer_id) ;
+					grs->PeerId(peer_id) ;
 
-				sendItem(grs) ;
+					generic_sendItem(grs) ;
+				}
 			}
-                    }
+		}
-	    }
 #ifdef NXS_NET_DEBUG_6
 	    else
 		    GXSNETDEBUG__G(it->first) << " up to date." << std::endl;
@@ -819,7 +979,7 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs)
 	    GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "  sending back statistics item with " << vec.size() << " elements." << std::endl;
 #endif
 
-	    sendItem(grs_resp) ;
+	    generic_sendItem(grs_resp) ;
     }
     else if(grs->request_type == RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_RESPONSE)
 	{
@@ -1391,11 +1551,11 @@ public:
 
     void operator() (RsItem* item)
     {
-        RsGxsMsgUpdateItem* mui;
+        RsGxsMsgUpdateItem        *mui;
-        RsGxsGrpUpdateItem* gui;
+        RsGxsGrpUpdateItem        *gui;
-        RsGxsServerGrpUpdateItem* gsui;
+        RsGxsServerGrpUpdateItem  *gsui;
-        RsGxsServerMsgUpdateItem* msui;
+        RsGxsServerMsgUpdateItem  *msui;
-        RsGxsGrpConfigItem* mgci;
+        RsGxsGrpConfigItem        *mgci;
 
         if((mui = dynamic_cast<RsGxsMsgUpdateItem*>(item)) != NULL)
             check_store(mui->peerID,mClientMsgMap,*mui);
@@ -1421,7 +1581,6 @@ private:
     RsGxsNetService::GrpConfigMap& mGrpConfigMap;
 
     RsGxsServerGrpUpdate& mServerGrpUpdate;
-
 };
 
 bool RsGxsNetService::loadList(std::list<RsItem *> &load)
@@ -1431,11 +1590,10 @@ bool RsGxsNetService::loadList(std::list<RsItem *> &load)
     // The delete is done in StoreHere, if necessary
 
     std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate));
+	time_t now = time(NULL);
 
     // We reset group statistics here. This is the best place since we know at this point which are all unsubscribed groups.
 
-    time_t now = time(NULL);
-
     for(GrpConfigMap::iterator it(mServerGrpConfigMap.begin());it!=mServerGrpConfigMap.end();++it)
 	{
 		// At each reload, we reset the count of visible messages. It will be rapidely restored to its real value from friends.
@@ -1489,6 +1647,7 @@ struct get_second : public std::unary_function<typename UpdateMap::value_type, R
     typename UpdateMap::key_type ItemClass::*ID_member ;
 };
 
+
 bool RsGxsNetService::saveList(bool& cleanup, std::list<RsItem*>& save)
 {
 	RS_STACK_MUTEX(mNxsMutex) ;
@@ -1520,11 +1679,44 @@ RsSerialiser *RsGxsNetService::setupSerialiser()
     return rss;
 }
 
+RsItem *RsGxsNetService::generic_recvItem()
+{
+	{
+		RsItem *item ;
+
+		if(NULL != (item=recvItem()))
+			return item ;
+	}
+
+	unsigned char *data = NULL ;
+	uint32_t size = 0 ;
+	RsGxsNetTunnelVirtualPeerId virtual_peer_id ;
+
+	while(mAllowDistSync && mGxsNetTunnel!=NULL && mGxsNetTunnel->receiveTunnelData(mServType,data,size,virtual_peer_id))
+	{
+		RsNxsItem *item = dynamic_cast<RsNxsItem*>(RsNxsSerialiser(mServType).deserialise(data,&size)) ;
+		item->PeerId(virtual_peer_id) ;
+
+		free(data) ;
+
+		if(!item)
+			continue ;
+
+#ifdef NXS_NET_DEBUG_8
+        GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetTunnelService Item:" << (void*)item << " of type: " << std::hex << item->PacketId() << std::dec
+		                               << " transaction " << item->transactionNumber << " from virtual peer " << item->PeerId() << std::endl ;
+#endif
+		return item ;
+	}
+
+	return NULL ;
+}
+
 void RsGxsNetService::recvNxsItemQueue()
 {
     RsItem *item ;
 
-    while(NULL != (item=recvItem()))
+    while(NULL != (item=generic_recvItem()))
     {
 #ifdef NXS_NET_DEBUG_1
         GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << " type=" << std::hex << item->PacketId() << std::dec << std::endl ;
@@ -1829,7 +2021,7 @@ void RsGxsNetService::debugDump()
 
     GXSNETDEBUG___<< "RsGxsNetService::debugDump():" << std::endl;
 
-    RsGxsMetaDataTemporaryMap<RsGxsGrpMetaData> grpMetas;
+    RsGxsGrpMetaTemporaryMap grpMetas;
 
     if(!group_id_to_print.isNull())
         grpMetas[group_id_to_print] = NULL ;
@@ -1842,7 +2034,7 @@ void RsGxsNetService::debugDump()
 
     for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
     {
-        RsGxsMetaDataTemporaryMap<RsGxsGrpMetaData>::const_iterator it2 = grpMetas.find(it->first) ;
+        RsGxsGrpMetaTemporaryMap::const_iterator it2 = grpMetas.find(it->first) ;
         RsGxsGrpMetaData *grpMeta = (it2 != grpMetas.end())? it2->second : NULL;
         std::string subscribe_string = (grpMeta==NULL)?"Unknown" :  ((grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)?" Subscribed":" NOT Subscribed") ;
 
@@ -2125,7 +2317,7 @@ void RsGxsNetService::processTransactions()
                     lit_end = tr->mItems.end();
 
                     for(; lit != lit_end; ++lit){
-                        sendItem(*lit);
+                        generic_sendItem(*lit);
                     }
 
                     tr->mItems.clear(); // clear so they don't get deleted in trans cleaning
@@ -2234,7 +2426,7 @@ void RsGxsNetService::processTransactions()
                     trans->transactFlag = RsNxsTransacItem::FLAG_END_SUCCESS;
                     trans->transactionNumber = transN;
                     trans->PeerId(tr->mTransaction->PeerId());
-                    sendItem(trans);
+                    generic_sendItem(trans);
 
                     // move to completed transactions
 
@@ -2277,7 +2469,7 @@ void RsGxsNetService::processTransactions()
                                     (tr->mTransaction->transactFlag & RsNxsTransacItem::FLAG_TYPE_MASK);
                     trans->transactionNumber = transN;
                     trans->PeerId(tr->mTransaction->PeerId());
-                    sendItem(trans);
+                    generic_sendItem(trans);
                     tr->mFlag = NxsTransaction::FLAG_STATE_RECEIVING;
 
                 }
@@ -2654,7 +2846,7 @@ void RsGxsNetService::locked_pushMsgTransactionFromList(std::list<RsNxsItem*>& r
     newTrans->mTransaction->PeerId(mOwnId);
 
     if (locked_addTransaction(newTrans))
-    	sendItem(transac);
+    	generic_sendItem(transac);
     else
     {
         delete newTrans;
@@ -2950,7 +3142,7 @@ void RsGxsNetService::locked_pushGrpTransactionFromList( std::list<RsNxsItem*>&
 	newTrans->mTransaction->PeerId(mOwnId);
 
     if (locked_addTransaction(newTrans))
-	    sendItem(transac);
+	    generic_sendItem(transac);
     else
     {
 	    delete newTrans;
@@ -3084,7 +3276,7 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
 {
 
 #ifdef NXS_NET_DEBUG_1
-	GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl;
+	GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send from TransN: " << tr->mTransaction->transactionNumber << std::endl;
 #endif
 
 	// go groups requested in transaction tr
@@ -3097,7 +3289,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
 	{
 		RsNxsSyncGrpItem* item = dynamic_cast<RsNxsSyncGrpItem*>(*lit);
 		if (item)
+		{
+#ifdef NXS_NET_DEBUG_1
+			GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "locked_genSendGrpsTransaction() retrieving data for group \"" << item->grpId << "\"" << std::endl;
+#endif
 			grps[item->grpId] = NULL;
+		}
 		else
 		{
 #ifdef NXS_NET_DEBUG_1
@@ -3109,7 +3306,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
 	if(!grps.empty())
 		mDataStore->retrieveNxsGrps(grps, false, false);
 	else
+	{
+#ifdef NXS_NET_DEBUG_1
+		GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "RsGxsNetService::locked_genSendGrpsTransaction(): no group to request! This is unexpected" << std::endl;
+#endif
 		return;
+	}
 
 	NxsTransaction* newTr = new NxsTransaction();
 	newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM;
@@ -3126,6 +3328,9 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
 		mit->second->transactionNumber = transN;
 		newTr->mItems.push_back(mit->second);
         	mit->second = NULL ; // avoids deletion
+#ifdef NXS_NET_DEBUG_1
+		GXSNETDEBUG_PG(tr->mTransaction->PeerId(),mit->first) << "RsGxsNetService::locked_genSendGrpsTransaction(): adding grp data of group \"" << mit->first << "\" to transaction" << std::endl;
+#endif
 	}
 
 	if(newTr->mItems.empty()){
@@ -3154,8 +3359,8 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
 	ntr->PeerId(tr->mTransaction->PeerId());
 
 	if(locked_addTransaction(newTr))
-		sendItem(ntr);
+		generic_sendItem(ntr);
-        else
+	else
         {
             delete ntr ;
             delete newTr;
@@ -3449,7 +3654,7 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr)
     ntr->PeerId(tr->mTransaction->PeerId());
 
     if(locked_addTransaction(newTr))
-	    sendItem(ntr);
+		generic_sendItem(ntr);
     else
     {
 	    delete ntr ;
@@ -3768,7 +3973,7 @@ void RsGxsNetService::locked_pushGrpRespFromList(std::list<RsNxsItem*>& respList
                                     << peer << " with " << respList.size() << " groups " << std::endl;
 #endif
 	if(locked_addTransaction(tr))
-	    sendItem(trItem);
+	    generic_sendItem(trItem);
     else
     {
         delete tr ;
@@ -3910,6 +4115,18 @@ bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData
 #ifdef NXS_NET_DEBUG_4
 	GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId()"<< std::endl;
 #endif
+	// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
+
+	RsGxsGroupId peer_grp ;
+	if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
+	{
+#warning (cyril) make sure that this is not a problem for cross-service sending of items
+#ifdef NXS_NET_DEBUG_4
+        GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "  Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
+#endif
+		return false ;
+	}
+
 	// first do the simple checks
 	uint8_t circleType = grpMeta.mCircleType;
 
@@ -3963,6 +4180,17 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs
     GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::checkCanRecvMsgFromPeer()";
     GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "  peer Id = " << sslId << ", grpId=" << grpMeta.mGroupId <<std::endl;
 #endif
+	// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
+
+	RsGxsGroupId peer_grp ;
+	if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
+	{
+#ifdef NXS_NET_DEBUG_4
+        GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "  Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
+#endif
+		return false ;
+	}
+
     // first do the simple checks
     uint8_t circleType = grpMeta.mCircleType;
     should_encrypt_id.clear() ;
@@ -4293,7 +4521,7 @@ void RsGxsNetService::locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, c
 #endif
 	// signal peer to prepare for transaction
 	if(locked_addTransaction(tr))
-		sendItem(trItem);
+		generic_sendItem(trItem);
 	else
 	{
 		delete tr ;
@@ -4306,6 +4534,16 @@ bool RsGxsNetService::canSendMsgIds(std::vector<RsGxsMsgMetaData*>& msgMetas, co
 #ifdef NXS_NET_DEBUG_4
     GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl;
 #endif
+	// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
+
+	RsGxsGroupId peer_grp ;
+	if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
+	{
+#ifdef NXS_NET_DEBUG_4
+        GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "  Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
+#endif
+		return false ;
+	}
 
     // first do the simple checks
     uint8_t circleType = grpMeta.mCircleType;
@@ -4546,7 +4784,7 @@ void RsGxsNetService::processExplicitGroupRequests()
 		for(; git != groupIdList.end(); ++git)
 		{
 #ifdef NXS_NET_DEBUG_0
-            GXSNETDEBUG_PG(peerId,*git) << "   group request for grp ID " << *git << " to peer " << peerId << std::endl;
+            GXSNETDEBUG_P_(peerId) << "   group request for grp ID " << *git << " to peer " << peerId << std::endl;
 #endif
             RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType);
 			item->grpId = *git;
@@ -4680,7 +4918,7 @@ void RsGxsNetService::sharePublishKeysPending()
             publishKeyItem->private_key = publishKey ;
             publishKeyItem->PeerId(*it);
 
-            sendItem(publishKeyItem);
+            generic_sendItem(publishKeyItem);
 #ifdef NXS_NET_DEBUG_3
             GXSNETDEBUG_PG(*it,grpMeta->mGroupId) << "  sent key item to " << *it << std::endl;
 #endif
@@ -4841,6 +5079,18 @@ bool RsGxsNetService::removeGroups(const std::list<RsGxsGroupId>& groups)
     return true ;
 }
 
+bool RsGxsNetService::isDistantPeer(const RsPeerId& pid)
+{
+    RS_STACK_MUTEX(mNxsMutex) ;
+
+	if(!mAllowDistSync || mGxsNetTunnel == NULL)
+		return false ;
+
+	RsGxsGroupId group_id ;
+
+	return mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(pid),group_id);
+}
+
 bool RsGxsNetService::stampMsgServerUpdateTS(const RsGxsGroupId& gid)
 {
     RS_STACK_MUTEX(mNxsMutex) ;
@@ -4856,3 +5106,283 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid)
 
     return true;
 }
+
+TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+	RS_STACK_MUTEX(mNxsMutex) ;
+
+    time_t now = time(NULL);
+    auto it = mSearchedGroups.find(group_id) ;
+
+    if(mSearchedGroups.end() != it && (it->second.ts + MIN_DELAY_BETWEEN_GROUP_SEARCH > now))
+    {
+        std::cerr << "(WW) Last turtle request was " << now - it->second.ts << " secs ago. Not searching again." << std::endl;
+        return it->second.request_id;
+    }
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG__G(group_id) << "  requesting group id " << group_id << " using turtle" << std::endl;
+#endif
+	TurtleRequestId req = mGxsNetTunnel->turtleGroupRequest(group_id,this) ;
+
+    GroupRequestRecord& rec(mSearchedGroups[group_id]) ;
+
+	rec.request_id = req;
+	rec.ts         = now;
+
+    mSearchRequests[req] = group_id;
+
+    return req;
+}
+TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_string)
+{
+    return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
+}
+
+static bool termSearch(const std::string& src, const std::string& substring)
+{
+		/* always ignore case */
+	return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
+}
+
+bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
+{
+    RS_STACK_MUTEX(mNxsMutex) ;
+
+    auto it = mDistantSearchResults.find(req) ;
+
+    if(it == mDistantSearchResults.end())
+        return false ;
+
+    group_infos = it->second;
+    return true ;
+}
+bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs)
+{
+	RS_STACK_MUTEX(mNxsMutex) ;
+    for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it)
+    {
+        auto it2 = it->second.find(group_id) ;
+
+        if(it2 != it->second.end())
+        {
+            gs = it2->second;
+            return true ;
+        }
+    }
+    return false ;
+}
+bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
+{
+    RS_STACK_MUTEX(mNxsMutex) ;
+    mDistantSearchResults.erase(id);
+    return true ;
+}
+void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos)
+{
+    RS_STACK_MUTEX(mNxsMutex) ;
+
+    RsGxsGrpMetaTemporaryMap grpMeta;
+    std::map<RsGxsGroupId,RsGxsGroupSummary>& search_results_map(mDistantSearchResults[req]) ;
+
+    for(auto it(group_infos.begin());it!=group_infos.end();++it)
+        if(search_results_map.find((*it).group_id) == search_results_map.end())
+			grpMeta[(*it).group_id] = NULL;
+
+    mDataStore->retrieveGxsGrpMetaData(grpMeta);
+
+	std::list<RsGxsGroupSummary> filtered_results ;
+
+	// only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure
+
+    for(auto it(group_infos.begin());it!=group_infos.end();++it)
+        if(grpMeta[(*it).group_id] == NULL)
+        {
+			filtered_results.push_back(*it) ;
+
+            auto it2 = search_results_map.find((*it).group_id) ;
+
+            if(it2 != search_results_map.end())
+            {
+                // update existing data
+
+                it2->second.popularity++ ;
+                it2->second.number_of_messages = std::max(it2->second.number_of_messages,(*it).number_of_messages) ;
+            }
+            else
+            {
+				search_results_map[(*it).group_id] = *it;
+				search_results_map[(*it).group_id].popularity = 1; // number of results so far
+            }
+
+			mObserver->receiveDistantSearchResults(req,(*it).group_id) ;
+        }
+}
+
+void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
+{
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
+#endif
+    auto it = mSearchRequests.find(req);
+
+    if(mSearchRequests.end() == it)
+    {
+        std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
+        return;
+    }
+    RsGxsGroupId grpId = it->second;
+
+    uint8_t encryption_master_key[32];
+    Sha256CheckSum s = RsDirUtil::sha256sum(grpId.toByteArray(),grpId.SIZE_IN_BYTES);
+    memcpy(encryption_master_key,s.toByteArray(),32);
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " attempting data decryption with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl;
+#endif
+    unsigned char *clear_group_data = NULL;
+    uint32_t clear_group_data_len ;
+
+    if(!librs::crypto::decryptAuthenticateData(encrypted_group_data,encrypted_group_data_len,encryption_master_key,clear_group_data,clear_group_data_len))
+    {
+        std::cerr << "(EE) Could not decrypt data. Something went wrong. Wrong key??" << std::endl;
+        return ;
+    }
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " successfuly decrypted data : " << RsUtil::BinToHex(clear_group_data,clear_group_data_len,50) << std::endl;
+#endif
+    RsItem *item = RsNxsSerialiser(mServType).deserialise(clear_group_data,&clear_group_data_len) ;
+	free(clear_group_data);
+    clear_group_data = NULL ;
+
+    RsNxsGrp *nxs_grp = dynamic_cast<RsNxsGrp*>(item) ;
+
+    if(nxs_grp == NULL)
+    {
+        std::cerr << "(EE) decrypted item is not a RsNxsGrp. Weird!" << std::endl;
+        return ;
+    }
+    std::vector<RsNxsGrp*> new_grps(1,nxs_grp);
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " passing the grp data to observer." << std::endl;
+#endif
+    mObserver->receiveNewGroups(new_grps);
+}
+
+bool RsGxsNetService::search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos)
+{
+	RsGxsGrpMetaTemporaryMap grpMetaMap;
+    {
+		RS_STACK_MUTEX(mNxsMutex) ;
+		mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
+    }
+
+    RsGroupNetworkStats stats ;
+
+    for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it)
+		if(termSearch(it->second->mGroupName,substring))
+		{
+			getGroupNetworkStats(it->first,stats) ;
+
+            RsGxsGroupSummary s ;
+            s.group_id           = it->first ;
+    		s.group_name         = it->second->mGroupName ;
+    		s.group_description  = it->second->mGroupName ; // to be filled with something better when we use the real search
+    		s.search_context     = it->second->mGroupName ;
+            s.sign_flags         = it->second->mSignFlags;
+    		s.publish_ts         = it->second->mPublishTs;
+    		s.author_id          = it->second->mAuthorId;
+    		s.number_of_messages = stats.mMaxVisibleCount ;
+    		s.last_message_ts    = stats.mLastGroupModificationTS ;
+            s.popularity         = it->second->mPop;
+
+            group_infos.push_back(s) ;
+		}
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << "  performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
+#endif
+    return !group_infos.empty();
+}
+
+bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)
+{
+    // First look into the grp hash cache
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " Received group data request for hash " << hashed_group_id << std::endl;
+#endif
+    auto it = mGroupHashCache.find(hashed_group_id) ;
+    RsNxsGrp *grp_data = NULL ;
+
+    if(mGroupHashCache.end() != it)
+    {
+        grp_data = it->second;
+    }
+    else
+	{
+		// Now check if the last request was too close in time, in which case, we dont retrieve data.
+
+		if(mLastCacheReloadTS + 60 > time(NULL))
+		{
+			std::cerr << "(WW) Not found in cache, and last cache reload less than 60 secs ago. Returning false. " << std::endl;
+			return false ;
+		}
+
+#ifdef NXS_NET_DEBUG_8
+		GXSNETDEBUG___ << " reloading group cache information" << std::endl;
+#endif
+		RsNxsGrpDataTemporaryMap grpDataMap;
+		{
+			RS_STACK_MUTEX(mNxsMutex) ;
+			mDataStore->retrieveNxsGrps(grpDataMap, true, true);
+            mLastCacheReloadTS = time(NULL);
+		}
+
+        for(auto it(grpDataMap.begin());it!=grpDataMap.end();++it)
+            if(it->second->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )	// only cache subscribed groups
+            {
+                RsNxsGrp *grp = it->second ;
+                delete grp->metaData ; 		// clean private keys
+                grp->metaData = NULL ;
+
+                Sha1CheckSum hash(RsDirUtil::sha1sum(it->first.toByteArray(),it->first.SIZE_IN_BYTES));
+
+				mGroupHashCache[hash] = grp ;
+                it->second = NULL ; // prevents deletion
+
+                if(hash == hashed_group_id)
+                    grp_data = grp ;
+            }
+	}
+
+    if(!grp_data)
+    {
+#ifdef NXS_NET_DEBUG_8
+		GXSNETDEBUG___ << " no group found for hash " << hashed_group_id << ": returning false." << std::endl;
+#endif
+        return false ;
+    }
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " found corresponding group data group id in cache group_id=" << grp_data->grpId << std::endl;
+#endif
+    // Finally, serialize and encrypt the grp data
+
+    uint32_t size = RsNxsSerialiser(mServType).size(grp_data);
+	RsTemporaryMemory mem(size) ;
+
+    RsNxsSerialiser(mServType).serialise(grp_data,mem,&size) ;
+
+    uint8_t encryption_master_key[32];
+    Sha256CheckSum s = RsDirUtil::sha256sum(grp_data->grpId.toByteArray(),grp_data->grpId.SIZE_IN_BYTES);
+    memcpy(encryption_master_key,s.toByteArray(),32);
+
+#ifdef NXS_NET_DEBUG_8
+	GXSNETDEBUG___ << " sending data encrypted with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl;
+#endif
+    return librs::crypto::encryptAuthenticateData(mem,size,encryption_master_key,encrypted_group_data,encrypted_group_data_len);
+}
+

libretroshare/src/gxs/rsgxsnetservice.h

@@ -35,6 +35,7 @@
 #include "pqi/p3linkmgr.h"
 #include "rsitems/rsnxsitems.h"
 #include "rsitems/rsgxsupdateitems.h"
+#include "rsgxsnettunnel.h"
 #include "rsgxsnetutils.h"
 #include "pqi/p3cfgmgr.h"
 #include "rsgixs.h"
@@ -58,6 +59,14 @@ class RsGroupNetworkStatsRecord
     time_t update_TS ;
 };
 
+struct GroupRequestRecord
+{
+    GroupRequestRecord(): ts(0), request_id(0) {}
+
+    time_t ts ;
+    TurtleRequestId request_id;
+};
+
 /*!
  * This class implements the RsNetWorkExchangeService
  * using transactions to handle synchrnisation of Nxs items between
@@ -89,8 +98,8 @@ public:
       			  RsNxsObserver *nxsObs,  // used to be = NULL.
       			  const RsServiceInfo serviceInfo,
       			  RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL,
-      			  PgpAuxUtils *pgpUtils = NULL,
+      			  PgpAuxUtils *pgpUtils = NULL, RsGxsNetTunnelService *mGxsNT = NULL,
-      			  bool grpAutoSync = true, bool msgAutoSync = true,
+      			  bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false,
 	                uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD,
 	                uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD);
 
@@ -103,6 +112,8 @@ public:
 public:
 
 
+	virtual uint16_t serviceType() const { return mServType ; }
+
     /*!
      * Use this to set how far back synchronisation and storage of messages should take place
      * @param age the max age a sync/storage item can to be allowed in a synchronisation
@@ -119,6 +130,23 @@ public:
 	virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; }
 	virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; }
 
+    /*!
+     * \brief Search methods.
+     * 			These four methods are used to request distant search and receive the results.
+     * \param group_id
+     */
+    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id);
+    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string);
+
+    virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) ;
+	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len);
+	virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos);
+	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len);
+
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos);
+	virtual bool clearDistantSearchResults(const TurtleRequestId& id);
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&);
+
     /*!
      * pauses synchronisation of subscribed groups and request for group id
      * from peers
@@ -167,6 +195,7 @@ public:
     virtual bool getGroupServerUpdateTS(const RsGxsGroupId& gid,time_t& grp_server_update_TS,time_t& msg_server_update_TS) ;
     virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) ;
     virtual bool removeGroups(const std::list<RsGxsGroupId>& groups);
+    virtual bool isDistantPeer(const RsPeerId& pid);
 
     /* p3Config methods */
 public:
@@ -394,6 +423,7 @@ private:
     void locked_pushGrpRespFromList(std::list<RsNxsItem*>& respList, const RsPeerId& peer, const uint32_t& transN);
     void locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN);
     
+	void checkDistantSyncState();
     void syncWithPeers();
     void syncGrpStatistics();
     void addGroupItemToList(NxsTransaction*& tr,
@@ -493,6 +523,9 @@ private:
     void cleanRejectedMessages();
     void processObserverNotifications();
 
+	void generic_sendItem(RsNxsItem *si);
+	RsItem *generic_recvItem();
+
 private:
 
 	static void locked_checkDelay(uint32_t& time_in_secs);
@@ -541,8 +574,11 @@ private:
     RsGixs *mGixs;
     RsGixsReputation* mReputations;
     PgpAuxUtils *mPgpUtils;
+	RsGxsNetTunnelService *mGxsNetTunnel;
+
     bool mGrpAutoSync;
     bool mAllowMsgSync;
+    bool mAllowDistSync;
 
     // need to be verfied
     std::vector<AuthorPending*> mPendingResp;
@@ -577,10 +613,18 @@ private:
     std::set<RsGxsGroupId> mNewStatsToNotify ;
     std::set<RsGxsGroupId> mNewPublishKeysToNotify ;
 
+    // Distant search result map
+    std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSummary> > mDistantSearchResults ;
+
     void debugDump();
 
 	uint32_t mDefaultMsgStorePeriod ;
 	uint32_t mDefaultMsgSyncPeriod ;
+
+    std::map<Sha1CheckSum, RsNxsGrp*> mGroupHashCache;
+    std::map<TurtleRequestId,RsGxsGroupId> mSearchRequests;
+    std::map<RsGxsGroupId,GroupRequestRecord> mSearchedGroups ;
+    time_t mLastCacheReloadTS ;
 };
 
 #endif // RSGXSNETSERVICE_H

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -0,0 +1,277 @@
+/*
+ * libretroshare/src/gxs: rsgxsnettunnel.h
+ *
+ * General Data service, interface for RetroShare.
+ *
+ * Copyright 2018-2018 by Cyril Soler
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare.project@gmail.com"
+ *
+ */
+
+#pragma once
+
+#include <map>
+
+#include "turtle/p3turtle.h"
+#include "retroshare/rsgxsdistsync.h"
+
+/*!
+ * \brief The RsGxsNetTunnelService class takes care of requesting tunnels to the turtle router, through which it is possible to sync
+ *        a particular GXS group. For each group, a set of virtual peers, corresponding to active tunnels will be made available
+ *        to RsGxsNetService.
+ *
+ *        It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group
+ *        is already available at regular friends or not.
+ *
+ *        Tunnel management is done at the level of groups rather than services, because we would like to keep the possibility to not
+ *        request tunnels for some groups which do not need it, and only request tunnels for specific groups that cannot be provided
+ *        by direct connections.
+ */
+
+//   Protocol:
+//      * request tunnels based on H(GroupId)
+//      * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does)
+//      * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias )
+//      * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync
+//      * only use a single tunnel per virtual peer ID
+//                                                                                        -
+//              Client  ------------------ TR(H(GroupId)) -------------->  Server         |
+//                                                                                        |   Turtle
+//              Client  <-------------------- T OK ----------------------  Server         |
+//                                                                                        -
+//                               Here, a turtle vpid is known                             |   [ addVirtualPeer() called by turtle ]
+//                                                                                        -
+//           [Encrypted traffic using H(GroupId | Tunnel ID, 96bits IV)]                  |
+//                                                                                        |
+//              Client  <--------- VPID = H( Random IV | GroupId ) ------  Server         |
+//                 |                                                         |            |
+//                 +--------------> Mark the virtual peer active <-----------+            |    Encrypted traffic decoded locally and sorted
+//                                                                                        |
+//                         Here, a consistent virtual peer ID is known                    |
+//                                                                                        |
+//              Client  <------------------- GXS Data ------------------>  Server         |
+//                                                                                        -
+//   Notes:
+//      * tunnels are established symmetrically. If a distant peers wants to sync the same group, they'll have to open a single tunnel, with a different ID.
+//        Groups therefore have two states:
+//          - managed : the group can be used to answer tunnel requests. If server tunnels are established, the group will be synced with these peers
+//          - tunneled: the group will actively request tunnels. If tunnels are established both ways, the same virtual peers will be used so the tunnels are "merged".
+//                        * In practice, that means one of the two tunnels will not be used and therefore die.
+//                        * If a tunneled group already has enough virtual peers, it will not request for tunnels itself.
+//
+//           Group policy      | Request tunnels   |    SyncWithPeers      |  Item receipt
+//         --------------------+-------------------+-----------------------+----------------
+//             Passive         |        no         |  If peers present     | If peers present
+//             Active          |  yes, if no peers |  If peers present     | If peers present
+//                             |                   |                       |
+//
+//      * when a service has the DistSync flag set, groups to sync are communicated passively to the GxsNetTunnel service when requesting distant peers.
+//        However, a call should be made to set a particular group policy to "ACTIVE" for group that do not have peers and need some.
+//
+//      * services also need to retrieve GXS data items that come out of tunnels. These will be available as (data,len) type, since they are not de-serialized.
+//
+//      * GxsNetService stores data information (such as update maps) per peerId, so it makes sense to use the same PeerId for all groups of a given service
+//        Therefore, virtual peers are stored separately from groups, because each one can sync multiple groups.
+//
+//      * virtual peers are also shared among services. This reduces the required amount of tunnels and tunnel requests to send.
+//
+//      * tunnels for a given service may also be used by the net service of p3IdService in order to explicitely request missing GxsIds.
+//        So GxsNetTunnel somehow allows different services to use the same tunnel. However we make sure that this traffic is limited to only p3IdService.
+//
+//   How do we know that a group needs distant sync?
+//		* look into GrpConfigMap for suppliers. Suppliers is cleared at load.
+//		* last_update_TS in GrpConfigMap is randomised so it cannot be used
+//      * we need a way to know that there's no suppliers for good reasons (not that we just started)
+//
+//   Security
+//      * the question of sync-ing with distant anonymous peers is a bit tricky because these peers can try to generate fake requests or change which messages are available
+//        and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
+//
+
+class RsGxsNetTunnelItem ;
+class RsNetworkExchangeService ;
+
+class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread, public p3Config, public RsGxsDistSync
+{
+public:
+	  RsGxsNetTunnelService() ;
+	  virtual ~RsGxsNetTunnelService() ;
+
+      /*!
+       * \brief registerSearchableService
+       * 			Adds the network exchange service as a possible search source. This is used to allow distant search on the corresponding
+       * 			GXS service.
+       * \return
+       * 		always returns true.
+       */
+      bool registerSearchableService(RsNetworkExchangeService *) ;
+
+	  /*!
+	   * \brief Manage tunnels for this group
+	   *	@param group_id group for which tunnels should be released
+	   */
+      bool requestDistantPeers(uint16_t service_id,const RsGxsGroupId&group_id) ;
+
+	  /*!
+	   * \brief Stop managing tunnels for this group
+	   *	@param group_id group for which tunnels should be released
+	   */
+      bool releaseDistantPeers(uint16_t service_id, const RsGxsGroupId&group_id) ;
+
+	  /*!
+	   * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and
+	   *        alive. This function also "registers" the group which allows to handle tunnel requests in the server side.
+	   */
+      bool getVirtualPeers(std::list<RsGxsNetTunnelVirtualPeerId>& peers) ;	// returns the virtual peers for this service
+
+      /*!
+       * \brief serviceId
+       * 			Overloads the method in RsTurtleClientService.
+       * \return
+       * 			The service id for RsGxsNetTunnel.
+       */
+      uint16_t serviceId() const ;
+
+	  /*!
+	   * \brief sendData
+	   *               send data to this virtual peer, and takes memory ownership (deletes the item)
+	   * \param item           item to send
+	   * \param virtual_peer   destination virtual peer
+	   * \return
+	   *               true if succeeded.
+	   */
+      bool sendTunnelData(uint16_t service_id,unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ;
+
+	  /*!
+	   * \brief receiveData
+	   *                 returns the next piece of data received, and the virtual GXS peer that sended it.
+	   * \param data              memory check containing the data. Memory ownership belongs to the client.
+	   * \param data_len          length of memory chunk
+	   * \param virtual_peer      peer who sent the data
+	   * \return
+	   *                          true if something is returned. If not, data is set to NULL, data_len to 0.
+	   */
+      bool receiveTunnelData(uint16_t service_id, unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) ;
+
+	  /*!
+	   * \brief isDistantPeer
+	   *                 returns wether the peer is in the list of available distant peers or not
+	   * \param group_id          returned by the service to indicate which group this peer id is designed for.
+	   * \return    true if the peer is a distant GXS peer.
+	   */
+
+	  bool isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer,RsGxsGroupId& group_id) ;
+
+	  /*!
+	   * \brief dumps all information about monitored groups.
+	   */
+	  void dump() const;
+
+      /*!
+       * \brief connectToTurtleRouter
+       * 			Should be called after allocating a RsGxsNetTunnelService
+       * \param tr turtle router object
+       */
+	  virtual void connectToTurtleRouter(p3turtle *tr) ;
+
+      TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
+      TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
+
+      /*!
+       * \brief receiveSearchRequest
+       * 			See RsTurtleClientService::@
+       */
+	  virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
+	  virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
+
+	  // Overloaded from RsTickingThread
+
+	  void data_tick() ;
+
+	  // Overloads p3Config
+
+	  RsSerialiser *setupSerialiser();
+	  bool saveList(bool& cleanup, std::list<RsItem*>& save);
+	  bool loadList(std::list<RsItem *> &load);
+
+      // Overloads RsGxsDistSync
+
+	  void getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,             // groups on the client and server side
+	  		std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,     // current virtual peers, which group they provide, and how to talk to them through turtle
+            std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
+		    Bias20Bytes& bias) const;
+
+protected:
+	  // interaction with turtle router
+
+	  virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
+	  virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+	  void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
+	  void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
+	  const Bias20Bytes& locked_randomBias() ;
+
+	  p3turtle 	*mTurtle ;
+
+	  static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ;
+	  static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE        = 32 ;
+
+	  mutable Bias20Bytes mRandomBias ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time.
+private:
+	  void autowash() ;
+	  void sendKeepAlivePackets() ;
+	  void handleIncoming(RsGxsNetTunnelItem *item) ;
+	  void flush_pending_items();
+
+	  std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ;	// groups on the client and server side
+
+	  std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo> mVirtualPeers ;	// current virtual peers, which group they provide, and how to talk to them through turtle
+	  std::map<RsFileHash, RsGxsGroupId>                                   mHandledHashes ; // hashes asked to turtle. Used to answer tunnel requests
+	  std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>           mTurtle2GxsPeer ; // convertion table to find GXS peer id from turtle
+
+	  std::list<std::pair<TurtleVirtualPeerId,RsTurtleGenericDataItem*> >  mPendingTurtleItems ; // items that need to be sent off-turtle Mutex.
+
+	  std::map<uint16_t, std::list<std::pair<RsGxsNetTunnelVirtualPeerId,RsTlvBinaryData *> > > mIncomingData; // list of incoming data items
+
+      std::map<uint16_t,RsNetworkExchangeService *> mSearchableServices ;
+
+	  /*!
+	   * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to
+	   * 		hide the real group id.
+	   */
+
+	  RsFileHash calculateGroupHash(const RsGxsGroupId&group_id) const ;
+
+	  /*!
+	   * \brief makeVirtualPeerIdForGroup creates a virtual peer id that can be used and that will be constant accross time, whatever the
+	   * 		tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently.
+	   */
+
+	  RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId(const RsGxsGroupId& group_id) ;
+
+	  static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ;
+
+	  mutable RsMutex mGxsNetTunnelMtx;
+
+	  friend class RsGxsTunnelRandomBiasItem ;
+	  friend class StoreHere ;
+
+	  time_t mLastKeepAlive ;
+	  time_t mLastAutoWash ;
+	  time_t mLastDump ;
+};
+

libretroshare/src/gxs/rsnxs.h

@@ -36,6 +36,7 @@
 #include "services/p3service.h"
 #include "retroshare/rsreputations.h"
 #include "retroshare/rsidentity.h"
+#include "retroshare/rsturtle.h"
 #include "rsgds.h"
 
 /*!
@@ -58,6 +59,7 @@
  *  2 transfers only between group
  *   - the also group matrix settings which is by default everyone can transfer to each other
  */
+
 class RsNetworkExchangeService
 {
 public:
@@ -65,6 +67,7 @@ public:
 	RsNetworkExchangeService(){ return;}
     virtual ~RsNetworkExchangeService() {}
 
+    virtual uint16_t serviceType() const =0;
     /*!
      * Use this to set how far back synchronisation of messages should take place
      * @param age in seconds the max age a sync/store item can to be allowed in a synchronisation
@@ -81,6 +84,65 @@ public:
     virtual uint32_t getDefaultSyncAge() =0;
     virtual uint32_t getDefaultKeepAge() =0;
 
+    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    ///                                          DISTANT SEARCH FUNCTIONS                                           ///
+    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    /*!
+     * \brief turtleGroupRequest
+     * 			Requests a particular group meta data. The request protects the group ID.
+     * \param group_id
+     * \return
+     * 			returns the turtle request ID that might be associated to some results.
+     */
+    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
+
+    /*!
+     * \brief turtleSearchRequest
+     * 			Uses distant search to match the substring to the group meta data.
+     * \param match_string
+     * \return
+     * 			returns the turtle request ID that might be associated to some results.
+     */
+    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
+
+    /*!
+     * \brief receiveTurtleSearchResults
+     * 			Called by turtle (through RsGxsNetTunnel) when new results are received
+     * \param req			Turtle search request ID associated with this result
+     * \param group_infos	Group summary information for the groups returned by the search
+     */
+    virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
+
+    /*!
+     * \brief receiveTurtleSearchResults
+     * 			Called by turtle (through RsGxsNetTunnel) when new data is received
+     * \param req			        Turtle search request ID associated with this result
+     * \param encrypted_group_data  Group data
+     */
+	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
+
+    /*!
+     * \brief retrieveTurtleSearchResults
+     * 			To be used to retrieve the search results that have been notified (or not)
+     * \param req			request that match the results to retrieve
+     * \param group_infos	results to retrieve.
+     * \return
+     * 			false when the request is unknown.
+     */
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos)=0;
+    /*!
+     * \brief getDistantSearchResults
+     * \param id
+     * \param group_infos
+     * \return
+     */
+    virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0;
+
+    virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
+	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
+
     /*!
      * Initiates a search through the network
      * This returns messages which contains the search terms set in RsGxsSearch
@@ -99,6 +161,9 @@ public:
      */
     //virtual int searchGrps(RsGxsSearch* search, uint8_t hops = 1, bool retrieve = 0) = 0;
 
+    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    ///                                           DATA ACCESS FUNCTIONS                                             ///
+    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
     /*!
      * pauses synchronisation of subscribed groups and request for group id
@@ -166,6 +231,14 @@ public:
      */
     virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) =0;
 
+    /*!
+     * \brief isDistantPeer
+     * \param pid		peer that is a virtual peer provided by GxsNetTunnel
+     * \return
+     * 					true if the peer exists (adn therefore is online)
+     */
+    virtual bool isDistantPeer(const RsPeerId& pid)=0;
+
     /*!
      * \brief removeGroups
      * 			Removes time stamp information from the list of groups. This allows to re-sync them if suppliers are present.

libretroshare/src/gxs/rsnxsobserver.h

@@ -29,6 +29,7 @@
 #include <set>
 #include "rsitems/rsnxsitems.h"
 
+typedef uint32_t TurtleRequestId ;
 
 class RsNxsObserver
 {
@@ -42,12 +43,22 @@ public:
     /*!
      * @param messages messages are deleted after function returns
      */
-    virtual void notifyNewMessages(std::vector<RsNxsMsg*>& messages) = 0;
+    virtual void receiveNewMessages(std::vector<RsNxsMsg*>& messages) = 0;
 
     /*!
      * @param groups groups are deleted after function returns
      */
-    virtual void notifyNewGroups(std::vector<RsNxsGrp*>& groups) = 0;
+    virtual void receiveNewGroups(std::vector<RsNxsGrp*>& groups) = 0;
+
+    /*!
+     * \brief receiveDistantSearchResults
+     * 				Called when new distant search result arrive.
+     * \param grpId
+     */
+    virtual void receiveDistantSearchResults(TurtleRequestId /*id*/,const RsGxsGroupId& /*grpId*/)
+    {
+        std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
+    }
 
     /*!
      * @param grpId group id

libretroshare/src/gxstrans/p3gxstrans.cc

@@ -839,7 +839,7 @@ bool p3GxsTrans::dispatchDecryptedMail( const RsGxsId& authorId,
 #endif
 
 	std::vector<RsNxsMsg*> rcct; rcct.push_back(receipt);
-	RsGenExchange::notifyNewMessages(rcct);
+	RsGenExchange::receiveNewMessages(rcct);
 
 	GxsTransClient* recipientService = NULL;
 	{

libretroshare/src/gxstunnel/p3gxstunnel.cc

@@ -685,7 +685,7 @@ void p3GxsTunnelService::removeVirtualPeer(const TurtleFileHash& hash,const Turt
     }
 }
 
-void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,const RsFileHash& hash, const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction)
+void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash& hash, const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction)
 {
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "GxsTunnelService::receiveTurtleData(): Received turtle data. " << std::endl;
@@ -697,7 +697,7 @@ void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,cons
     (void) direction;
 #endif
 
-    RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
+    const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
 
     if(item == NULL)
     {

libretroshare/src/gxstunnel/p3gxstunnel.h

@@ -95,7 +95,7 @@
 //		 by a mix between our own GXS id and the GXS id we're talking to. That is what the TunnelVirtualPeer is.
 //
 //      
-//	RequestTunnel(source_own_id,destination_id)                                                                              -
+//	RequestTunnel(source_own_id,destination_id)                                                                                  -
 //                            |                                                                                                  |
 //                            +---------------------------> p3Turtle::monitorTunnels(  hash(destination_id)   )                  |
 //                                                                            |                                                  |
@@ -127,6 +127,8 @@ public:
     explicit p3GxsTunnelService(RsGixs *pids) ;
     virtual void connectToTurtleRouter(p3turtle *) ;
 
+    uint16_t serviceId() const { return RS_SERVICE_TYPE_GXS_TUNNEL ; }
+
     // Creates the invite if the public key of the distant peer is available.
     // Om success, stores the invite in the map above, so that we can respond to tunnel requests.
     //
@@ -211,7 +213,7 @@ private:
     // Overloaded from RsTurtleClientService
 
     virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
-    virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
+    virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
     void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
     void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
     

libretroshare/src/libretroshare.pro

@@ -137,6 +137,7 @@ PUBLIC_HEADERS =	retroshare/rsdisc.h \
 					retroshare/rsconfig.h \
 					retroshare/rsversion.h \
 					retroshare/rsservicecontrol.h \
+					retroshare/rsgxsdistsync.h 
 
 HEADERS += plugins/pluginmanager.h \
 		plugins/dlfcn_win32.h \
@@ -351,7 +352,8 @@ HEADERS +=	ft/ftchunkmap.h \
 			ft/ftturtlefiletransferitem.h 
 
 HEADERS += crypto/chacha20.h \
-				crypto/hashstream.h
+				crypto/hashstream.h \
+				crypto/rscrypto.h
 
 HEADERS += directory_updater.h \
 				directory_list.h \
@@ -519,7 +521,8 @@ SOURCES +=	ft/ftchunkmap.cc \
             ft/ftturtlefiletransferitem.cc
 
 SOURCES += crypto/chacha20.cpp \
-			  crypto/hashstream.cc
+			  crypto/hashstream.cc \
+			  crypto/rscrypto.cpp
 
 SOURCES += chat/distantchat.cc \
 			  chat/p3chatservice.cc \
@@ -707,6 +710,7 @@ HEADERS += rsitems/rsnxsitems.h \
 	gxs/rsgxs.h \
 	gxs/rsdataservice.h \
 	gxs/rsgxsnetservice.h \
+	gxs/rsgxsnettunnel.h \
 	retroshare/rsgxsflags.h \
 	retroshare/rsgxsifacetypes.h \
 	gxs/rsgenexchange.h \
@@ -732,6 +736,7 @@ SOURCES += rsitems/rsnxsitems.cc \
 	gxs/rsdataservice.cc \
 	gxs/rsgenexchange.cc \
 	gxs/rsgxsnetservice.cc \
+	gxs/rsgxsnettunnel.cc \
 	gxs/rsgxsdata.cc \
 	rsitems/rsgxsitems.cc \
 	gxs/rsgxsdataaccess.cc \

libretroshare/src/pqi/p3cfgmgr.cc

@@ -65,7 +65,7 @@ void	p3ConfigMgr::tick()
 
 #ifdef CONFIG_DEBUG
 			std::cerr << "p3ConfigMgr::tick() Config Changed - Element: ";
-			std::cerr << it->first;
+			std::cerr << *it;
 			std::cerr << std::endl;
 #endif
 
@@ -111,7 +111,7 @@ void p3ConfigMgr::saveConfig()
 		{
 #ifdef CONFIG_DEBUG
 			std::cerr << "p3ConfigMgr::globalSaveConfig() Saving Element: ";
-			std::cerr << it->first;
+			std::cerr << *it;
 			std::cerr << std::endl;
 #endif
 			ok &= (*it)->saveConfiguration();
@@ -137,7 +137,7 @@ void p3ConfigMgr::loadConfig()
 	{
 #ifdef CONFIG_DEBUG
 		std::cerr << "p3ConfigMgr::loadConfig() Element: ";
-		std::cerr << cit->first <<"Dummy Hash: " << dummyHash;
+		std::cerr << *cit <<" Dummy Hash: " << dummyHash;
 		std::cerr << std::endl;
 #endif
 

libretroshare/src/pqi/p3notify.cc

@@ -231,7 +231,9 @@ void p3Notify::notifyChatLobbyTimeShift     (int                time_shift)
 void p3Notify::notifyCustomState            (const std::string& peer_id   , const std::string&               status_string )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState       (peer_id,status_string) ; }
 void p3Notify::notifyHashingInfo            (uint32_t           type      , const std::string&               fileinfo      )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo       (type,fileinfo) ; } 
 void p3Notify::notifyTurtleSearchResult     (uint32_t           search_id , const std::list<TurtleFileInfo>& files         )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; } 
-void p3Notify::notifyPeerHasNewAvatar       (std::string        peer_id   )                                                     { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; } 
+#warning MISSING CODE HERE
+//void p3Notify::notifyTurtleSearchResult     (uint32_t           search_id , const std::list<TurtleGxsInfo>&  groups        )    { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; }
+void p3Notify::notifyPeerHasNewAvatar       (std::string        peer_id   )                                                     { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; }
 void p3Notify::notifyOwnAvatarChanged       ()                                                                                  { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnAvatarChanged() ; } 
 void p3Notify::notifyOwnStatusMessageChanged()                                                                                  { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnStatusMessageChanged() ; } 
 void p3Notify::notifyDiskFull               (uint32_t           location  , uint32_t                         size_limit_in_MB ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyDiskFull          (location,size_limit_in_MB) ; }

libretroshare/src/pqi/p3notify.h

@@ -106,6 +106,8 @@ class p3Notify: public RsNotify
 		void notifyCustomState                (const std::string& /* peer_id   */, const std::string&               /* status_string */) ;
 		void notifyHashingInfo                (uint32_t           /* type      */, const std::string&               /* fileinfo      */) ;
 		void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleFileInfo>& /* files         */) ;
+#warning MISSING CODE HERE
+//		void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) ;
 		void notifyPeerHasNewAvatar           (std::string        /* peer_id   */) ;
 		void notifyOwnAvatarChanged           () ;
 		void notifyOwnStatusMessageChanged    () ;

libretroshare/src/pqi/p3peermgr.cc

@@ -89,11 +89,11 @@ static const std::string kConfigKeyProxyServerIpAddrTor = "PROXY_SERVER_IPADDR";
 static const std::string kConfigKeyProxyServerPortTor = "PROXY_SERVER_PORT";
 static const std::string kConfigKeyProxyServerIpAddrI2P = "PROXY_SERVER_IPADDR_I2P";
 static const std::string kConfigKeyProxyServerPortI2P = "PROXY_SERVER_PORT_I2P";
-	
+
 void  printConnectState(std::ostream &out, peerState &peer);
 
 peerState::peerState()
-	:netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0), 
+	:netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0),
 	 hiddenNode(false), hiddenPort(0), hiddenType(RS_HIDDEN_TYPE_NONE)
 {
         sockaddr_storage_clear(localaddr);
@@ -107,13 +107,13 @@ std::string textPeerConnectState(peerState &state)
 	std::string out = "Id: " + state.id.toStdString() + "\n";
 	rs_sprintf_append(out, "NetMode: %lu\n", state.netMode);
 	rs_sprintf_append(out, "VisState: Disc: %u Dht: %u\n", state.vs_disc, state.vs_dht);
-	
+
 	out += "laddr: ";
 	out += sockaddr_storage_tostring(state.localaddr);
 	out += "\neaddr: ";
 	out += sockaddr_storage_tostring(state.serveraddr);
 	out += "\n";
-	
+
 	return out;
 }
 
@@ -136,7 +136,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_
 		mOwnState.netMode = RS_NET_MODE_UPNP; // Default to UPNP.
 		mOwnState.vs_disc = RS_VS_DISC_FULL;
 		mOwnState.vs_dht = RS_VS_DHT_FULL;
-	
+
 		// setup default ProxyServerAddress.
 		// Tor
 		sockaddr_storage_clear(mProxyServerAddressTor);
@@ -154,7 +154,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_
 		mProxyServerStatusTor = RS_NET_PROXY_STATUS_UNKNOWN ;
 		mProxyServerStatusI2P = RS_NET_PROXY_STATUS_UNKNOWN;
 	}
-	
+
 #ifdef PEER_DEBUG
 	std::cerr << "p3PeerMgr() Startup" << std::endl;
 #endif
@@ -222,8 +222,8 @@ bool p3PeerMgrIMPL::forceHiddenNode()
 	struct sockaddr_storage loopback;
 	sockaddr_storage_clear(loopback);
 	sockaddr_storage_ipv4_aton(loopback, "127.0.0.1");
-	uint16_t port = sockaddr_storage_port(mOwnState.localaddr); 
+	uint16_t port = sockaddr_storage_port(mOwnState.localaddr);
-	sockaddr_storage_ipv4_setport(loopback, port); 
+	sockaddr_storage_ipv4_setport(loopback, port);
 
 	setLocalAddress(AuthSSL::getAuthSSL()->OwnId(), loopback);
 
@@ -254,7 +254,7 @@ bool p3PeerMgrIMPL::setOwnNetworkMode(uint32_t netMode)
 			changed = true;
 		}
 	}
-	
+
 	// Pass on Flags to NetMgr.
 	mNetMgr->setNetworkMode((netMode & RS_NET_MODE_ACTUAL));
 	return changed;
@@ -267,7 +267,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
 		RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
 
 		std::string out;
-		rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u", 
+		rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u",
 				   mOwnState.vs_disc, mOwnState.vs_dht, vs_disc, vs_dht);
 		rslog(RSL_WARNING, p3peermgrzone, out);
 
@@ -275,7 +275,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
 		std::cerr << out.c_str() << std::endl;
 #endif
 
-		if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht) 
+		if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht)
 		{
 			mOwnState.vs_disc = vs_disc;
 			mOwnState.vs_dht = vs_dht;
@@ -283,7 +283,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
 			IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
 		}
 	}
-	
+
 	// Pass on Flags to NetMgr.
 	mNetMgr->setVisState(vs_disc, vs_dht);
 
@@ -575,7 +575,7 @@ bool p3PeerMgrIMPL::setHiddenDomainPort(const RsPeerId &ssl_id, const std::strin
 
 	IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
 
-	if (ssl_id == AuthSSL::getAuthSSL()->OwnId()) 
+	if (ssl_id == AuthSSL::getAuthSSL()->OwnId())
 	{
 		mOwnState.hiddenNode = true;
 		mOwnState.hiddenDomain = domain;
@@ -917,7 +917,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
 		RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
 
 
-		if (id == AuthSSL::getAuthSSL()->OwnId()) 
+		if (id == AuthSSL::getAuthSSL()->OwnId())
 		{
 #ifdef PEER_DEBUG
 			std::cerr << "p3PeerMgrIMPL::addFriend() cannot add own id as a friend." << std::endl;
@@ -973,7 +973,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
 			/* setup connectivity parameters */
 			it->second.vs_disc = vs_disc;
 			it->second.vs_dht = vs_dht;
-			
+
 			it->second.netMode  = netMode;
 			it->second.lastcontact = lastContact;
 
@@ -995,7 +995,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
 			pstate.id = id;
 			pstate.gpg_id = gpg_id;
 			pstate.name = AuthGPG::getAuthGPG()->getGPGName(gpg_id);
-			
+
 			pstate.vs_disc = vs_disc;
 			pstate.vs_dht = vs_dht;
 			pstate.netMode = netMode;
@@ -1018,7 +1018,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
 		mLinkMgr->addFriend(id, vs_dht != RS_VS_DHT_OFF);
 	}
 
-	service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions. 
+	service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions.
 #ifdef RS_CHATSERVER //Defined by chatserver
 	setServicePermissionFlags(gpg_id,RS_NODE_PERM_NONE) ;
 #else
@@ -1029,7 +1029,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
 	printPeerLists(std::cerr);
 	mLinkMgr->printPeerLists(std::cerr);
 #endif
-	
+
 	return true;
 }
 
@@ -1071,10 +1071,10 @@ bool p3PeerMgrIMPL::removeFriend(const RsPgpId &id)
 		}
 
 		for(std::list<RsPeerId>::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit)
-			if (mFriendList.end() != (it = mFriendList.find(*rit))) 
+			if (mFriendList.end() != (it = mFriendList.find(*rit)))
 				mFriendList.erase(it);
 
-		std::map<RsPgpId,ServicePermissionFlags>::iterator it2 = mFriendsPermissionFlags.find(id) ; 
+		std::map<RsPgpId,ServicePermissionFlags>::iterator it2 = mFriendsPermissionFlags.find(id) ;
 
 		if(it2 != mFriendsPermissionFlags.end())
 			mFriendsPermissionFlags.erase(it2);
@@ -1146,13 +1146,13 @@ bool p3PeerMgrIMPL::removeFriend(const RsPeerId &id, bool removePgpId)
 		}
 
 		for(std::list<RsPeerId>::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit)
-			if (mFriendList.end() != (it = mFriendList.find(*rit))) 
+			if (mFriendList.end() != (it = mFriendList.find(*rit)))
 				mFriendList.erase(it);
 
 		std::map<RsPgpId,ServicePermissionFlags>::iterator it2 ;
 
 		for(std::list<RsPgpId>::iterator rit = pgpid_toRemove.begin(); rit != pgpid_toRemove.end(); ++rit)
-			if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit))) 
+			if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit)))
 				mFriendsPermissionFlags.erase(it2);
 
 #ifdef PEER_DEBUG
@@ -1432,7 +1432,7 @@ bool p3PeerMgrIMPL::setLocalAddress( const RsPeerId &id,
 		if (changed)
 		{
 			IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
-			
+
 			mNetMgr->setLocalAddress(addr);
 			mLinkMgr->setLocalAddress(addr);
 		}
@@ -1499,9 +1499,9 @@ bool p3PeerMgrIMPL::setExtAddress( const RsPeerId &id,
 				changed = true;
 			}
 		}
-		
+
 		mNetMgr->setExtAddress(addr);
-		
+
 		return changed;
 	}
 
@@ -1584,12 +1584,16 @@ bool p3PeerMgrIMPL::setDynDNS(const RsPeerId &id, const std::string &dyndns)
     return changed;
 }
 
+namespace pqi {
+
 struct ZeroedInt
 {
     ZeroedInt() { n=0 ;}
     int n ;
 };
 
+}
+
 bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, const sockaddr_storage &addr)
 {
     // The algorithm is the following:
@@ -1617,10 +1621,10 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
     // Update a list of own IPs:
     //	- remove old values for that same peer
     //	- remove values for non connected peers
-    
+
     {
 	    RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
-            
+
 	    mReportedOwnAddresses[from] = addr_filtered ;
 
 	    for(std::map<RsPeerId,sockaddr_storage>::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();)
@@ -1641,8 +1645,8 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
 
 	    std::cerr << "p3PeerMgr::  Current external address is calculated to be: " << sockaddr_storage_iptostring(current_best_ext_address_guess) << " (simultaneously reported by " << count << " peers)." << std::endl;
     }
-       
+
-    // now current 
+    // now current
 
     sockaddr_storage own_addr ;
 
@@ -1665,7 +1669,7 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
 
         RsServer::notify()->AddFeedItem(RS_FEED_ITEM_SEC_IP_WRONG_EXTERNAL_IP_REPORTED, from.toStdString(), sockaddr_storage_iptostring(own_addr), sockaddr_storage_iptostring(addr));
     }
-    
+
     // we could also sweep over all connected friends and see if some report a different address.
 
     return true ;
@@ -1673,46 +1677,46 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
 
 bool p3PeerMgrIMPL::locked_computeCurrentBestOwnExtAddressCandidate(sockaddr_storage& addr, uint32_t& count)
 {
-    std::map<sockaddr_storage,ZeroedInt> addr_counts ;
+    std::map<sockaddr_storage, pqi::ZeroedInt> addr_counts ;
-    
+
     for(std::map<RsPeerId,sockaddr_storage>::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();++it)
 	    ++addr_counts[it->second].n ;
 
 #ifdef PEER_DEBUG
     std::cerr << "Current ext addr statistics:" << std::endl;
 #endif
-    
+
     count = 0 ;
-    
+
-    for(std::map<sockaddr_storage,ZeroedInt>::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it)
+    for(std::map<sockaddr_storage, pqi::ZeroedInt>::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it)
     {
         if(uint32_t(it->second.n) > count)
         {
             addr = it->first ;
             count = it->second.n ;
         }
-        
+
 #ifdef PEER_DEBUG
         std::cerr << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl;
 #endif
     }
-    
+
     return true ;
 }
- 
+
 bool p3PeerMgrIMPL::getExtAddressReportedByFriends(sockaddr_storage &addr, uint8_t& /*isstable*/)
 {
         RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
-        
+
         uint32_t count ;
-        
+
         locked_computeCurrentBestOwnExtAddressCandidate(addr,count) ;
-        
+
 #ifdef PEER_DEBUG
         std::cerr << "Estimation count = " << count << ". Trusted? = " << (count>=2) << std::endl;
 #endif
-        
+
-        return count >= 2 ;// 2 is not conservative enough. 3 should be probably better.	
+        return count >= 2 ;// 2 is not conservative enough. 3 should be probably better.
 }
 
 static bool cleanIpList(std::list<pqiIpAddress>& lst,const RsPeerId& pid,p3LinkMgr *link_mgr)
@@ -1765,7 +1769,7 @@ bool    p3PeerMgrIMPL::updateAddressList(const RsPeerId& id, const pqiIpAddrSet
 	RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
 
 	/* check if it is our own ip */
-	if (id == getOwnId()) 
+	if (id == getOwnId())
 	{
         mOwnState.ipAddrs.updateAddrs(clean_set);
 		return true;
@@ -1811,11 +1815,11 @@ bool    p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre
 #ifdef PEER_DEBUG
 	std::cerr << "p3PeerMgrIMPL::updateCurrentAddress() called for id : " << id << std::endl;
 #endif
-	
+
 	RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
-	
+
 	/* cannot be own id */
-	
+
 	/* check if it is a friend */
 	std::map<RsPeerId, peerState>::iterator it;
 	if (mFriendList.end() == (it = mFriendList.find(id)))
@@ -1846,23 +1850,23 @@ bool    p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre
 	std::cerr << addrstr;
 	std::cerr << std::endl;
 #endif
-	
+
 	IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
-	
+
 	return true;
 }
-	
+
 
 bool    p3PeerMgrIMPL::updateLastContact(const RsPeerId& id)
 {
 #ifdef PEER_DEBUG
 	std::cerr << "p3PeerMgrIMPL::updateLastContact() called for id : " << id << std::endl;
 #endif
-	
+
 	RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
-	
+
 	/* cannot be own id */
-	
+
 	/* check if it is a friend */
 	std::map<RsPeerId, peerState>::iterator it;
 	if (mFriendList.end() == (it = mFriendList.find(id)))
@@ -1877,7 +1881,7 @@ bool    p3PeerMgrIMPL::updateLastContact(const RsPeerId& id)
 	it->second.lastcontact = time(NULL);
 
 	IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
-	
+
 	return true;
 }
 
@@ -2061,7 +2065,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 	getProxyServerAddress(RS_HIDDEN_TYPE_TOR, proxy_addr_tor);
 	getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy_addr_i2p);
 
-	mPeerMtx.lock(); /****** MUTEX LOCKED *******/ 
+	mPeerMtx.lock(); /****** MUTEX LOCKED *******/
 
 	RsPeerNetItem *item = new RsPeerNetItem();
 	item->clear();
@@ -2088,14 +2092,14 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 
 	item->vs_disc = mOwnState.vs_disc;
 	item->vs_dht = mOwnState.vs_dht;
-	
+
 	item->lastContact = mOwnState.lastcontact;
 
 	item->localAddrV4.addr = mOwnState.localaddr;
 	item->extAddrV4.addr = mOwnState.serveraddr;
 	sockaddr_storage_clear(item->localAddrV6.addr);
 	sockaddr_storage_clear(item->extAddrV6.addr);
-	
+
 	item->dyndns = mOwnState.dyndns;
 	mOwnState.ipAddrs.mLocal.loadTlv(item->localAddrList);
 	mOwnState.ipAddrs.mExt.loadTlv(item->extAddrList);
@@ -2125,20 +2129,20 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 		item->vs_dht = (it->second).vs_dht;
 
 		item->lastContact = (it->second).lastcontact;
-		
+
 		item->localAddrV4.addr = (it->second).localaddr;
 		item->extAddrV4.addr = (it->second).serveraddr;
 		sockaddr_storage_clear(item->localAddrV6.addr);
 		sockaddr_storage_clear(item->extAddrV6.addr);
-		
+
-		
+
 		item->dyndns = (it->second).dyndns;
 		(it->second).ipAddrs.mLocal.loadTlv(item->localAddrList);
 		(it->second).ipAddrs.mExt.loadTlv(item->extAddrList);
 
 		item->domain_addr = (it->second).hiddenDomain;
 		item->domain_port = (it->second).hiddenPort;
-        
+
 		saveData.push_back(item);
 #ifdef PEER_DEBUG
 		std::cerr << "p3PeerMgrIMPL::saveList() Peer Config Item:" << std::endl;
@@ -2150,7 +2154,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 	RsPeerBandwidthLimitsItem *pblitem = new RsPeerBandwidthLimitsItem ;
     	pblitem->peers = mPeerBandwidthLimits ;
 	saveData.push_back(pblitem) ;
-    
+
 	RsPeerServicePermissionItem *sitem = new RsPeerServicePermissionItem ;
 
 	for(std::map<RsPgpId,ServicePermissionFlags>::const_iterator it(mFriendsPermissionFlags.begin());it!=mFriendsPermissionFlags.end();++it)
@@ -2158,11 +2162,11 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 		sitem->pgp_ids.push_back(it->first) ;
 		sitem->service_flags.push_back(it->second) ;
 	}
-    
+
 	saveData.push_back(sitem) ;
 
 	// Now save config for network digging strategies
-	
+
 	RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet ;
 
 	RsTlvKeyValue kv;
@@ -2199,7 +2203,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 	kv.key = kConfigKeyProxyServerPortI2P;
 	kv.value = sockaddr_storage_porttostring(proxy_addr_i2p);
 	vitem->tlvkvs.pairs.push_back(kv) ;
-	
+
 	saveData.push_back(vitem);
 
 	/* save groups */
@@ -2213,7 +2217,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 	return true;
 }
 
-bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn) 
+bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn)
 {
 	RsPgpId pgp_id ;
 
@@ -2235,7 +2239,7 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& ma
 	return getMaxRates(pgp_id,maxUp,maxDn) ;
 }
 
-bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn) 
+bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn)
 {
     RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
 
@@ -2254,25 +2258,25 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& max
 	    return false ;
     }
 }
-bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn) 
+bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn)
 {
 	RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
 
         PeerBandwidthLimits& p(mPeerBandwidthLimits[pid]) ;
-        
+
         if(maxUp == p.max_up_rate_kbs && maxDn == p.max_dl_rate_kbs)
             return true ;
-        
+
         std::cerr << "Updating max rates for peer " << pid << " to " << maxUp << " kB/s (up), " << maxDn << " kB/s (dn)" << std::endl;
-        
+
         p.max_up_rate_kbs = maxUp ;
         p.max_dl_rate_kbs = maxDn ;
-        
+
         IndicateConfigChanged();
-        
+
         return true ;
 }
-        
+
 void    p3PeerMgrIMPL::saveDone()
 {
 	/* clean up the save List */
@@ -2396,7 +2400,7 @@ bool  p3PeerMgrIMPL::loadList(std::list<RsItem *>& load)
 #ifdef PEER_DEBUG
 				    std::cerr << "setting use_extr_addr_finder to " << useExtAddrFinder << std::endl ;
 #endif
-			    } 
+			    }
 			    // Tor
 			    else if (kit->key == kConfigKeyProxyServerIpAddrTor)
 			    {
@@ -2859,13 +2863,13 @@ bool p3PeerMgrIMPL::removeAllFriendLocations(const RsPgpId &gpgid)
 	{
 		return false;
 	}
-	
+
 	std::list<RsPeerId>::iterator it;
 	for(it = sslIds.begin(); it != sslIds.end(); ++it)
 	{
 		removeFriend(*it, true);
 	}
-	
+
 	return true;
 }
 
@@ -2877,7 +2881,7 @@ bool	p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::list<RsPeerId
 #ifdef PEER_DEBUG
 	std::cerr << "p3PeerMgr::getAssociatedPeers() for id : " << gpg_id << std::endl;
 #endif
-	
+
 	int count = 0;
 	std::map<RsPeerId, peerState>::iterator it;
 	for(it = mFriendList.begin(); it != mFriendList.end(); ++it)
@@ -2890,10 +2894,10 @@ bool	p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::list<RsPeerId
 #ifdef PEER_DEBUG
 			std::cerr << "p3PeerMgr::getAssociatedPeers() found ssl id :  " << it->first << std::endl;
 #endif
-			
+
 		}
 	}
-	
+
 	return (count > 0);
 }
 
@@ -2930,10 +2934,10 @@ bool p3PeerMgrIMPL::removeBannedIps()
     return true ;
 }
 
-// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id 
+// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id
 //  * We are removing the concept of a "DummyId" - There is no need for it.
 //  */
-// 
+//
 // bool isDummyFriend(RsPeerId id)
 // {
 // 	bool ret = (id.substr(0,5) == "dummy");
@@ -2974,7 +2978,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
 
 	{
 		RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
-	
+
         // First put a sensible number in all PGP ids
 
         for(std::list<RsPgpId>::const_iterator it = pgpList.begin(); it != pgpList.end(); ++it)
@@ -2984,7 +2988,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
 		std::cerr << "p3PeerMgr::removeUnusedLocations()" << std::endl;
 #endif
         // Then compute the most recently used location for all PGP ids
-		
+
         for( std::map<RsPeerId, peerState>::iterator it = mFriendList.begin(); it != mFriendList.end(); ++it)
         {
             time_t& bst(mostRecentTime[it->second.gpg_id]) ;
@@ -3010,4 +3014,4 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
 	return true;
 }
 
-	
+

libretroshare/src/retroshare/rsfiles.h

@@ -32,6 +32,7 @@
 #include <string>
 
 #include "rstypes.h"
+#include "rsturtle.h"
 
 class RsFiles;
 extern RsFiles  *rsFiles;
@@ -212,6 +213,8 @@ public:
 		virtual uint32_t getMaxUploadSlotsPerFriend()=0;
 		virtual void setFilePermDirectDL(uint32_t perm)=0;
 		virtual uint32_t filePermDirectDL()=0;
+		virtual TurtleRequestId turtleSearch(const std::string& string_to_match) =0;
+        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) =0;
 
 		/***
 		 * Control of Downloads Priority.

libretroshare/src/retroshare/rsgxschannels.h

@@ -33,8 +33,7 @@
 #include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 #include "retroshare/rsgxscommon.h"
-
+#include "retroshare/rsturtle.h"
-
 
 /* The Main Interface Class - for information about your Peers */
 class RsGxsChannels;
@@ -77,55 +76,49 @@ std::ostream &operator<<(std::ostream &out, const RsGxsChannelPost &post);
 
 class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
 {
-	public:
+public:
 
 	explicit RsGxsChannels(RsGxsIface *gxs)
-	  :RsGxsIfaceHelper(gxs)  {}
+	    :RsGxsIfaceHelper(gxs)  {}
 	virtual ~RsGxsChannels() {}
 
 	/* Specific Service Data */
-virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups) = 0;
+	virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups) = 0;
-virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
+	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
-virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
+	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
-//Not currently used
-//virtual bool getRelatedPosts(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
 
-	/* From RsGxsCommentService */
+    //////////////////////////////////////////////////////////////////////////////
-//virtual bool getCommentData(const uint32_t &token, std::vector<RsGxsComment> &comments) = 0;
+    ///                     Distant synchronisation methods                    ///
-//virtual bool getRelatedComments(const uint32_t &token, std::vector<RsGxsComment> &comments) = 0;
+    //////////////////////////////////////////////////////////////////////////////
-//virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0;
+    ///
-//virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0;
+	virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
+	virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) =0;
+	virtual bool clearDistantSearchResults(TurtleRequestId req)=0;
+	virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0;
 
-        //////////////////////////////////////////////////////////////////////////////
+	//////////////////////////////////////////////////////////////////////////////
-virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
+	virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
 
-virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0;
+	virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0;
-virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0;
+	virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0;
 
-virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0;
+	virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0;
-virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0;
+	virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0;
 
-//virtual void setChannelAutoDownload(uint32_t& token, const RsGxsGroupId& groupId, bool autoDownload) = 0;
+	virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set<RsPeerId>& peers)=0;
-
-//virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
-//virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
-
-//virtual bool groupRestoreKeys(const std::string &groupId);
-    virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set<RsPeerId>& peers)=0;
 
 	// Overloaded subscribe fn.
-virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0;
+	virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0;
 
-virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
+	virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
-virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0;
+	virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0;
-
-virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
-
-        // File Interface
-virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0;
-virtual bool ExtraFileRemove(const RsFileHash &hash) = 0;
 
+	virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
 
+	// File Interface
+	virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0;
+	virtual bool ExtraFileRemove(const RsFileHash &hash) = 0;
 };
 
 

libretroshare/src/retroshare/rsgxsdistsync.h

@@ -0,0 +1,92 @@
+/*
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2018 by Cyril Soler
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#pragma once
+
+#include "retroshare/rsfiles.h"
+#include "retroshare/rsturtle.h"
+
+typedef RsPeerId RsGxsNetTunnelVirtualPeerId ;
+
+struct RsGxsNetTunnelVirtualPeerInfo
+{
+	enum {   RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN   = 0x00,		// unknown status.
+		     RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01,		// tunnel has been established and we're waiting for virtual peer id
+		     RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE    = 0x02		// virtual peer id is known. Data can transfer.
+	     };
+
+	RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; }
+	virtual ~RsGxsNetTunnelVirtualPeerInfo(){}
+
+	uint8_t vpid_status ;					// status of the peer
+	time_t  last_contact ;					// last time some data was sent/recvd
+	uint8_t side ;	                        // client/server
+	uint8_t encryption_master_key[32];
+
+	TurtleVirtualPeerId      turtle_virtual_peer_id ;  // turtle peer to use when sending data to this vpid.
+
+	RsGxsGroupId group_id ;					// group that virtual peer is providing
+	uint16_t service_id ; 					// this is used for checkng consistency of the incoming data
+};
+
+struct RsGxsNetTunnelGroupInfo
+{
+	enum GroupStatus {
+		    RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN            = 0x00,	// unknown status
+		    RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE               = 0x01,	// no virtual peers requested, just waiting
+		    RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE    = 0x02	// some virtual peers are available. Data can be read/written
+	};
+
+	enum GroupPolicy {
+		    RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN            = 0x00,	// nothing has been set
+		    RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE            = 0x01,	// group is available for server side tunnels, but does not explicitely request tunnels
+		    RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE             = 0x02,	// group will only explicitely request tunnels if none available
+		    RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING         = 0x03,	// group explicitely requests tunnels
+    };
+
+	RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {}
+
+	GroupPolicy    group_policy ;
+	GroupStatus    group_status ;
+	time_t         last_contact ;
+	RsFileHash     hash ;
+	uint16_t       service_id ;
+
+	std::set<TurtleVirtualPeerId> virtual_peers ; // list of which virtual peers provide this group. Can me more than 1.
+};
+
+// This class is here to provide statistics about GXS dist sync internals. It
+//
+class RsGxsDistSync
+{
+   public:
+		virtual void getStatistics(
+	        std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,                                 // groups on the client and server side
+	  		std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,    // current virtual peers, which group they provide, and how to talk to them through turtle
+            std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
+		    Bias20Bytes& bias
+		) const =0;
+};
+
+extern RsGxsDistSync *rsGxsDistSync ;
+

libretroshare/src/retroshare/rsgxsiface.h

@@ -32,6 +32,30 @@
 #include "gxs/rsgxsdata.h"
 #include "retroshare/rsgxsifacetypes.h"
 
+/*!
+ * \brief The RsGxsGroupSymmary struct
+ * 			This structure is used to transport group summary information when a GXS service is searched. It contains the group information
+ * 			as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make
+ * 			search responses as light as possible.
+ */
+struct RsGxsGroupSummary
+{
+    RsGxsGroupSummary() : publish_ts(0), number_of_messages(0),last_message_ts(0),sign_flags(0),popularity(0) {}
+
+    RsGxsGroupId group_id ;
+
+    std::string  group_name ;
+    std::string  group_description ;
+    std::string  search_context ;
+    RsGxsId      author_id ;
+    time_t       publish_ts ;
+    uint32_t     number_of_messages ;
+    time_t       last_message_ts ;
+    uint32_t     sign_flags ;
+    uint32_t     popularity ;
+};
+
+
 /*!
  * Stores ids of changed gxs groups and messages. It is used to notify the GUI about changes.
  */
@@ -44,6 +68,7 @@ public:
     std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mMsgsMeta;
     std::list<RsGxsGroupId> mGrps;
     std::list<RsGxsGroupId> mGrpsMeta;
+    std::list<TurtleRequestId> mDistantSearchReqs;
 };
 
 /*!

libretroshare/src/retroshare/rsgxsifacetypes.h

@@ -69,18 +69,18 @@ struct RsGroupMetaData : RsSerializable
 
     RsGxsGroupId mGroupId;
     std::string mGroupName;
-	uint32_t    mGroupFlags;  // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC
+	uint32_t    mGroupFlags;  // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC: diffusion
-    uint32_t    mSignFlags;   // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK.
+    uint32_t    mSignFlags;   // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK, i.e. what signatures are required for parent and child msgs
 
     time_t      mPublishTs; // Mandatory.
-    RsGxsId    mAuthorId;   // Optional.
+    RsGxsId    mAuthorId;   // Author of the group. Left to "000....0" if anonymous
 
     // for circles
-    RsGxsCircleId mCircleId;
+    RsGxsCircleId mCircleId;	// Id of the circle to which the group is restricted
-    uint32_t mCircleType;
+    uint32_t mCircleType;		// combination of CIRCLE_TYPE_{ PUBLIC,EXTERNAL,YOUR_FRIENDS_ONLY,LOCAL,EXT_SELF,YOUR_EYES_ONLY }
 
     // other stuff.
-    uint32_t mAuthenFlags;
+    uint32_t mAuthenFlags;		// Actually not used yet.
     RsGxsGroupId mParentGrpId;
 
     // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.

libretroshare/src/retroshare/rsgxsservice.h

@@ -6,6 +6,7 @@
 #include "retroshare/rstokenservice.h"
 
 struct RsMsgMetaData ;
+typedef uint32_t TurtleRequestId;
 
 typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > GxsMsgMetaMap;
 typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMetaMap;
@@ -18,7 +19,7 @@ typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMe
 struct RsGxsNotify
 {
 	enum NotifyType
-	{ TYPE_PUBLISH, TYPE_RECEIVE, TYPE_PROCESSED, TYPE_PUBLISHKEY };
+	{ TYPE_PUBLISHED, TYPE_RECEIVED_NEW, TYPE_PROCESSED, TYPE_RECEIVED_PUBLISHKEY, TYPE_RECEIVED_DISTANT_SEARCH_RESULTS };
 
 	virtual ~RsGxsNotify() {}
 	virtual NotifyType getType() = 0;
@@ -39,6 +40,17 @@ private:
     bool mMetaChange;
 };
 
+class RsGxsDistantSearchResultChange: public RsGxsNotify
+{
+public:
+    RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& group_id) : mRequestId(id),mGroupId(group_id){}
+
+    NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; }
+
+    TurtleRequestId mRequestId ;
+ 	RsGxsGroupId mGroupId;
+};
+
 /*!
  * Relevant to message changes
  */

libretroshare/src/retroshare/rsids.h

@@ -238,12 +238,14 @@ static const uint32_t RS_GENERIC_ID_GXS_TUNNEL_ID_TYPE       = 0x0010 ;
 static const uint32_t RS_GENERIC_ID_GXS_DISTANT_CHAT_ID_TYPE = 0x0011 ;
 static const uint32_t RS_GENERIC_ID_NODE_GROUP_ID_TYPE       = 0x0012 ;
 static const uint32_t RS_GENERIC_ID_SHA256_ID_TYPE           = 0x0013 ;
+static const uint32_t RS_GENERIC_ID_20_BYTES_UNTYPED         = 0x0014 ;
 
 typedef t_RsGenericIdType<  SSL_ID_SIZE             , false, RS_GENERIC_ID_SSL_ID_TYPE>          SSLIdType ;
 typedef t_RsGenericIdType<  PGP_KEY_ID_SIZE         , true,  RS_GENERIC_ID_PGP_ID_TYPE>          PGPIdType ;
 typedef t_RsGenericIdType<  SHA1_SIZE               , false, RS_GENERIC_ID_SHA1_ID_TYPE>         Sha1CheckSum ;
 typedef t_RsGenericIdType<  SHA256_SIZE             , false, RS_GENERIC_ID_SHA256_ID_TYPE>       Sha256CheckSum ;
 typedef t_RsGenericIdType<  PGP_KEY_FINGERPRINT_SIZE, true,  RS_GENERIC_ID_PGP_FINGERPRINT_TYPE> PGPFingerprintType ;
+typedef t_RsGenericIdType<  SHA1_SIZE               , true,  RS_GENERIC_ID_20_BYTES_UNTYPED>     Bias20Bytes ;
 
 typedef t_RsGenericIdType<  CERT_SIGN_LEN           , false, RS_GENERIC_ID_GXS_GROUP_ID_TYPE   > GXSGroupId ;
 typedef t_RsGenericIdType<  CERT_SIGN_LEN           , false, RS_GENERIC_ID_GXS_ID_TYPE         > GXSId ;

libretroshare/src/retroshare/rsnotify.h

@@ -228,6 +228,8 @@ class NotifyClient
 		virtual void notifyCustomState                (const std::string& /* peer_id   */, const std::string&               /* status_string */) {}
 		virtual void notifyHashingInfo                (uint32_t           /* type      */, const std::string&               /* fileinfo      */) {}
 		virtual void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleFileInfo>& /* files         */) {}
+#warning MISSING CODE HERE
+		// virtual void notifyTurtleSearchResult         (uint32_t           /* search_id */, const std::list<TurtleGxsInfo >& /* groups        */) {}
 		virtual void notifyPeerHasNewAvatar           (std::string        /* peer_id   */) {}
 		virtual void notifyOwnAvatarChanged           () {}
 		virtual void notifyOwnStatusMessageChanged    () {}

libretroshare/src/retroshare/rsturtle.h

@@ -33,7 +33,9 @@
 #include <list>
 #include <vector>
 
+#include "serialiser/rstlvbinary.h"
 #include "retroshare/rstypes.h"
+#include "retroshare/rsgxsifacetypes.h"
 
 namespace RsRegularExpression { class LinearizedExpression ; }
 class RsTurtleClientService ;
@@ -42,6 +44,7 @@ class RsTurtle;
 extern RsTurtle   *rsTurtle ;
 
 typedef uint32_t TurtleRequestId ;
+typedef RsPeerId TurtleVirtualPeerId;
 
 // This is the structure used to send back results of the turtle search 
 // to the notifyBase class, or send info to the GUI.
@@ -52,6 +55,7 @@ struct TurtleFileInfo
 	std::string name ;
 	uint64_t size ;
 };
+
 struct TurtleTunnelRequestDisplayInfo
 {
 	uint32_t request_id ;     // Id of the request
@@ -109,8 +113,7 @@ class RsTurtle
 		// the request id, which will be further used by the gui to store results
 		// as they come back.
 		//
-		virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ;
+        virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) =0;
-        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ;
 
 		// Initiates tunnel handling for the given file hash.  tunnels.  Launches
 		// an exception if an error occurs during the initialization process. The

libretroshare/src/rsitems/rsgxsitems.cc

@@ -6,6 +6,8 @@
  */
 
 
+#include "serialiser/rstypeserializer.h"
+#include "serialiser/rsbaseserial.h"
 #include "rsgxsitems.h"
 #include "gxs/rsgxsdata.h"
 #include <iostream>
@@ -70,4 +72,62 @@ std::ostream &operator<<(std::ostream &out, const RsMsgMetaData &meta)
         return out;
 }
 
+template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i)
+{
+    uint32_t s = 0 ;
+
+    s += 2 ; // service_id
+    s += i.group_id.SIZE_IN_BYTES ;
+    s += GetTlvStringSize(i.name) ;
+
+    return s;
+}
+
+template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i)
+{
+    uint32_t saved_offset = offset ;
+    bool ok = true ;
+
+	ok &= getRawUInt16(data, size, &offset, &i.service_id); 			 // service_id
+	ok &= i.group_id.deserialise(data, size, offset);                    // group_id
+	ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
+
+    if(!ok)
+        offset = saved_offset ;
+
+    return ok;
+}
+
+template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i)
+{
+	uint32_t saved_offset = offset ;
+    bool ok = true ;
+
+	ok &= setRawUInt16(data, size, &offset, i.service_id); 					 // service_id
+	ok &= i.group_id.serialise(data, size, offset);					         // group_id
+	ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name);  // group name
+
+	if(!ok)
+		offset = saved_offset ;
+
+	return ok;
+}
+
+template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i)
+{
+    std::cerr << "  [GXS Info  ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl;
+}
+
+void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process(j,ctx,result,"result") ;
+}
+void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::TlvMemBlock_proxy prox(encrypted_nxs_group_data,encrypted_nxs_group_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"encrypted_nxs_data") ;
+}
+
+RS_TYPE_SERIALIZER_FROM_JSON_NOT_IMPLEMENTED_DEF(TurtleGxsInfo)
+RS_TYPE_SERIALIZER_TO_JSON_NOT_IMPLEMENTED_DEF(TurtleGxsInfo)
 

libretroshare/src/rsitems/rsgxsitems.h

@@ -57,6 +57,82 @@ public:
     RsMsgMetaData meta;
 };
 
+// We should make these items templates or generic classes so that each GXS service will handle them on its own.
+
+static const uint8_t  RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM = 0x20 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM     = 0x21 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM    = 0x22 ;
+static const uint8_t  RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM       = 0x23 ;
+
+class RsGxsTurtleSubStringSearchItem: public RsItem
+{
+	public:
+		RsGxsTurtleSubStringSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM) {}
+        virtual ~RsGxsTurtleSubStringSearchItem() {}
+
+		std::string match_string ;	// string to match
+
+		std::string GetKeywords() { return match_string; }
+        void clear() { match_string.clear() ; }
+
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+class RsGxsTurtleGroupSearchItem: public RsItem
+{
+	public:
+		RsGxsTurtleGroupSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM) {}
+        virtual ~RsGxsTurtleGroupSearchItem() {}
+
+        uint16_t service_id ;		// searvice to search
+		Sha1CheckSum hashed_group_id ;	// the group ID is hashed in order to keep it private.
+
+		std::string GetKeywords() { return std::string("Group request for [hashed] ")+hashed_group_id.toStdString() ; }
+        void clear() { hashed_group_id.clear() ; }
+
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+struct TurtleGxsInfo
+{
+	uint16_t     service_id ;
+	RsGxsGroupId group_id ;
+	RsGxsId      author;
+	std::string  name ;
+	std::string  description ;
+	time_t       last_post ;
+	uint32_t     number_of_posts ;
+};
+
+class RsTurtleGxsSearchResultGroupSummaryItem: public RsItem
+{
+	public:
+        RsTurtleGxsSearchResultGroupSummaryItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM){}
+        virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {}
+
+		std::list<TurtleGxsInfo> result ;
+
+        void clear() { result.clear() ; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+class RsTurtleGxsSearchResultGroupDataItem: public RsItem
+{
+	public:
+        RsTurtleGxsSearchResultGroupDataItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM){}
+        virtual ~RsTurtleGxsSearchResultGroupDataItem() {}
+
+        unsigned char *encrypted_nxs_group_data;	// data is encrypted with group ID. Only the requester, or anyone who already know the group id can decrypt.
+        uint32_t encrypted_nxs_group_data_len ;
+
+        void clear() { free(encrypted_nxs_group_data); encrypted_nxs_group_data=NULL; encrypted_nxs_group_data_len=0; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
 
 
 

libretroshare/src/rsitems/rsgxsupdateitems.cc

@@ -97,6 +97,31 @@ void RsGxsMsgUpdateItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe
     RsTypeSerializer::serial_process(j,ctx,msgUpdateInfos,"msgUpdateInfos");
 }
 
+template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const RsGxsMsgUpdateItem::MsgUpdateInfo& info)
+{
+    bool ok = true ;
+
+    ok = ok && setRawUInt32(data,size,&offset,info.time_stamp);
+    ok = ok && setRawUInt32(data,size,&offset,info.message_count);
+
+    return ok;
+}
+template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, RsGxsMsgUpdateItem::MsgUpdateInfo& info)
+{
+    bool ok = true ;
+
+    ok = ok && getRawUInt32(data,size,&offset,&info.time_stamp);
+    ok = ok && getRawUInt32(data,size,&offset,&info.message_count);
+
+    return ok;
+}
+template<> uint32_t RsTypeSerializer::serial_size(const RsGxsMsgUpdateItem::MsgUpdateInfo& /* info */) { return 8; }
+
+template<> void RsTypeSerializer::print_data(const std::string& name,const RsGxsMsgUpdateItem::MsgUpdateInfo& info)
+{
+    std::cerr << "[MsgUpdateInfo]: " << name << ": " << info.time_stamp << ", " << info.message_count << std::endl;
+}
+
 void RsGxsServerMsgUpdateItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process          (j,ctx,grpId,"grpId");
@@ -109,3 +134,4 @@ void RsGxsGrpConfigItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,msg_send_delay,"msg_send_delay") ;
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,msg_req_delay,"msg_req_delay") ;
 }
+

libretroshare/src/rsitems/rsgxsupdateitems.h

@@ -30,6 +30,7 @@
 
 #include "gxs/rsgxs.h"
 #include "gxs/rsgxsdata.h"
+#include "gxs/rsgxsnettunnel.h"
 #include "serialiser/rstlvidset.h"
 #include "serialiser/rstypeserializer.h"
 #include "serialiser/rsserializable.h"
@@ -41,6 +42,7 @@ const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE             = 0x03;
 const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE      = 0x04;
 const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE      = 0x08;
 const uint8_t RS_PKT_SUBTYPE_GXS_GRP_CONFIG             = 0x09;
+const uint8_t RS_PKT_SUBTYPE_GXS_RANDOM_BIAS            = 0x0a;
 
 class RsGxsNetServiceItem: public RsItem
 {
@@ -188,7 +190,6 @@ public:
 	RsGxsGroupId grpId;
 };
 
-
 class RsGxsUpdateSerialiser : public RsServiceSerializer
 {
 public:

libretroshare/src/rsserver/rsinit.cc

@@ -62,6 +62,7 @@
 #include <fcntl.h>
 
 #include "gxstunnel/p3gxstunnel.h"
+#include "retroshare/rsgxsdistsync.h"
 #include "file_sharing/p3filelists.h"
 
 #define ENABLE_GROUTER
@@ -1279,6 +1280,15 @@ int RsServer::StartupRetroShare()
 
         RsNxsNetMgr* nxsMgr =  new RsNxsNetMgrImpl(serviceCtrl);
 
+        /**** GXS Dist sync service ****/
+
+#ifdef RS_USE_GXS_DISTANT_SYNC
+	RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ;
+    rsGxsDistSync = mGxsNetTunnel ;
+#else
+	RsGxsNetTunnelService *mGxsNetTunnel = NULL ;
+#endif
+
         /**** Identity service ****/
 
         RsGeneralDataService* gxsid_ds = new RsDataService(currGxsDir + "/", "gxsid_db",
@@ -1300,9 +1310,10 @@ int RsServer::StartupRetroShare()
                         RS_SERVICE_GXS_TYPE_GXSID, gxsid_ds, nxsMgr,
 			mGxsIdService, mGxsIdService->getServiceInfo(),
 			mReputations, mGxsCircles,mGxsIdService,
-			pgpAuxUtils,
+			pgpAuxUtils,mGxsNetTunnel,
-            false,false); // don't synchronise group automatic (need explicit group request)
+            false,false,true); // don't synchronise group automatic (need explicit group request)
                         // don't sync messages at all.
+						// allow distsync, so that we can grab GXS id requests for other services
 
         // Normally we wouldn't need this (we do in other service):
         //	mGxsIdService->setNetworkExchangeService(gxsid_ns) ;
@@ -1319,9 +1330,7 @@ int RsServer::StartupRetroShare()
                         RS_SERVICE_GXS_TYPE_GXSCIRCLE, gxscircles_ds, nxsMgr,
                         mGxsCircles, mGxsCircles->getServiceInfo(), 
 			mReputations, mGxsCircles,mGxsIdService,
-			pgpAuxUtils,
+			pgpAuxUtils);
-	            	true,	// synchronise group automatic 
-                    	true); 	// sync messages automatic, since they contain subscription requests.
 
 		mGxsCircles->setNetworkExchangeService(gxscircles_ns) ;
     
@@ -1373,7 +1382,7 @@ int RsServer::StartupRetroShare()
                         RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
                         mGxsForums, mGxsForums->getServiceInfo(),
 			mReputations, mGxsCircles,mGxsIdService,
-			pgpAuxUtils);
+		    pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
 
     mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
 
@@ -1386,10 +1395,10 @@ int RsServer::StartupRetroShare()
 
         // create GXS photo service
         RsGxsNetService* gxschannels_ns = new RsGxsNetService(
-                        RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr,
+		            RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr,
-                        mGxsChannels, mGxsChannels->getServiceInfo(), 
+		            mGxsChannels, mGxsChannels->getServiceInfo(),
-			mReputations, mGxsCircles,mGxsIdService,
+		            mReputations, mGxsCircles,mGxsIdService,
-			pgpAuxUtils);
+		    		pgpAuxUtils,mGxsNetTunnel,true,true,true);
 
     mGxsChannels->setNetworkExchangeService(gxschannels_ns) ;
 
@@ -1444,7 +1453,7 @@ int RsServer::StartupRetroShare()
 	RsGxsNetService* gxstrans_ns = new RsGxsNetService(
 	            RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans,
 	            mGxsTrans->getServiceInfo(), mReputations, mGxsCircles,
-	            mGxsIdService, pgpAuxUtils,true,true,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD);
+	            mGxsIdService, pgpAuxUtils,NULL,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD);
 
 	mGxsTrans->setNetworkExchangeService(gxstrans_ns);
 	pqih->addService(gxstrans_ns, true);
@@ -1477,9 +1486,11 @@ int RsServer::StartupRetroShare()
     pqih -> addService(fdb,true);
     pqih -> addService(ftserver,true);
 
-        mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
+	mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
-        mGxsTunnels->connectToTurtleRouter(tr) ;
+	mGxsTunnels->connectToTurtleRouter(tr) ;
-        rsGxsTunnel = mGxsTunnels;
+	rsGxsTunnel = mGxsTunnels;
+
+	mGxsNetTunnel->connectToTurtleRouter(tr) ;
 
 	rsDisc  = mDisc;
 	rsMsgs  = new p3Msgs(msgSrv, chatSrv);
@@ -1612,48 +1623,51 @@ int RsServer::StartupRetroShare()
 	serviceCtrl->registerServiceMonitor(mBwCtrl, mBwCtrl->getServiceInfo().mServiceType);
 
 	/**************************************************************************/
+    // Turtle search for GXS services
+
+    mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
+
+	/**************************************************************************/
 
 	//mConfigMgr->addConfiguration("ftserver.cfg", ftserver);
 	//
-	mConfigMgr->addConfiguration("gpg_prefs.cfg", AuthGPG::getAuthGPG());
+	mConfigMgr->addConfiguration("gpg_prefs.cfg"   , AuthGPG::getAuthGPG());
-	mConfigMgr->loadConfiguration();
+	mConfigMgr->addConfiguration("gxsnettunnel.cfg", mGxsNetTunnel);
-
+	mConfigMgr->addConfiguration("peers.cfg"       , mPeerMgr);
-	mConfigMgr->addConfiguration("peers.cfg", mPeerMgr);
+	mConfigMgr->addConfiguration("general.cfg"     , mGeneralConfig);
-	mConfigMgr->addConfiguration("general.cfg", mGeneralConfig);
+	mConfigMgr->addConfiguration("msgs.cfg"        , msgSrv);
-	mConfigMgr->addConfiguration("msgs.cfg", msgSrv);
+	mConfigMgr->addConfiguration("chat.cfg"        , chatSrv);
-	mConfigMgr->addConfiguration("chat.cfg", chatSrv);
+	mConfigMgr->addConfiguration("p3History.cfg"   , mHistoryMgr);
-	mConfigMgr->addConfiguration("p3History.cfg", mHistoryMgr);
+	mConfigMgr->addConfiguration("p3Status.cfg"    , mStatusSrv);
-	mConfigMgr->addConfiguration("p3Status.cfg", mStatusSrv);
+	mConfigMgr->addConfiguration("turtle.cfg"      , tr);
-	mConfigMgr->addConfiguration("turtle.cfg", tr);
 #ifndef RETROTOR
-	mConfigMgr->addConfiguration("banlist.cfg", mBanList);
+	mConfigMgr->addConfiguration("banlist.cfg"     , mBanList);
 #endif
 	mConfigMgr->addConfiguration("servicecontrol.cfg", serviceCtrl);
-	mConfigMgr->addConfiguration("reputations.cfg", mReputations);
+	mConfigMgr->addConfiguration("reputations.cfg" , mReputations);
 #ifdef ENABLE_GROUTER
-	mConfigMgr->addConfiguration("grouter.cfg", gr);
+	mConfigMgr->addConfiguration("grouter.cfg"     , gr);
 #endif
 
 #ifdef RS_USE_BITDHT
-	mConfigMgr->addConfiguration("bitdht.cfg", mBitDht);
+	mConfigMgr->addConfiguration("bitdht.cfg"      , mBitDht);
 #endif
 
 #ifdef RS_ENABLE_GXS
 
 #	ifdef RS_GXS_TRANS
 	mConfigMgr->addConfiguration("gxs_trans_ns.cfg", gxstrans_ns);
-	mConfigMgr->addConfiguration("gxs_trans.cfg", mGxsTrans);
+	mConfigMgr->addConfiguration("gxs_trans.cfg"   , mGxsTrans);
 #	endif // RS_GXS_TRANS
 
-	mConfigMgr->addConfiguration("p3identity.cfg", mGxsIdService);
+	mConfigMgr->addConfiguration("p3identity.cfg"  , mGxsIdService);
-
+	mConfigMgr->addConfiguration("identity.cfg"    , gxsid_ns);
-	mConfigMgr->addConfiguration("identity.cfg", gxsid_ns);
+	mConfigMgr->addConfiguration("gxsforums.cfg"   , gxsforums_ns);
-	mConfigMgr->addConfiguration("gxsforums.cfg", gxsforums_ns);
 	mConfigMgr->addConfiguration("gxsforums_srv.cfg", mGxsForums);
-	mConfigMgr->addConfiguration("gxschannels.cfg", gxschannels_ns);
+	mConfigMgr->addConfiguration("gxschannels.cfg" , gxschannels_ns);
 	mConfigMgr->addConfiguration("gxschannels_srv.cfg", mGxsChannels);
-	mConfigMgr->addConfiguration("gxscircles.cfg", gxscircles_ns);
+	mConfigMgr->addConfiguration("gxscircles.cfg"  , gxscircles_ns);
-	mConfigMgr->addConfiguration("posted.cfg", posted_ns);
+	mConfigMgr->addConfiguration("posted.cfg"      , posted_ns);
 #ifdef RS_USE_WIKI
 	mConfigMgr->addConfiguration("wiki.cfg", wiki_ns);
 #endif
@@ -1824,6 +1838,8 @@ int RsServer::StartupRetroShare()
     //rsWire = mWire;
 
 	/*** start up GXS core runner ***/
+
+	startServiceThread(mGxsNetTunnel, "gxs net tunnel");
 	startServiceThread(mGxsIdService, "gxs id");
 	startServiceThread(mGxsCircles, "gxs circle");
 	startServiceThread(mPosted, "gxs posted");

libretroshare/src/serialiser/rsserializer.h

@@ -154,7 +154,11 @@
 #include <string.h>
 #include <iostream>
 #include <string>
+#ifdef HAS_RAPIDJSON
+#include <rapidjson/document.h>
+#else
 #include <rapid_json/document.h>
+#endif // HAS_RAPIDJSON
 
 #include "retroshare/rsflags.h"
 #include "serialiser/rsserial.h"

libretroshare/src/serialiser/rstypeserializer.cc

@@ -36,8 +36,11 @@
 #include <string>
 #include <typeinfo> // for typeid
 
-#include <rapid_json/document.h>
+#ifdef HAS_RAPIDJSON
+#include <rapidjson/prettywriter.h>
+#else
 #include <rapid_json/prettywriter.h>
+#endif // HAS_RAPIDJSON
 
 //static const uint32_t MAX_SERIALIZED_ARRAY_SIZE = 500 ;
 static const uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.

libretroshare/src/serialiser/rstypeserializer.h

@@ -35,7 +35,11 @@
 #include "serialiser/rsserializer.h"
 #include "serialiser/rsserializable.h"
 
+#ifdef HAS_RAPIDJSON
+#include <rapidjson/document.h>
+#else
 #include <rapid_json/document.h>
+#endif // HAS_RAPIDJSON
 #include <typeinfo> // for typeid
 #include <type_traits>
 #include <errno.h>

libretroshare/src/services/p3banlist.cc

@@ -55,7 +55,7 @@
 #define RSBANLIST_DELAY_BETWEEN_TALK_TO_DHT 		240	// every 4 mins.
 
 /************ IMPLEMENTATION NOTES *********************************
- * 
+ *
  * Get Bad Peers passed to us (from DHT mainly).
  * we distribute and track the network list of bad peers.
  *
@@ -113,6 +113,8 @@ void p3BanList::setAutoRangeLimit(int n)
     IndicateConfigChanged();
 }
 
+namespace services {
+
 class ZeroedInt
 {
     public:
@@ -120,6 +122,8 @@ class ZeroedInt
     uint32_t n ;
 };
 
+}
+
 BanListPeer::BanListPeer()
 {
     memset(&addr, 0, sizeof(addr));
@@ -220,14 +224,14 @@ void p3BanList::autoFigureOutBanRanges()
     std::cerr << "Automatically figuring out IP ranges from banned IPs." << std::endl;
 #endif
 
-    std::map<sockaddr_storage,ZeroedInt> range_map ;
+    std::map<sockaddr_storage, services::ZeroedInt> range_map ;
 
     for(std::map<sockaddr_storage,BanListPeer>::iterator it(mBanSet.begin());it!=mBanSet.end();++it)
         ++range_map[makeBitsRange(it->first,1)].n ;
 
     time_t now = time(NULL) ;
 
-    for(std::map<sockaddr_storage,ZeroedInt>::const_iterator it=range_map.begin();it!=range_map.end();++it)
+    for(std::map<sockaddr_storage, services::ZeroedInt>::const_iterator it=range_map.begin();it!=range_map.end();++it)
     {
 #ifdef DEBUG_BANLIST
         std::cerr << "Ban range: " << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl;
@@ -646,7 +650,7 @@ bool p3BanList::processIncoming()
 				break;
 			case RS_PKT_SUBTYPE_BANLIST_ITEM:
 			{
-				// Order is important!.	
+				// Order is important!.
 				updated = (recvBanItem((RsBanListItem *) item) || updated);
 			}
 				break;
@@ -669,8 +673,8 @@ bool p3BanList::processIncoming()
 	}
 
 	return true ;
-} 
+}
-	
+
 
 bool p3BanList::recvBanItem(RsBanListItem *item)
 {
@@ -681,7 +685,7 @@ bool p3BanList::recvBanItem(RsBanListItem *item)
 
 	for(it = item->peerList.mList.begin(); it != item->peerList.mList.end(); ++it)
 	{
-		// Order is important!.	
+		// Order is important!.
         updated = (addBanEntry(item->PeerId(), it->addr.addr, it->level,  it->reason, now - it->age) || updated);
 	}
 	return updated;
@@ -961,7 +965,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId,
 		it = mBanSources.find(peerId);
 		updated = true;
 	}
-	
+
 	// index is FAMILY + IP - the rest should be Zeros..
 	struct sockaddr_storage bannedaddr;
 	sockaddr_storage_clear(bannedaddr);
@@ -980,7 +984,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId,
         blp.level = level;
         blp.mTs = time_stamp ;
         blp.masked_bytes = 0 ;
-		
+
 		it->second.mBanPeers[bannedaddr] = blp;
 		it->second.mLastUpdate = now;
 		updated = true;
@@ -1036,7 +1040,7 @@ int p3BanList::condenseBanSources_locked()
 
     time_t now = time(NULL);
 	RsPeerId ownId = mServiceCtrl->getOwnId();
-	
+
 #ifdef DEBUG_BANLIST
 	std::cerr << "p3BanList::condenseBanSources_locked()";
 	std::cerr << std::endl;
@@ -1062,7 +1066,7 @@ int p3BanList::condenseBanSources_locked()
 		std::cerr << " Condensing Info from peer: " << it->first;
 		std::cerr << std::endl;
 #endif
-		
+
 		std::map<struct sockaddr_storage, BanListPeer>::const_iterator lit;
         for(lit = it->second.mBanPeers.begin(); lit != it->second.mBanPeers.end(); ++lit)
     {
@@ -1135,7 +1139,7 @@ int p3BanList::condenseBanSources_locked()
     }
 	}
 
-	
+
 #ifdef DEBUG_BANLIST
 	std::cerr << "p3BanList::condenseBanSources_locked() Printing New Set:";
 	std::cerr << std::endl;
@@ -1280,7 +1284,7 @@ int p3BanList::printBanSet_locked(std::ostream &out)
 int p3BanList::printBanSources_locked(std::ostream &out)
 {
 	time_t now = time(NULL);
-	
+
 	std::map<RsPeerId, BanList>::const_iterator it;
 	for(it = mBanSources.begin(); it != mBanSources.end(); ++it)
 	{

libretroshare/src/services/p3gxschannels.cc

@@ -236,7 +236,7 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
 		RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
 		if (msgChange)
 		{
-			if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVE)
+			if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
 			{
 				/* message received */
 				if (notify)
@@ -289,11 +289,12 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
 				{
 					switch (grpChange->getType())
 					{
+                    default:
 						case RsGxsNotify::TYPE_PROCESSED:
-						case RsGxsNotify::TYPE_PUBLISH:
+						case RsGxsNotify::TYPE_PUBLISHED:
 							break;
 
-						case RsGxsNotify::TYPE_RECEIVE:
+						case RsGxsNotify::TYPE_RECEIVED_NEW:
 						{
 							/* group received */
 							std::list<RsGxsGroupId> &grpList = grpChange->mGrpIdList;
@@ -311,7 +312,7 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
 							break;
 						}
 
-						case RsGxsNotify::TYPE_PUBLISHKEY:
+						case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
 						{
 							/* group received */
 							std::list<RsGxsGroupId> &grpList = grpChange->mGrpIdList;
@@ -1686,3 +1687,57 @@ void p3GxsChannels::handle_event(uint32_t event_type, const std::string &elabel)
 	}
 }
 
+TurtleRequestId p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id)
+{
+    return netService()->turtleGroupRequest(group_id) ;
+}
+TurtleRequestId p3GxsChannels::turtleSearchRequest(const std::string& match_string)
+{
+    return netService()->turtleSearchRequest(match_string) ;
+}
+
+bool p3GxsChannels::clearDistantSearchResults(TurtleRequestId req)
+{
+    return netService()->clearDistantSearchResults(req);
+}
+bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& results)
+{
+    return netService()->retrieveDistantSearchResults(req,results);
+}
+
+bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
+{
+    RsGxsGroupSummary gs ;
+
+    if(netService()->retrieveDistantGroupSummary(group_id,gs))
+    {
+        // This is a placeholder information by the time we receive the full group meta data.
+
+		distant_group.mDescription           = gs.group_description;
+
+		distant_group.mMeta.mGroupId         = gs.group_id ;
+		distant_group.mMeta.mGroupName       = gs.group_name;
+		distant_group.mMeta.mGroupFlags      = GXS_SERV::FLAG_PRIVACY_PUBLIC ;
+		distant_group.mMeta.mSignFlags       = gs.sign_flags;
+
+		distant_group.mMeta.mPublishTs       = gs.publish_ts;
+    	distant_group.mMeta.mAuthorId        = gs.author_id;
+
+    	distant_group.mMeta.mCircleType      = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able.
+
+		// other stuff.
+		distant_group.mMeta.mAuthenFlags     = 0;	// wild guess...
+
+    	distant_group.mMeta.mSubscribeFlags  = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ;
+
+		distant_group.mMeta.mPop             = gs.popularity; 			// Popularity = number of friend subscribers
+    	distant_group.mMeta.mVisibleMsgCount = gs.number_of_messages; 	// Max messages reported by friends
+    	distant_group.mMeta.mLastPost        = gs.last_message_ts; 		// Timestamp for last message. Not used yet.
+
+		return true ;
+    }
+    else
+        return false ;
+}
+
+

libretroshare/src/services/p3gxschannels.h

@@ -72,6 +72,12 @@ virtual void service_tick();
 	virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList); // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
 	virtual bool loadList(std::list<RsItem *>& loadList);               // @see p3Config::loadList(std::list<RsItem *>&)
 
+    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id);
+    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string);
+    virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) ;
+    virtual bool clearDistantSearchResults(TurtleRequestId req);
+    virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group);
+
 	// Overloaded to cache new groups.
 virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
 

libretroshare/src/services/p3gxscircles.cc

@@ -218,7 +218,8 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
 				std::cerr << "    Msgs for Group: " << mit->first << std::endl;
 #endif
 				force_cache_reload(RsGxsCircleId(mit->first));
-				if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) )
+
+				if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) )
 					for (auto msgIdIt(mit->second.begin()), end(mit->second.end()); msgIdIt != end; ++msgIdIt)
 					{
 						const RsGxsMessageId& msgId = *msgIdIt;
@@ -261,7 +262,7 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
 				std::cerr << "  forcing cache loading for circle " << *git << " in order to trigger subscribe update." << std::endl;
 #endif
 				force_cache_reload(RsGxsCircleId(*git)) ;
-				if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) )
+				if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) )
 					notify->AddFeedItem(RS_FEED_ITEM_CIRCLE_INVIT_REC,RsGxsCircleId(*git).toStdString(),"");
 			}
 

libretroshare/src/services/p3gxsforums.cc

@@ -196,11 +196,12 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 
 				switch (c->getType())
 				{
+                default:
 					case RsGxsNotify::TYPE_PROCESSED:
-					case RsGxsNotify::TYPE_PUBLISH:
+					case RsGxsNotify::TYPE_PUBLISHED:
 						break;
 
-					case RsGxsNotify::TYPE_RECEIVE:
+					case RsGxsNotify::TYPE_RECEIVED_NEW:
 					{
 						RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange*>(c);
 						if (msgChange)
@@ -239,7 +240,7 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 						break;
 					}
 
-					case RsGxsNotify::TYPE_PUBLISHKEY:
+					case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
 					{
 						RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange *>(*it);
 						if (grpChange)

libretroshare/src/services/p3idservice.cc

@@ -278,7 +278,7 @@ void p3IdService::timeStampKey(const RsGxsId& gxs_id, const RsIdentityUsage& rea
         return ;
     }
 #ifdef DEBUG_IDS
-    std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason << std::endl;
+    std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason.mUsageCode << std::endl;
 #endif
 
     RS_STACK_MUTEX(mIdMtx) ;
@@ -2700,7 +2700,7 @@ void p3IdService::requestIdsFromNet()
         bool request_can_proceed = false ;
 
         for(cit2 = peers.begin(); cit2 != peers.end(); ++cit2)
-            if(rsPeers->isOnline(*cit2)) 		// make sure that the peer in online, so that we know that the request has some chance to succeed.
+            if(rsPeers->isOnline(*cit2) || mNes->isDistantPeer(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed.
             {
                 requests[*cit2].push_back(cit->first);
                 request_can_proceed = true ;

libretroshare/src/services/p3postbase.cc

@@ -122,7 +122,7 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
 				// It could be taken a step further and directly request these msgs for an update.
 				addGroupForProcessing(mit->first);
 
-				if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVE)
+				if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
 				{
 					for (auto mit1 = mit->second.begin(); mit1 != mit->second.end(); ++mit1)
 					{
@@ -149,7 +149,7 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
 				std::cerr << std::endl;
 #endif
 
-				if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVE)
+				if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
 				{
 					notify->AddFeedItem(RS_FEED_ITEM_POSTED_NEW, git->toStdString());
 				}

libretroshare/src/turtle/p3turtle.cc

@@ -34,6 +34,7 @@
 #endif
 
 #include "rsserver/p3face.h"
+#include "crypto/rscrypto.h"
 
 #include "pqi/authssl.h"
 #include "pqi/p3linkmgr.h"
@@ -63,11 +64,14 @@ static std::map<TurtleTunnelRequestId, std::vector<time_t> > TS_request_bounces
 void TS_dumpState() ;
 #endif
 
+#define TURTLE_DEBUG() std::cerr << time(NULL) << " : TURTLE : " << __FUNCTION__ << " : "
+#define TURTLE_ERROR() std::cerr << "(EE) TURTLE ERROR : "
+
 // These number may be quite important. I setup them with sensible values, but
 // an in-depth test would be better to get an idea of what the ideal values
 // could ever be.
 //
-// update of 14-03-11: 
+// update of 14-03-11:
 // 	- I raised the cache time for tunnel requests. This avoids inconsistencies such as:
 // 		* tunnel requests bouncing back while the original request is not in the cache anymore
 // 		* special case of this for own file transfer: an outgoing tunnel is built with no end.
@@ -79,17 +83,18 @@ void TS_dumpState() ;
 //    - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5
 //    - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels.
 //
-static const time_t TUNNEL_REQUESTS_LIFE_TIME 	         = 240 ;		/// life time for tunnel requests in the cache.
+static const time_t TUNNEL_REQUESTS_LIFE_TIME 	           = 240 ;		/// life time for tunnel requests in the cache.
-static const time_t SEARCH_REQUESTS_LIFE_TIME 	         = 240 ;		/// life time for search requests in the cache
+static const time_t SEARCH_REQUESTS_LIFE_TIME 	           = 240 ;		/// life time for search requests in the cache
-static const time_t REGULAR_TUNNEL_DIGGING_TIME          = 300 ;		/// maximum interval between two tunnel digging campaigns.
+static const time_t REGULAR_TUNNEL_DIGGING_TIME            = 300 ;		/// maximum interval between two tunnel digging campaigns.
-static const time_t MAXIMUM_TUNNEL_IDLE_TIME 	         =  60 ;		/// maximum life time of an unused tunnel.
+static const time_t MAXIMUM_TUNNEL_IDLE_TIME 	           =  60 ;		/// maximum life time of an unused tunnel.
-static const time_t EMPTY_TUNNELS_DIGGING_TIME 	         =  50 ;		/// look into tunnels regularly every 50 sec.
+static const time_t EMPTY_TUNNELS_DIGGING_TIME 	           =  50 ;		/// look into tunnels regularly every 50 sec.
-static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE	         =   5 ;		/// estimate tunnel speed every 5 seconds
+static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE	           =   5 ;		/// estimate tunnel speed every 5 seconds
-static const time_t TUNNEL_CLEANING_LAPS_TIME  	         =  10 ;		/// clean tunnels every 10 secs
+static const time_t TUNNEL_CLEANING_LAPS_TIME  	           =  10 ;		/// clean tunnels every 10 secs
-static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS =   2 ;                /// Tunnel management calls every 2 secs.
+static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS   =   2 ;        /// Tunnel management calls every 2 secs.
-static const uint32_t MAX_TUNNEL_REQS_PER_SECOND         =   1 ;		/// maximum number of tunnel requests issued per second. Was 0.5 before
+static const uint32_t MAX_TUNNEL_REQS_PER_SECOND           =   1 ;		/// maximum number of tunnel requests issued per second. Was 0.5 before
-static const uint32_t MAX_ALLOWED_SR_IN_CACHE            = 120 ;		/// maximum number of search requests allowed in cache. That makes 2 per sec.
+static const uint32_t MAX_ALLOWED_SR_IN_CACHE              = 120 ;		/// maximum number of search requests allowed in cache. That makes 2 per sec.
-static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS      =5000 ;		/// maximum number of search results forwarded back to the source.
+static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES  =5000 ;		/// maximum number of search results forwarded back to the source.
+static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ;		/// default maximum number of search results forwarded back source.
 
 static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
 
@@ -151,13 +156,15 @@ void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
 {
 	names.clear();
 
-	names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST	] = "Search request";
+	names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST	] = "Filename substring search request";
-	names[RS_TURTLE_SUBTYPE_SEARCH_RESULT  			] = "Search result";
+	names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST  ] = "Generic search request";
+	names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT		] = "File search result";
+	names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT   ] = "Generic search result";
 	names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL    			] = "Tunnel request";
 	names[RS_TURTLE_SUBTYPE_TUNNEL_OK      			] = "Tunnel response";
 	names[RS_TURTLE_SUBTYPE_FILE_REQUEST   			] = "Data request";
 	names[RS_TURTLE_SUBTYPE_FILE_DATA      			] = "Data chunk";
-	names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST   ] = "RegExp search";
+	names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST   ] = "Filename RegExp search request";
 	names[RS_TURTLE_SUBTYPE_GENERIC_DATA     		] = "Generic data";
 	names[RS_TURTLE_SUBTYPE_FILE_MAP                ] = "Chunk map";
 	names[RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST        ] = "Chunk map request";
@@ -165,7 +172,7 @@ void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
 	names[RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST       ] = "Chunk CRC request";
 }
 
-void p3turtle::setEnabled(bool b) 
+void p3turtle::setEnabled(bool b)
 {
 	RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 	_turtle_routing_enabled = b;
@@ -184,7 +191,7 @@ bool p3turtle::enabled() const
 }
 
 
-void p3turtle::setSessionEnabled(bool b) 
+void p3turtle::setSessionEnabled(bool b)
 {
 	RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 	_turtle_routing_session_enabled = b;
@@ -242,7 +249,7 @@ int p3turtle::tick()
 			RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 			_last_tunnel_management_time = now ;
 
-			// Update traffic statistics. The constants are important: they allow a smooth variation of the 
+			// Update traffic statistics. The constants are important: they allow a smooth variation of the
 			// traffic speed, which is used to moderate tunnel requests statistics.
 			//
 			_traffic_info = _traffic_info*0.9 + _traffic_info_buffer* (0.1 / (float)TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS) ;
@@ -370,8 +377,8 @@ void p3turtle::manageTunnels()
 	//  - the hash hasn't been tunneled for more than REGULAR_TUNNEL_DIGGING_TIME seconds, even if downloading.
 	//
 	// Candidate hashes are sorted, by olderness. The older gets tunneled first. At most MAX_TUNNEL_REQS_PER_SECOND are
-	// treated at once, as this method is called every second. 
+	// treated at once, as this method is called every second.
-	// Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get 
+	// Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get
 	// re-tunneled in priority. As this happens less, they don't obliterate tunneling for files that have no tunnels yet.
 
 	std::vector<std::pair<TurtleFileHash,time_t> > hashes_to_digg ;
@@ -552,7 +559,7 @@ void p3turtle::autoWash()
 
 	// Now remove all the virtual peers ids at the client services. Off mutex!
 	//
-	
+
 	for(uint32_t i=0;i<services_vpids_to_remove.size();++i)
 	{
 #ifdef P3TURTLE_DEBUG
@@ -614,7 +621,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector<std::pair<RsTur
 		// Let's be cautious. Normally we should never be here without consistent information,
 		// but still, this happens, rarely.
 		//
-		if(_virtual_peers.find(vpid) != _virtual_peers.end())  
+		if(_virtual_peers.find(vpid) != _virtual_peers.end())
 			_virtual_peers.erase(_virtual_peers.find(vpid)) ;
 
 		std::map<TurtleFileHash,TurtleHashInfo>::iterator it(_incoming_file_hashes.find(hash)) ;
@@ -659,7 +666,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector<std::pair<RsTur
 
 			// Also remove the associated virtual peer
 			//
-			if(_virtual_peers.find(vpid) != _virtual_peers.end())  
+			if(_virtual_peers.find(vpid) != _virtual_peers.end())
 				_virtual_peers.erase(_virtual_peers.find(vpid)) ;
 		}
 	}
@@ -724,7 +731,7 @@ bool p3turtle::loadList(std::list<RsItem*>& load)
 		RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it) ;
 
 		if(vitem != NULL)
-			for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) 
+			for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit)
 			{
 				if(kit->key == "TURTLE_CONFIG_MAX_TR_RATE")
 				{
@@ -824,7 +831,7 @@ int p3turtle::handleIncoming()
 			RsTurtleGenericTunnelItem *gti = dynamic_cast<RsTurtleGenericTunnelItem *>(item) ;
 
 			if(gti != NULL)
-				routeGenericTunnelItem(gti) ;	/// Generic packets, that travel through established tunnels. 
+				routeGenericTunnelItem(gti) ;	/// Generic packets, that travel through established tunnels.
 			else			 							/// These packets should be destroyed by the client.
 			{
 				/// Special packets that require specific treatment, because tunnels do not exist for these packets.
@@ -833,10 +840,12 @@ int p3turtle::handleIncoming()
 				switch(item->PacketSubType())
 				{
 					case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST:
+					case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST:
 					case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast<RsTurtleSearchRequestItem *>(item)) ;
 																				 break ;
 
-					case RS_TURTLE_SUBTYPE_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
+					case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT :
+					case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
 																		break ;
 
 					case RS_TURTLE_SUBTYPE_OPEN_TUNNEL   : handleTunnelRequest(dynamic_cast<RsTurtleOpenTunnelItem *>(item)) ;
@@ -862,8 +871,6 @@ int p3turtle::handleIncoming()
 //
 void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 {
-	RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
-    
 	// take a look at the item and test against inconsistent values
 	// 	- If the item destimation is
 
@@ -871,7 +878,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 	std::cerr << "Received search request from peer " << item->PeerId() << ": " << std::endl ;
 	item->print(std::cerr,0) ;
 #endif
-    
+
     uint32_t item_size = RsTurtleSerialiser().size(item);
 
 	if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE)
@@ -882,27 +889,57 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 		std::cerr << "  Caught a turtle search item with arbitrary large size from " << item->PeerId() << " of size " << item_size << " and depth " << item->depth << ". This is not allowed => dropping." << std::endl;
 		return ;
 	}
-    
+
-	if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
 	{
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+
+		if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
+		{
 #ifdef P3TURTLE_DEBUG
-		std::cerr << "  Dropping, because the search request cache is full." << std::endl ;
+			std::cerr << "  Dropping, because the search request cache is full." << std::endl ;
 #endif
-		std::cerr << "  More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl;
+			std::cerr << "  More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl;
-		return ;
+			return ;
+		}
+
+		// If the item contains an already handled search request, give up.  This
+		// happens when the same search request gets relayed by different peers
+		//
+		if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end())
+		{
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "  This is a bouncing request. Ignoring and deleting it." << std::endl ;
+#endif
+			return ;
+		}
 	}
 
-	// If the item contains an already handled search request, give up.  This
+    // Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain.
-	// happens when the same search request gets relayed by different peers
+
-	//
+    uint32_t search_result_count = 0;
-	if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end())
+    uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT;
+
+	if(item->PeerId() != _own_id) // is the request not coming from us?
 	{
 #ifdef P3TURTLE_DEBUG
-		std::cerr << "  This is a bouncing request. Ignoring and deleting it." << std::endl ;
+		std::cerr << "  Request not from us. Performing local search" << std::endl ;
 #endif
-		return ;
+        std::list<RsTurtleSearchResultItem*> search_results ;
+
+        performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ;
+
+        for(auto it(search_results.begin());it!=search_results.end();++it)
+        {
+            (*it)->request_id = item->request_id ;
+            (*it)->depth = 0 ;
+            (*it)->PeerId(item->PeerId()) ;
+
+            sendItem(*it) ;
+        }
 	}
 
+	RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+
 	// This is a new request. Let's add it to the request map, and forward it to
 	// open peers.
 
@@ -910,67 +947,17 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 	req.origin = item->PeerId() ;
 	req.time_stamp = time(NULL) ;
 	req.depth = item->depth ;
-	req.result_count = 0;
+	req.result_count = search_result_count;
 	req.keywords = item->GetKeywords() ;
-
+	req.service_id = item->serviceId() ;
-	// If it's not for us, perform a local search. If something found, forward the search result back.
+    req.max_allowed_hits = max_allowed_hits;
-
-	if(item->PeerId() != _own_id)
-	{
-#ifdef P3TURTLE_DEBUG
-		std::cerr << "  Request not from us. Performing local search" << std::endl ;
-#endif
-
-		std::list<TurtleFileInfo> result ;
-
-		item->performLocalSearch(result) ;
-
-		RsTurtleSearchResultItem *res_item = NULL ;
-		uint32_t item_size = 0 ;
-
-#ifdef P3TURTLE_DEBUG
-		if(!result.empty())
-			std::cerr << "  " << result.size() << " matches found. Sending back to origin (" << item->PeerId() << ")." << std::endl ;
-#endif
-		while(!result.empty() && req.result_count < TURTLE_SEARCH_RESULT_MAX_HITS)
-		{
-			// Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity.
-			//
-			static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
-
-			if(res_item == NULL)
-			{
-				res_item = new RsTurtleSearchResultItem ;
-				item_size = 0 ;
-
-				res_item->depth = 0 ;
-				res_item->request_id = item->request_id ;
-				res_item->PeerId(item->PeerId()) ;			// send back to the same guy
-			}
-			res_item->result.push_back(result.front()) ;
-
-			++req.result_count ;	// increase hit number for this particular search request.
-
-            item_size += 8 /* size */ + result.front().hash.serial_size() + result.front().name.size() ;
-			result.pop_front() ;
-
-			if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || result.empty() || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
-			{
-#ifdef P3TURTLE_DEBUG
-				std::cerr << "  Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
-#endif
-				sendItem(res_item) ;
-				res_item = NULL ;
-			}
-		}
-	}
 
 	// if enough has been sent back already, do not sarch further
 
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  result count = " << req.result_count << std::endl;
 #endif
-	if(req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
+	if(req.result_count >= max_allowed_hits)
 		return ;
 
 	// If search depth not too large, also forward this search request to all other peers.
@@ -1007,14 +994,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 				// Copy current item and modify it.
 				RsTurtleSearchRequestItem *fwd_item = item->clone() ;
 
-				// increase search depth, except in some rare cases, to prevent correlation between 
+				// increase search depth, except in some rare cases, to prevent correlation between
 				// TR sniffing and friend names. The strategy is to not increase depth if the depth
 				// is 1:
 				// 	If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the
 				// 	file, since A might have shifted the depth.
 				//
 				if(!random_dshift)
-					++(fwd_item->depth) ;		
+					++(fwd_item->depth) ;
 
 				fwd_item->PeerId(*it) ;
 
@@ -1028,67 +1015,194 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
 #endif
 }
 
+// This function should be removed in the future, when file search will also use generic search items.
+
+void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results,uint32_t& max_allowed_hits)
+{
+    RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast<RsTurtleFileSearchRequestItem*>(item) ;
+
+    if(ftsearch != NULL)
+    {
+        performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ;
+        return ;
+    }
+
+    RsTurtleGenericSearchRequestItem *gnsearch = dynamic_cast<RsTurtleGenericSearchRequestItem*>(item) ;
+
+    if(gnsearch != NULL)
+    {
+        performLocalSearch_generic(gnsearch,req_result_count,search_results,max_allowed_hits) ;
+        return ;
+    }
+}
+
+void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
+{
+    unsigned char *search_result_data = NULL ;
+    uint32_t search_result_data_len = 0 ;
+
+    RsTurtleClientService *client = NULL ;
+
+	{
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+		auto it = _registered_services.find(item->service_id) ;
+
+		if(it == _registered_services.end())
+			return ;
+
+		client = it->second ;
+	}
+
+    if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len,max_allowed_hits))
+    {
+		RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ;
+
+        result_item->result_data = search_result_data ;
+        result_item->result_data_len = search_result_data_len ;
+
+        result.push_back(result_item) ;
+    }
+}
+
+void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
+{
+#ifdef P3TURTLE_DEBUG
+	std::cerr << "Performing rsFiles->search()" << std::endl ;
+#endif
+	// now, search!
+    std::list<TurtleFileInfo> initialResults ;
+    item->search(initialResults) ;
+
+#ifdef P3TURTLE_DEBUG
+	std::cerr << initialResults.size() << " matches found." << std::endl ;
+#endif
+	result.clear() ;
+	RsTurtleFTSearchResultItem *res_item = NULL ;
+	uint32_t item_size = 0 ;
+
+	static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
+    max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_FILES;
+
+	for(auto it(initialResults.begin());it!=initialResults.end();++it)
+	{
+		if(res_item == NULL)
+		{
+			res_item = new RsTurtleFTSearchResultItem ;
+			item_size = 0 ;
+
+            result.push_back(res_item) ;
+		}
+		res_item->result.push_back(*it);
+
+		// Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity.
+		//
+		++req_result_count ;	// increase hit number for this particular search request.
+
+		item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ;
+
+		if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= max_allowed_hits)
+		{
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "  Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
+#endif
+			res_item = NULL ;	// forces creation of a new item.
+		}
+	}
+}
+
 void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 {
-	RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+    std::list<std::pair<RsTurtleSearchResultItem*,RsTurtleClientService*> > results_to_notify_off_mutex ;
-	// Find who actually sent the corresponding request.
+
-	//
-	std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
-#ifdef P3TURTLE_DEBUG
-	std::cerr << "Received search result:" << std::endl ;
-	item->print(std::cerr,0) ;
-#endif
-	if(it == _search_requests_origins.end())
 	{
-		// This is an error: how could we receive a search result corresponding to a search item we
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
-		// have forwarded but that it not in the list ??
+		// Find who actually sent the corresponding request.
+		//
+		std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
 
-		std::cerr << __PRETTY_FUNCTION__ << ": search result has no peer direction!" << std::endl ;
-		return ;
-	}
-
-	// Is this result's target actually ours ?
-
-	if(it->second.origin == _own_id)
-	{
-		it->second.result_count += item->result.size() ;
-		returnSearchResult(item) ;		// Yes, so send upward.
-	}
-	else
-	{									// Nope, so forward it back.
 #ifdef P3TURTLE_DEBUG
-		std::cerr << "  Forwarding result back to " << it->second.origin << std::endl;
+		std::cerr << "Received search result:" << std::endl ;
+		item->print(std::cerr,0) ;
 #endif
-		// We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS.
+		if(it == _search_requests_origins.end())
-
-		uint32_t n = item->result.size(); // not so good!
-
-		if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
 		{
-			std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
+			// This is an error: how could we receive a search result corresponding to a search item we
+			// have forwarded but that it not in the list ??
+
+			std::cerr << __PRETTY_FUNCTION__ << ": search result for request " << std::hex << item->request_id << std::dec << " has no peer direction!" << std::endl ;
 			return ;
 		}
 
-		if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS)
+		// Is this result's target actually ours ?
-		{
-			for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i)
-				item->result.pop_back() ;
 
-			it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ;
+		if(it->second.origin == _own_id)
+		{
+			it->second.result_count += item->count() ;
+
+            auto it2 = _registered_services.find(it->second.service_id) ;
+
+            if(it2 != _registered_services.end())
+				results_to_notify_off_mutex.push_back(std::make_pair(item,it2->second)) ;
+            else
+                std::cerr << "(EE) cannot find client service for ID " << std::hex << it->second.service_id << std::dec << ": search result item will be dropped." << std::endl;
 		}
 		else
-			it->second.result_count += n ;
+		{									// Nope, so forward it back.
+#ifdef P3TURTLE_DEBUG
+			std::cerr << "  Forwarding result back to " << it->second.origin << std::endl;
+#endif
+			// We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS.
 
-		RsTurtleSearchResultItem *fwd_item = new RsTurtleSearchResultItem(*item) ;	// copy the item
+			uint32_t n = item->count(); // not so good!
 
-		// Normally here, we should setup the forward adress, so that the owner's
+			if(it->second.result_count >= it->second.max_allowed_hits)
-		// of the files found can be further reached by a tunnel.
+			{
+				std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
+				return ;
+			}
 
-		fwd_item->PeerId(it->second.origin) ;
+			if(it->second.result_count + n > it->second.max_allowed_hits)
-		fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information.
+			{
+				for(uint32_t i=it->second.result_count + n; i>it->second.max_allowed_hits;--i)
+					item->pop() ;
 
-		sendItem(fwd_item) ;
+				it->second.result_count = it->second.max_allowed_hits ;
+			}
+			else
+				it->second.result_count += n ;
+
+			RsTurtleSearchResultItem *fwd_item = item->duplicate();
+
+			// Normally here, we should setup the forward adress, so that the owner's
+			// of the files found can be further reached by a tunnel.
+
+			fwd_item->PeerId(it->second.origin) ;
+			fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information.
+
+			sendItem(fwd_item) ;
+		}
 	}
+
+    // now we notify clients off-mutex.
+
+    for(auto it(results_to_notify_off_mutex.begin());it!=results_to_notify_off_mutex.end();++it)
+    {
+        // Hack to use the old search result handling in ftServer. Normally ftServer should use the new method with serialized result.
+
+#warning make sure memory is correctly deleted here
+        RsTurtleFTSearchResultItem *ftsr = dynamic_cast<RsTurtleFTSearchResultItem*>(it->first) ;
+
+        if(ftsr!=NULL)
+        {
+			RsServer::notify()->notifyTurtleSearchResult(ftsr->request_id,ftsr->result) ;
+            continue ;
+        }
+
+        RsTurtleGenericSearchResultItem *gnsr = dynamic_cast<RsTurtleGenericSearchResultItem*>(it->first) ;
+
+        if(gnsr!=NULL)
+			(*it).second->receiveSearchResult(gnsr->request_id,gnsr->result_data,gnsr->result_data_len) ;
+    }
 }
 
 // -----------------------------------------------------------------------------------//
@@ -1150,8 +1264,8 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
 
 		// Let's figure out whether this packet is for us or not.
 
-		if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT && 
+		if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT &&
-		{														 	
+		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Forwarding generic item to peer " << tunnel.local_src << std::endl ;
 #endif
@@ -1189,7 +1303,7 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
 	// The packet was not forwarded, so it is for us. Let's treat it.
 	// This is done off-mutex, to avoid various deadlocks
 	//
-	
+
     handleRecvGenericTunnelItem(item) ;
 
 	delete item ;
@@ -1312,13 +1426,13 @@ void p3turtle::sendTurtleData(const RsPeerId& virtual_peer_id,RsTurtleGenericTun
 
 	if(tunnel.local_src == _own_id)
 	{
-		item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ;	
+		item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ;
 		item->PeerId(tunnel.local_dst) ;
 		_traffic_info_buffer.data_dn_Bps += ss ;
 	}
 	else if(tunnel.local_dst == _own_id)
 	{
-		item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;	
+		item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
 		item->PeerId(tunnel.local_src) ;
 		_traffic_info_buffer.data_up_Bps += ss ;
 	}
@@ -1456,7 +1570,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 
 		std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo>::iterator it = _tunnel_requests_origins.find(item->request_id) ;
-		
+
 		if(it != _tunnel_requests_origins.end())
 		{
 #ifdef P3TURTLE_DEBUG
@@ -1559,7 +1673,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 	// - the tunnel id will now be unique for a given route
 	// - allows a better balance of bandwidth for a given transfer
 	// - avoid the waste of items that get lost when re-routing a tunnel
-	
+
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "Perturbating partial tunnel id. Original=" << std::hex << item->partial_tunnel_id ;
 #endif
@@ -1590,7 +1704,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 			forward_probability = 1.0f / nb_online_ids ;
 
 			// Setting forward_probability to 1/nb_online_ids forces at most one TR up per TR dn. But if we are overflooded by
-			// TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the 
+			// TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the
 			// forward probability so as to reduct the output rate accordingly.
 			//
 			if(_traffic_info.tr_dn_Bps / (float)TUNNEL_REQUEST_PACKET_SIZE > _max_tr_up_rate)
@@ -1616,7 +1730,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
 				// Copy current item and modify it.
 				RsTurtleOpenTunnelItem *fwd_item = new RsTurtleOpenTunnelItem(*item) ;
 
-				// increase search depth, except in some rare cases, to prevent correlation between 
+				// increase search depth, except in some rare cases, to prevent correlation between
 				// TR sniffing and friend names. The strategy is to not increase depth if the depth
 				// is 1:
 				// 	If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the
@@ -1773,7 +1887,9 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
 // ------------------------------  IO with libretroshare  ----------------------------//
 // -----------------------------------------------------------------------------------//
 //
-void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInfo>& result) const
+
+
+void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
 {
 	/* call to core */
 	std::list<DirDetails> initialResults;
@@ -1796,7 +1912,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
 	for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
 	{
 		// retain only file type
-		if (it->type == DIR_TYPE_DIR) 
+		if (it->type == DIR_TYPE_DIR)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Skipping directory " << it->name << std::endl ;
@@ -1812,7 +1928,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
 		result.push_back(i) ;
 	}
 }
-void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInfo>& result) const
+void RsTurtleRegExpSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
 {
 	/* call to core */
 	std::list<DirDetails> initialResults;
@@ -1835,7 +1951,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
 	for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
 	{
 		// retain only file type
-		if (it->type == DIR_TYPE_DIR) 
+		if (it->type == DIR_TYPE_DIR)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Skipping directory " << it->name << std::endl ;
@@ -1907,6 +2023,34 @@ TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpr
 	return id ;
 }
 
+TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service)
+{
+	// generate a new search id.
+
+	TurtleRequestId id = generateRandomRequestId() ;
+
+	// Form a request packet that simulates a request from us.
+	//
+	RsTurtleGenericSearchRequestItem item ;
+
+#ifdef P3TURTLE_DEBUG
+	std::cerr << "performing search. OwnId = " << _own_id << std::endl ;
+#endif
+
+	item.PeerId(_own_id) ;
+    item.service_id = client_service->serviceId();
+	item.search_data = search_bin_data ;
+	item.search_data_len = search_bin_data_len ;
+	item.request_id = id ;
+	item.depth = 0 ;
+
+	// send it
+
+	handleSearchRequest(&item) ;
+
+	return id ;
+}
+
 void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *client_service,bool allow_multi_tunnels)
 {
 	{
@@ -1947,29 +2091,49 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie
 	IndicateConfigChanged() ;	// initiates saving of handled hashes.
 }
 
-void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item)
-{
-	// just cout for now, but it should be notified to the gui
-
 #ifdef P3TURTLE_DEBUG
 	std::cerr << "  Returning result for search request " << HEX_PRINT(item->request_id) << " upwards." << std::endl ;
 #endif
 
-	RsServer::notify()->notifyTurtleSearchResult(item->request_id,item->result) ;
+
-}
+//    RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast<RsTurtleGxsSearchResultGroupSummaryItem*>(item) ;
+//
+//    if(gxs_sr_gs != NULL)
+//    {
+//		RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ;
+//        return ;
+//    }
+//    RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast<RsTurtleGxsSearchResultGroupDataItem*>(item) ;
+//
+//    if(gxs_sr_gd != NULL)
+//    {
+//#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA.
+//		//RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ;
+//        return ;
+//    }
 
 /// Warning: this function should never be called while the turtle mutex is locked.
 /// Otherwize this is a possible source of cross-lock with the File mutex.
 //
 bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& peer_id,RsTurtleClientService *& service)
 {
-	if(_registered_services.empty())
+    std::map<uint16_t,RsTurtleClientService*> client_map ;
-		std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl;
+	{
+		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
 
-	for(std::list<RsTurtleClientService*>::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it)
+		if(_registered_services.empty())
-		if( (*it)->handleTunnelRequest(hash,peer_id))
+        {
+			std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl;
+            return false ;
+        }
+
+		client_map = _registered_services ;
+	}
+
+	for(auto it(client_map.begin());it!=client_map.end();++it)
+		if( (*it).second->handleTunnelRequest(hash,peer_id))
 		{
-			service = *it ;
+			service = it->second ;
 			return true ;
 		}
 
@@ -1979,14 +2143,9 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId&
 
 void p3turtle::registerTunnelService(RsTurtleClientService *service)
 {
-#ifdef P3TURTLE_DEBUG
+	std::cerr << "p3turtle: registered new tunnel service with ID=" << std::hex << service->serviceId() << std::dec << " and pointer " << (void*)service << std::endl;
-	for(std::list<RsTurtleClientService*>::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it)
-		if(service == *it)
-			throw std::runtime_error("p3turtle::registerTunnelService(): Cannot register the same service twice. Please fix the code!") ;
-#endif
-	std::cerr << "p3turtle: registered new tunnel service " << (void*)service << std::endl;
 
-	_registered_services.push_back(service) ;
+	_registered_services[service->serviceId()] = service ;
 	_serialiser->registerClientService(service) ;
 }
 
@@ -2068,7 +2227,7 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i
 	std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(virtual_peer_id)) ;
 	if(it != _virtual_peers.end())
 	{
-		std::map<TurtleTunnelId,TurtleTunnel>::iterator it2( _local_tunnels.find(it->second) ) ;	
+		std::map<TurtleTunnelId,TurtleTunnel>::iterator it2( _local_tunnels.find(it->second) ) ;
 		if(it2 != _local_tunnels.end())
 		{
 			if(it2->second.local_src == _own_id)
@@ -2080,6 +2239,28 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i
 	return name;
 }
 
+bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item)
+{
+    unsigned char *encrypted_data = NULL ;
+    uint32_t encrypted_data_len = 0 ;
+
+    if(!librs::crypto::encryptAuthenticateData(clear_data,clear_data_size,encryption_master_key,encrypted_data,encrypted_data_len))
+    {
+        delete encrypted_item ;
+        return false ;
+    }
+	encrypted_item = new RsTurtleGenericDataItem ;
+
+    encrypted_item->data_bytes = encrypted_data ;
+    encrypted_item->data_size = encrypted_data_len ;
+    return true;
+}
+
+bool p3turtle::decryptItem(const RsTurtleGenericDataItem* encrypted_item, uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size)
+{
+   return librs::crypto::decryptAuthenticateData((unsigned char*)encrypted_item->data_bytes,encrypted_item->data_size,encryption_master_key,decrypted_data,decrypted_data_size);
+}
+
 void p3turtle::getInfo(	std::vector<std::vector<std::string> >& hashes_info,
 								std::vector<std::vector<std::string> >& tunnels_info,
 								std::vector<TurtleSearchRequestDisplayInfo >& search_reqs_info,
@@ -2114,12 +2295,12 @@ void p3turtle::getInfo(	std::vector<std::vector<std::string> >& hashes_info,
 		tunnel.push_back(printNumber(it->first,true)) ;
 
 		std::string name;
-		if(mLinkMgr->getPeerName(it->second.local_src,name)) 
+		if(mLinkMgr->getPeerName(it->second.local_src,name))
 			tunnel.push_back(name) ;
 		else
 			tunnel.push_back(it->second.local_src.toStdString()) ;
 
-		if(mLinkMgr->getPeerName(it->second.local_dst,name)) 
+		if(mLinkMgr->getPeerName(it->second.local_dst,name))
 			tunnel.push_back(name) ;
 		else
 			tunnel.push_back(it->second.local_dst.toStdString());

libretroshare/src/turtle/p3turtle.h

@@ -169,10 +169,12 @@ class TurtleSearchRequestInfo
 {
 	public:
 		TurtlePeerId origin ;         // where the request came from.
-		uint32_t	time_stamp ;      // last time the tunnel was actually used. Used for cleaning old tunnels.
+		uint32_t	 time_stamp ;     // last time the tunnel was actually used. Used for cleaning old tunnels.
-		int depth ;                   // depth of the request. Used to optimize tunnel length.
+		int          depth ;          // depth of the request. Used to optimize tunnel length.
-		uint32_t    result_count;     // responses to this request. Useful to avoid spamming tunnel responses.
+		uint32_t     result_count;    // responses to this request. Useful to avoid spamming tunnel responses.
-		std::string keywords;
+		std::string  keywords;
+        uint16_t     service_id;      // ID of the client service who issues the request. This is null if the request does not have a local origin.
+        uint32_t     max_allowed_hits;// Max number of hits allowed for this search. This actually depends on the type of search (files, GXS groups, GXS group data, etc)
 };
 class TurtleTunnelRequestInfo
 {
@@ -244,11 +246,16 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		// the request id, which will be further used by the gui to store results
 		// as they come back.
 		//
-		// Eventually, search requests should be handled by client services. We will therefore
+        // The first two methods are old style search requests for FT, while the 3rd one is using a generic search data type, that is only to
-		// remove the specific file search packets from the turtle router.
+        // be deserialized by the service. The memory ownership is kept by the calling function. Similarly, the search response will be a
-		//
+        // generic data type that is to be deserialized by the client service.
-		virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
+        //
-        virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+        // Eventually, search requests will use the generic system
+        // even for FT. We need to keep the old method for a while for backward compatibility.
+        //
+        virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
+        virtual TurtleRequestId turtleSearch(const std::string& string_to_match) ;
+        virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) ;
 
 		// Initiates tunnel handling for the given file hash.  tunnels.  Launches
 		// an exception if an error occurs during the initialization process. The
@@ -332,6 +339,12 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		/// Send a data request into the correct tunnel for the given file hash
 		void sendTurtleData(const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem *item) ;
 
+		/// Encrypts/decrypts an item, using a autenticated construction + chacha20, based on the given 32 bytes master key.
+		/// Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
+		///
+		static bool encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item);
+		static bool decryptItem(const RsTurtleGenericDataItem *item, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size);
+
 	private:
 		//--------------------------- Admin/Helper functions -------------------------//
 		
@@ -385,10 +398,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		//------ Functions connecting the turtle router to other components.----------//
 		
 		/// Performs a search calling local cache and search structure.
-		void performLocalSearch(const std::string& match_string,std::list<TurtleFileInfo>& result) ;
+		void performLocalSearch        (RsTurtleSearchRequestItem        *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits) ;
-
+		void performLocalSearch_files  (RsTurtleFileSearchRequestItem    *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
-		/// Returns a search result upwards (possibly to the gui)
+		void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
-		void returnSearchResult(RsTurtleSearchResultItem *item) ;
 
 		/// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer.
 		virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service);
@@ -410,7 +422,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
 		std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo> 	_tunnel_requests_origins ;
 
 		/// stores adequate tunnels for each file hash locally managed
-		std::map<TurtleFileHash,TurtleHashInfo>				_incoming_file_hashes ;		
+		std::map<TurtleFileHash,TurtleHashInfo>			   	_incoming_file_hashes ;
 
 		/// stores file info for each file we provide.
         std::map<TurtleTunnelId,RsTurtleClientService *>	_outgoing_tunnel_client_services ;
@@ -425,7 +437,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
         std::set<TurtleFileHash>								_hashes_to_remove ;
 
 		/// List of client services that have regitered.
-		std::list<RsTurtleClientService*>						_registered_services ;
+		std::map<uint16_t,RsTurtleClientService*>						_registered_services ;
 
 		time_t _last_clean_time ;
 		time_t _last_tunnel_management_time ;

libretroshare/src/turtle/rsturtleitem.cc

@@ -28,10 +28,12 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
 	{
 	case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST	:	return new RsTurtleStringSearchRequestItem();
 	case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST	:	return new RsTurtleRegExpSearchRequestItem();
-	case RS_TURTLE_SUBTYPE_SEARCH_RESULT			:	return new RsTurtleSearchResultItem();
+	case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT			:	return new RsTurtleFTSearchResultItem();
 	case RS_TURTLE_SUBTYPE_OPEN_TUNNEL  			:	return new RsTurtleOpenTunnelItem();
 	case RS_TURTLE_SUBTYPE_TUNNEL_OK    			:	return new RsTurtleTunnelOkItem();
 	case RS_TURTLE_SUBTYPE_GENERIC_DATA 			:	return new RsTurtleGenericDataItem();
+	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST	:	return new RsTurtleGenericSearchRequestItem();
+	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT	:	return new RsTurtleGenericSearchResultItem();
 
 	default:
 		break ;
@@ -48,20 +50,44 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
 	return NULL ;
 }
 
+std::string RsTurtleGenericSearchRequestItem::GetKeywords()
+{
+    return std::string("Generic search : " + RsUtil::BinToHex(search_data,search_data_len,10));
+}
+
 void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process          (j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ;
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth     ,"depth") ;
 }
-
 void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
     RsTypeSerializer::serial_process(j,ctx,expr,"expr") ;
 }
+void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
+    RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
+    RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
+    RsTypeSerializer::serial_process<uint8_t >(j,ctx,request_type,"request_type") ;
 
+    RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
+}
+RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const
+{
+    RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ;
+
+    memcpy(sr,this,sizeof(RsTurtleGenericSearchRequestItem)) ;
+
+    sr->search_data = (unsigned char*)rs_malloc(search_data_len) ;
+    memcpy(sr->search_data,search_data,search_data_len) ;
+
+    return sr ;
+}
 template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r)
 {
     uint32_t s = 0 ;
@@ -143,12 +169,31 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const RsRegul
 RS_TYPE_SERIALIZER_TO_JSON_NOT_IMPLEMENTED_DEF(RsRegularExpression::LinearizedExpression)
 RS_TYPE_SERIALIZER_FROM_JSON_NOT_IMPLEMENTED_DEF(RsRegularExpression::LinearizedExpression)
 
-void RsTurtleSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
     RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth     ,"depth") ;
     RsTypeSerializer::serial_process          (j,ctx,result    ,"result") ;
 }
+void RsTurtleGenericSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
+    RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth     ,"depth") ;
+
+    RsTypeSerializer::TlvMemBlock_proxy prox(result_data,result_data_len) ;
+    RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
+}
+RsTurtleSearchResultItem *RsTurtleGenericSearchResultItem::duplicate() const
+{
+    RsTurtleGenericSearchResultItem *sr = new RsTurtleGenericSearchResultItem ;
+
+    sr->result_data = (unsigned char*)rs_malloc(result_data_len) ;
+    memcpy(sr->result_data,result_data,result_data_len) ;
+    sr->result_data_len = result_data_len ;
+    sr->request_id = request_id ;
+    sr->depth = depth ;
+    return sr ;
+}
 
 template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i)
 {

libretroshare/src/turtle/rsturtleitem.h

@@ -16,20 +16,24 @@
 #include "serialiser/rsserializer.h"
 
 const uint8_t RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST	= 0x01 ;
-const uint8_t RS_TURTLE_SUBTYPE_SEARCH_RESULT  			= 0x02 ;
+const uint8_t RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT		= 0x02 ;
 const uint8_t RS_TURTLE_SUBTYPE_OPEN_TUNNEL    			= 0x03 ;
 const uint8_t RS_TURTLE_SUBTYPE_TUNNEL_OK      			= 0x04 ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST   			= 0x07 ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA      			= 0x08 ;
 const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST   = 0x09 ;
 const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA     		= 0x0a ;
+const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST  = 0x0b ;
+const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT   = 0x0c ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP                = 0x10 ;
 const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST        = 0x11 ;
+// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC                = 0x12 ; // unused
+// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST        = 0x13 ;
 const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC               = 0x14 ;
 const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST       = 0x15 ;
 
-// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC                = 0x12 ; // unused
+
-// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST        = 0x13 ;
+class TurtleSearchRequestInfo ;
 
 /***********************************************************************************/
 /*                           Basic Turtle Item Class                               */
@@ -46,49 +50,66 @@ class RsTurtleItem: public RsItem
 /*                                Specific packets                                 */
 /***********************************************************************************/
 
-class RsTurtleSearchResultItem: public RsTurtleItem
+// Class hierarchy is
-{
+//
-	public:
+//     RsTurtleItem
-        RsTurtleSearchResultItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;}
+//         |
+//         +---- RsTurtleSearchRequestItem
+//         |               |
+//         |               +---- RsTurtleFileSearchRequestItem
+//         |               |                 |
+//         |               |                 +---- RsTurtleStringSearchRequestItem
+//         |               |                 |
+//         |               |                 +---- RsTurtleReqExpSearchRequestItem
+//         |               |
+//         |               +---- RsTurtleGenericSearchRequestItem
+//         |
+//         +---- RsTurtleSearchResultItem
+//                         |
+//                         +---- RsTurtleFTSearchResultItem
+//                         |
+//                         +---- RsTurtleGenericSearchResultItem
+//
 
-		TurtleSearchRequestId request_id ;	// Randomly generated request id.
+class RsTurtleSearchResultItem ;
-
-		uint16_t depth ;					// The depth of a search result is obfuscated in this way:
-											// 	If the actual depth is 1, this field will be 1.
-											// 	If the actual depth is > 1, this field is a larger arbitrary integer.
-		std::list<TurtleFileInfo> result ;
-
-        void clear() { result.clear() ; }
-	protected:
-		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
-
-};
 
 class RsTurtleSearchRequestItem: public RsTurtleItem
 {
 	public:
         RsTurtleSearchRequestItem(uint32_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST) ;}
+        virtual ~RsTurtleSearchRequestItem() {}
+
 		virtual RsTurtleSearchRequestItem *clone() const = 0 ;					// used for cloning in routing methods
 
-		virtual void performLocalSearch(std::list<TurtleFileInfo>&) const = 0 ;	// abstracts the search method
-		
 		virtual std::string GetKeywords() = 0;
-		
+        virtual uint16_t serviceId() const= 0 ;
+
 		uint32_t request_id ; 		// randomly generated request id.
 		uint16_t depth ;				// Used for limiting search depth.
 };
 
-class RsTurtleStringSearchRequestItem: public RsTurtleSearchRequestItem
+class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem
 {
 	public:
-		RsTurtleStringSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {} 
+        RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {}
-			
+        virtual ~RsTurtleFileSearchRequestItem() {}
+
+        virtual uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
+		virtual void search(std::list<TurtleFileInfo> &) const =0;
+};
+
+class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem
+{
+	public:
+		RsTurtleStringSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {}
+        virtual ~RsTurtleStringSearchRequestItem() {}
+
+		virtual void search(std::list<TurtleFileInfo> &) const ;
+
 		std::string match_string ;	// string to match
-		
 		std::string GetKeywords() { return match_string; }
-		
+
 		virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleStringSearchRequestItem(*this) ; }
-		virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;
 
         void clear() { match_string.clear() ; }
 
@@ -96,10 +117,11 @@ class RsTurtleStringSearchRequestItem: public RsTurtleSearchRequestItem
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };
 
-class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
+class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem
 {
 	public:
-		RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {} 
+		RsTurtleRegExpSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {}
+        virtual ~RsTurtleRegExpSearchRequestItem() {}
 
         RsRegularExpression::LinearizedExpression expr ;	// Reg Exp in linearised mode
 
@@ -110,15 +132,91 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
 			delete ex;
 			return exs;
 		}
-		
-		virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
-		virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;
 
+		virtual void search(std::list<TurtleFileInfo> &) const ;
+
+		virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
 		void clear() { expr = RsRegularExpression::LinearizedExpression(); }
 	protected:
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };
 
+class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem
+{
+	public:
+		RsTurtleGenericSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {}
+        virtual ~RsTurtleGenericSearchRequestItem() { clear(); }
+
+        uint16_t service_id ;		// service to search
+        uint32_t search_data_len ;
+        uint8_t  request_type ;     // type of request. This is used to limit the number of responses.
+        unsigned char *search_data ;
+
+		std::string GetKeywords() ;
+        virtual uint16_t serviceId() const { return service_id ; }
+
+		virtual RsTurtleSearchRequestItem *clone() const ;
+        virtual uint32_t requestType() const { return request_type; }
+
+		void clear() { free(search_data); search_data=NULL; search_data_len=0; }
+
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+
+    private:
+        RsTurtleGenericSearchRequestItem(const RsTurtleGenericSearchRequestItem&): RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {}				// make the object non copi-able.
+        RsTurtleGenericSearchRequestItem& operator=(const RsTurtleGenericSearchRequestItem&) { return *this;}
+};
+class RsTurtleSearchResultItem: public RsTurtleItem
+{
+	public:
+        RsTurtleSearchResultItem(uint8_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;}
+
+		TurtleSearchRequestId request_id ;	// Randomly generated request id.
+		uint16_t depth ;					// The depth of a search result is obfuscated in this way:
+											// 	If the actual depth is 1, this field will be 1.
+											// 	If the actual depth is > 1, this field is a larger arbitrary integer.
+
+        virtual uint32_t count() const =0;
+        virtual void pop() =0;
+
+		virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)=0;
+        virtual RsTurtleSearchResultItem *duplicate() const =0;
+};
+
+class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem
+{
+	public:
+        RsTurtleFTSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT){}
+
+		std::list<TurtleFileInfo> result ;
+
+        void clear() { result.clear() ; }
+        uint32_t count() const { return result.size() ; }
+        virtual void pop() { result.pop_back() ;}
+        virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleFTSearchResultItem(*this) ; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
+class RsTurtleGenericSearchResultItem: public RsTurtleSearchResultItem
+{
+	public:
+        RsTurtleGenericSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT){}
+        virtual ~RsTurtleGenericSearchResultItem() {}
+
+        uint32_t count() const { return result_data_len/50 ; }	// This is a blind size estimate. We should probably use the actual size to limit search results.
+        virtual void pop() {}
+
+        unsigned char *result_data ;
+        uint32_t result_data_len ;
+
+        virtual RsTurtleSearchResultItem *duplicate() const ;
+		void clear() { free(result_data); result_data=NULL; result_data_len=0; }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
+
 /***********************************************************************************/
 /*                           Turtle Tunnel Item classes                            */
 /***********************************************************************************/
@@ -128,10 +226,10 @@ class RsTurtleOpenTunnelItem: public RsTurtleItem
 	public:
         RsTurtleOpenTunnelItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_OPEN_TUNNEL), request_id(0), partial_tunnel_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_OPEN_TUNNEL) ;}
 
-		TurtleFileHash file_hash ;	  // hash to match
+		TurtleFileHash file_hash ;	 // hash to match
-		uint32_t request_id ;		  // randomly generated request id.
+		uint32_t request_id ;		 // randomly generated request id.
 		uint32_t partial_tunnel_id ; // uncomplete tunnel id. Will be completed at destination.
-		uint16_t depth ;				  // Used for limiting search depth.
+		uint16_t depth ;			 // Used for limiting search depth.
 
         void clear() { file_hash.clear() ;}
 	protected:
@@ -167,24 +265,24 @@ class RsTurtleGenericTunnelItem: public RsTurtleItem
 
 		/// Does this packet stamps tunnels when it passes through ?
 		/// This is used for keeping trace weither tunnels are active or not.
-		
+
 		virtual bool shouldStampTunnel() const = 0 ;
 
-		/// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to 
+		/// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to
 		/// indicate which tunnel they are travelling through.
-		
+
 		virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
 
 		/// Indicate weither the packet is a client packet (goign back to the
 		/// client) or a server packet (going to the server. Typically file
 		/// requests are server packets, whereas file data are client packets.
-		
+
 		virtual Direction travelingDirection() const { return direction ; }
 		virtual void setTravelingDirection(Direction d) { direction = d; }
 
 		Direction direction ;	// This does not need to be serialised. It's only used by the client services, optionnally,
 										// and is set by the turtle router according to which direction the item travels.
-										
+
 		uint32_t tunnel_id ;		// Id of the tunnel to travel through
 };
 

libretroshare/src/turtle/turtleclientservice.h

@@ -42,51 +42,125 @@ class p3turtle ;
 class RsTurtleClientService
 {
 	public:
-		// Handling of tunnel request for the given hash. Most of the time, it's a search in a predefined list.
+    	/*!
-		// The output info_string is used by the turtle router to display info about tunnels it manages. It is
+         * \brief serviceId
-		// not passed to the tunnel.
+         * 			Returns the ID of the client service. This is used to pass the ID to search requests, from the client services
+         * \return
+         * 			The service ID.
+         */
 
-        virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; }
+    	virtual uint16_t serviceId() const
+		{
+			std::cerr << "!!!!!! Received request for service ID in turtle router client, but the client service is not handling it !!!!!!!" << std::endl ;
+            return 0 ;
+    	}
+
+		/*!
+		 * \brief handleTunnelRequest
+		              Handling of tunnel request for the given hash. To be derived by the service in order to tell the turtle router
+                      whether the service handles this hash or not. Most of the time, it's a search in a predefined list.
+
+		 * \return true if the service
+		 */
+		virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; }
 		
-		// This method is called by the turtle router to send data that comes out of a turtle tunnel.
+	    /*!
-		// The turtle router stays responsible for the memory management of data. Most of the  time the
+		 * \brief receiveTurtleData
-		// data chunk is a serialized item to be de-serialized by the client service.
+		 *           This method is called by the turtle router to send data that comes out of a turtle tunnel, and should
-		//
+		 * 			 be overloaded by the client service.
-		// Parameters:
+		 *           The turtle router stays responsible for the memory management of data. Most of the  time the
-		// 		virtual_peer_id	: name of the tunnel that sent the data
+		 *           data chunk is a serialized item to be de-serialized by the client service.
-		// 		data					: memory chunk for the data
+		 *
-		// 		size					: size of data
+		 *           Parameters:
-		// 		item->direction	: direction of travel:
+		 *           		virtual_peer_id	: name of the tunnel that sent the data
-		// 										RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client 
+		 *           		data					: memory chunk for the data
-		// 										RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server
+		 *           		size					: size of data
-		//
+		 *           		item->direction	: direction of travel:
-		// 			Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go 
+		 *           										RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client
-		// 			both ways, and their nature cannot suffice to determine where they should be handled.
+		 *           										RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server
-		//
+		 *
-		// By default (if not overloaded), the method will just free the data, as any subclass should do as well.
+		 *           			Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go
-		// Note: p3turtle stays owner of the item, so the client should not delete it!
+		 *           			both ways, and their nature cannot suffice to determine where they should be handled.
-		//
+		 *
-        virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
+		 *           By default (if not overloaded), the method will just free the data, as any subclass should do as well.
+		 *           Note: p3turtle stays owner of the item, so the client should not delete it!
+		 */
+        virtual void receiveTurtleData(const RsTurtleGenericTunnelItem * /* item */,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
 		{ 
 			std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ; 
 		}
 
-		// Method for creating specific items of the client service. The
+    	/*!
-		// method has a default behavior of not doing anything, since most client
+         * \brief receiveSearchRequest
-		// services might only use the generic item already provided by the turtle
+         * 			This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
-		// router: RsTurtleGenericDataItem
+         * 			result contains the serialised generic result returned by the client.
+         *
+         * 			The turtle router keeps the memory ownership over search_request_data
+         *
+         * \param search_request_data      generic serialized search data
+         * \param search_request_data_len  length of the serialized search data
+         * \param search_result_data       generic serialized search result data
+         * \param search_result_data_len   length of the serialized search result data
+         * \param max_allowed_hits         max number of hits allowed to be sent back and forwarded
+         *
+         * \return true if the search is successful.
+         */
+    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
+                                          uint32_t /*search_request_data_len*/,
+                                          unsigned char *& /*search_result_data*/,
+                                          uint32_t& /*search_result_data_len*/,
+                                          uint32_t& /* max_allows_hits */)
+		{
+			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+            return false;
+		}
 
+    	/*!
+         * \brief receiveSearchResult
+         * 			This method is called by the turtle router to notify the client of a search result. The result is serialized for the current class to read.
+         *
+         * \param search_result_data  result data. Memory ownership is owned by the turtle router. So do not delete!
+         * \param search_result_data  length of result data
+         */
+    	virtual void receiveSearchResult(TurtleSearchRequestId /* request_id */,unsigned char * /*search_result_data*/,uint32_t  /*search_result_data_len*/)
+		{
+			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+		}
+
+	    /*!
+		 * \brief serializer
+		 *           Method for creating specific items of the client service. The
+		 *           method has a default behavior of not doing anything, since most client
+		 *           services might only use the generic item already provided by the turtle
+		 *           router: RsTurtleGenericDataItem
+		 *
+		 * \return the client's serializer is returned
+		 */
 		virtual RsServiceSerializer *serializer() { return NULL ; }
 
-		// These methods are called by the turtle router to add/remove virtual peers when tunnels are created/deleted
+		 /*!
-		//
+		 * \brief addVirtualPeer
+		 *           These methods are called by the turtle router to notify the client in order to add/remove virtual peers when tunnels are created/deleted
+		 *           These methods must be overloaded, because a service which does not care about tunel being openned or closed is not supposed to need tunnels.
+		 *
+		 * \param hash                hash that the tunnel responds to
+		 * \param virtual_peer_id     virtual peer id provided by turtle to allow the client to send data into this tunnel. This peer is related to the tunnel itself
+		 *                              rather than to its destination. As such, multiple peer ids may actually send data to the same computer because multiple tunnels
+		 *                              arrive at the same location.
+		 * \param dir                 dir indicates which side the cient will be talking to: CLIENT means that the client is the server. SERVER means that the client acts
+		 *                            as a client (and therefore actually requested the tunnel).
+		 */
 		virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) = 0 ;
 		virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) = 0 ;
 
-		// This function is mandatory. It should do two things:
+	    /*!
-		// 	1 - keep a pointer to the turtle router, so as to be able to send data (e.g. copy pt into a local variable)
+		 * \brief connectToTurtleRouter
-		// 	2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it.
+		 *           This function must be overloaded by the client. It should do two things:
-		//
+		 *           	1 - keep a pointer to the turtle router, so as to be able to send data (e.g. store pt into a local variable)
+		 *           	2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it.
+		 *
+		 * \param pt A pointer to the turtle router.
+		 */
 		virtual void connectToTurtleRouter(p3turtle *pt) = 0 ;
 };
 

libretroshare/src/use_libretroshare.pri

@@ -16,7 +16,15 @@ bitdht {
 # when rapidjson is mainstream on all distribs, we will not need the sources
 # anymore in the meantime, they are part of the RS directory so that it is
 # always possible to find them
-INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0))
+RAPIDJSON_AVAILABLE = $$system(pkg-config --atleast-version 1.1 RapidJSON && echo yes)
+isEmpty(RAPIDJSON_AVAILABLE) {
+    message("using built-in rapidjson")
+    INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0))
+} else {
+    message("using systems rapidjson")
+    DEFINES *= HAS_RAPIDJSON
+}
+
 
 sLibs =
 mLibs = $$RS_SQL_LIB ssl crypto $$RS_THREAD_LIB $$RS_UPNP_LIB

retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp

@@ -165,8 +165,10 @@ SearchDialog::SearchDialog(QWidget *parent)
     QHeaderView_setSectionResizeModeColumn(_smheader, SS_KEYWORDS_COL, QHeaderView::Interactive);
     QHeaderView_setSectionResizeModeColumn(_smheader, SS_RESULTS_COL, QHeaderView::Interactive);
 
-    _smheader->resizeSection ( SS_KEYWORDS_COL, 160 );
+    float f = QFontMetricsF(font()).height()/14.0 ;
-    _smheader->resizeSection ( SS_RESULTS_COL, 50 );
+
+    _smheader->resizeSection ( SS_KEYWORDS_COL, 160*f );
+    _smheader->resizeSection ( SS_RESULTS_COL, 50*f );
 
     ui.searchResultWidget->setColumnCount(SR_COL_COUNT);
     _smheader = ui.searchResultWidget->header () ;
@@ -174,12 +176,12 @@ SearchDialog::SearchDialog(QWidget *parent)
     QHeaderView_setSectionResizeModeColumn(_smheader, SR_SIZE_COL, QHeaderView::Interactive);
     QHeaderView_setSectionResizeModeColumn(_smheader, SR_SOURCES_COL, QHeaderView::Interactive);
 
-    _smheader->resizeSection ( SR_NAME_COL, 240 );
+    _smheader->resizeSection ( SR_NAME_COL, 240*f );
-    _smheader->resizeSection ( SR_SIZE_COL, 75 );
+    _smheader->resizeSection ( SR_SIZE_COL, 75*f );
-    _smheader->resizeSection ( SR_SOURCES_COL, 75 );
+    _smheader->resizeSection ( SR_SOURCES_COL, 75*f );
-    _smheader->resizeSection ( SR_TYPE_COL, 75 );
+    _smheader->resizeSection ( SR_TYPE_COL, 75*f );
-    _smheader->resizeSection ( SR_AGE_COL, 90 );
+    _smheader->resizeSection ( SR_AGE_COL, 90*f );
-    _smheader->resizeSection ( SR_HASH_COL, 240 );
+    _smheader->resizeSection ( SR_HASH_COL, 240*f );
 
     // set header text aligment
     QTreeWidgetItem * headerItem = ui.searchResultWidget->headerItem();
@@ -201,10 +203,10 @@ SearchDialog::SearchDialog(QWidget *parent)
     // load settings
     processSettings(true);
   
-  	ui._ownFiles_CB->setMinimumWidth(20);
+  	ui._ownFiles_CB->setMinimumWidth(20*f);
-  	ui._friendListsearch_SB->setMinimumWidth(20);
+  	ui._friendListsearch_SB->setMinimumWidth(20*f);
-    ui._anonF2Fsearch_CB->setMinimumWidth(20);
+    ui._anonF2Fsearch_CB->setMinimumWidth(20*f);
-    ui.label->setMinimumWidth(20);
+    ui.label->setMinimumWidth(20*f);
 
     // workaround for Qt bug, be solved in next Qt release 4.7.0
     // https://bugreports.qt-project.org/browse/QTBUG-8270
@@ -792,7 +794,7 @@ void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression)
     RsRegularExpression::LinearizedExpression e ;
 	expression->linearize(e) ;
 
-	TurtleRequestId req_id = rsTurtle->turtleSearch(e) ;
+	TurtleRequestId req_id = rsFiles->turtleSearch(e) ;
 
 	// This will act before turtle results come to the interface, thanks to the signals scheduling policy.
 	initSearchResult(QString::fromStdString(e.GetStrings()),req_id, ui.FileTypeComboBox->currentIndex(), true) ;
@@ -858,9 +860,9 @@ void SearchDialog::searchKeywords(const QString& keywords)
 	if(ui._anonF2Fsearch_CB->isChecked())
 	{
 		if(n==1)
-			req_id = rsTurtle->turtleSearch(words.front()) ;
+			req_id = rsFiles->turtleSearch(words.front()) ;
 		else
-			req_id = rsTurtle->turtleSearch(lin_exp) ;
+			req_id = rsFiles->turtleSearch(lin_exp) ;
 	}
 	else
 		req_id = RSRandom::random_u32() ; // generate a random 32 bits request id

retroshare-gui/src/gui/FileTransfer/SearchDialog.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>758</width>
+    <width>1531</width>
-    <height>339</height>
+    <height>889</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -32,7 +32,16 @@
       <enum>QFrame::Sunken</enum>
      </property>
      <layout class="QHBoxLayout" name="horizontalLayout_4">
-      <property name="margin">
+      <property name="leftMargin">
+       <number>2</number>
+      </property>
+      <property name="topMargin">
+       <number>2</number>
+      </property>
+      <property name="rightMargin">
+       <number>2</number>
+      </property>
+      <property name="bottomMargin">
        <number>2</number>
       </property>
       <item>
@@ -55,15 +64,24 @@
            </size>
           </property>
           <layout class="QGridLayout" name="gridLayout">
+           <property name="leftMargin">
+            <number>0</number>
+           </property>
+           <property name="topMargin">
+            <number>0</number>
+           </property>
+           <property name="rightMargin">
+            <number>0</number>
+           </property>
+           <property name="bottomMargin">
+            <number>0</number>
+           </property>
            <property name="horizontalSpacing">
             <number>0</number>
            </property>
            <property name="verticalSpacing">
             <number>1</number>
            </property>
-           <property name="margin">
-            <number>0</number>
-           </property>
            <item row="0" column="0">
             <widget class="LineEditClear" name="lineEdit">
              <property name="toolTip">
@@ -307,7 +325,7 @@
              <string>Any</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeAny.png</normaloff>:/images/FileTypeAny.png</iconset>
             </property>
            </item>
@@ -316,7 +334,7 @@
              <string>Archive</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeArchive.png</normaloff>:/images/FileTypeArchive.png</iconset>
             </property>
            </item>
@@ -325,7 +343,7 @@
              <string>Audio</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeAudio.png</normaloff>:/images/FileTypeAudio.png</iconset>
             </property>
            </item>
@@ -334,7 +352,7 @@
              <string>CD-Image</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeCDImage.png</normaloff>:/images/FileTypeCDImage.png</iconset>
             </property>
            </item>
@@ -343,7 +361,7 @@
              <string>Document</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeDocument.png</normaloff>:/images/FileTypeDocument.png</iconset>
             </property>
            </item>
@@ -352,7 +370,7 @@
              <string>Picture</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypePicture.png</normaloff>:/images/FileTypePicture.png</iconset>
             </property>
            </item>
@@ -361,7 +379,7 @@
              <string>Program</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeProgram.png</normaloff>:/images/FileTypeProgram.png</iconset>
             </property>
            </item>
@@ -370,7 +388,7 @@
              <string>Video</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/FileTypeVideo.png</normaloff>:/images/FileTypeVideo.png</iconset>
             </property>
            </item>
@@ -379,7 +397,7 @@
              <string>Directory</string>
             </property>
             <property name="icon">
-             <iconset resource="images.qrc">
+             <iconset resource="../images.qrc">
               <normaloff>:/images/folder16.png</normaloff>:/images/folder16.png</iconset>
             </property>
            </item>
@@ -406,7 +424,7 @@
             <string>Download selected</string>
            </property>
            <property name="icon">
-            <iconset resource="images.qrc">
+            <iconset resource="../images.qrc">
              <normaloff>:/images/download16.png</normaloff>:/images/download16.png</iconset>
            </property>
           </widget>
@@ -423,7 +441,7 @@
   <customwidget>
    <class>LineEditClear</class>
    <extends>QLineEdit</extends>
-   <header>gui/common/LineEditClear.h</header>
+   <header location="global">gui/common/LineEditClear.h</header>
   </customwidget>
   <customwidget>
    <class>SearchTreeWidget</class>
@@ -432,7 +450,7 @@
   </customwidget>
  </customwidgets>
  <resources>
-  <include location="images.qrc"/>
+  <include location="../images.qrc"/>
  </resources>
  <connections/>
 </ui>

retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp

@@ -1140,14 +1140,14 @@ void LocalSharedFilesDialog::spawnCustomPopupMenu( QPoint point )
 	{
 		shareChannelMenu.setIcon(QIcon(IMAGE_CHANNEL));
 
-		std::list<RsGroupMetaData> grp_metas ;
+		std::map<RsGxsGroupId,RsGroupMetaData> grp_metas ;
 		channelDialog->getGroupList(grp_metas) ;
 
 		std::vector<std::pair<std::string,RsGxsGroupId> > grplist ; // I dont use a std::map because two or more channels may have the same name.
 
 		for(auto it(grp_metas.begin());it!=grp_metas.end();++it)
-			if(IS_GROUP_PUBLISHER((*it).mSubscribeFlags) && IS_GROUP_SUBSCRIBED((*it).mSubscribeFlags))
+			if(IS_GROUP_PUBLISHER((*it).second.mSubscribeFlags) && IS_GROUP_SUBSCRIBED((*it).second.mSubscribeFlags))
-				grplist.push_back(std::make_pair((*it).mGroupName, (*it).mGroupId));
+				grplist.push_back(std::make_pair((*it).second.mGroupName, (*it).second.mGroupId));
 
 		std::sort(grplist.begin(),grplist.end(),ChannelCompare()) ;
 
@@ -1164,14 +1164,14 @@ void LocalSharedFilesDialog::spawnCustomPopupMenu( QPoint point )
 	{
 		shareForumMenu.setIcon(QIcon(IMAGE_FORUMS));
 
-		std::list<RsGroupMetaData> grp_metas ;
+		std::map<RsGxsGroupId,RsGroupMetaData> grp_metas ;
 		forumsDialog->getGroupList(grp_metas) ;
 
 		std::vector<std::pair<std::string,RsGxsGroupId> > grplist ; // I dont use a std::map because two or more channels may have the same name.
 
 		for(auto it(grp_metas.begin());it!=grp_metas.end();++it)
-			if(IS_GROUP_SUBSCRIBED((*it).mSubscribeFlags))
+			if(IS_GROUP_SUBSCRIBED((*it).second.mSubscribeFlags))
-				grplist.push_back(std::make_pair((*it).mGroupName, (*it).mGroupId));
+				grplist.push_back(std::make_pair((*it).second.mGroupName, (*it).second.mGroupId));
 
 		std::sort(grplist.begin(),grplist.end(),ChannelCompare()) ;
 

retroshare-gui/src/gui/MainWindow.cpp

@@ -433,7 +433,7 @@ void MainWindow::initStackedPage()
 
 #ifndef RS_RELEASE_VERSION
 #ifdef PLUGINMGR
-  addPage(pluginsPage = new PluginsPage(ui->stackPages), grp, NULL);
+  addPage(pluginsPage = new gui::PluginsPage(ui->stackPages), grp, NULL);
 #endif
 #endif
 
@@ -643,10 +643,10 @@ const QList<UserNotify*> &MainWindow::getUserNotifyList()
 
 /*static*/ void MainWindow::displayLobbySystrayMsg(const QString& title,const QString& msg)
 {
-    if (_instance == NULL) 
+    if (_instance == NULL)
         return;
 
-    if(Settings->getDisplayTrayChatLobby()) 
+    if(Settings->getDisplayTrayChatLobby())
 		 _instance->displaySystrayMsg(title,msg) ;
 }
 
@@ -1011,7 +1011,7 @@ void SetForegroundWindowInternal(HWND hWnd)
        return NULL;
    }
 
-   switch (page) 
+   switch (page)
 	{
 		case Network:
 			return _instance->friendsDialog->networkDialog;
@@ -1457,7 +1457,7 @@ void MainWindow::externalLinkActivated(const QUrl &url)
 
 		int res = mb.exec() ;
 
-		if (res == QMessageBox::No) 
+		if (res == QMessageBox::No)
 			return ;
 
 		if(dontAsk_CB->isChecked())

retroshare-gui/src/gui/PluginsPage.h

@@ -15,7 +15,7 @@
  *
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, 
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
 
@@ -39,6 +39,8 @@ class QScriptEngine;
 
 class PluginManager;
 
+namespace gui {
+
 
 //! A demo widget for showing plugin engine in action :)
 
@@ -46,7 +48,7 @@ class PluginManager;
 //! loaded plugin widgets. All specific actions moved to
 //! PluginManagerWidget class. It contains a PluginManager instance, but it's
 //! supposed that in future a pluginManager will become a global variable
-class PluginsPage : public MainPage 
+class PluginsPage : public MainPage
 {
     Q_OBJECT
 
@@ -70,10 +72,10 @@ protected:
     QVBoxLayout* pluginPageLayout;
     QGroupBox* pluginPanel;
     QVBoxLayout* pluginPanelLayout;
-  
+
     //! Plugin widgets will be loaded into this tabs
     QTabWidget* pluginTabs ;
-    
+
     QVBoxLayout* pmLay;
     QFrame* pmFrame;
     QSpacerItem* pmSpacer;
@@ -82,5 +84,7 @@ protected:
     PluginManager* pluginManager;
 };
 
+} // namespace gui
+
 #endif
 

retroshare-gui/src/gui/Posted/PostedListWidget.cpp

@@ -454,6 +454,10 @@ void PostedListWidget::applyRanking()
 	ui->scrollAreaWidgetContents->update();
 }
 
+void PostedListWidget::blank()
+{
+    clearPosts();
+}
 void PostedListWidget::clearPosts()
 {
 	/* clear all messages */

retroshare-gui/src/gui/Posted/PostedListWidget.h

@@ -63,6 +63,7 @@ protected:
 	virtual void insertAllPosts(const uint32_t &token, GxsMessageFramePostThread *thread);
 	virtual void insertPosts(const uint32_t &token);
 	virtual void clearPosts();
+	virtual void blank();
 	virtual bool navigatePostItem(const RsGxsMessageId& msgId);
 
 	/* GxsMessageFrameWidget */

retroshare-gui/src/gui/RetroShareLink.cpp

@@ -860,17 +860,17 @@ QString RetroShareLink::title() const
 		break;
 
 		case TYPE_FILE:
-		return QString("%1 (%2)").arg(hash()).arg(misc::friendlyUnit(size()));
+		return QString("Size: %2  hash: %1").arg(hash()).arg(misc::friendlyUnit(size()));
 
 		case TYPE_PERSON:
 		return PeerDefs::rsidFromId(RsPgpId(hash().toStdString()));
 
 		case TYPE_FORUM:
-			/* fallthrough */
+			return QString("Forum id: %1").arg(hash());
 		case TYPE_CHANNEL:
-			/* fallthrough */
+			return QString("Channel id: %1").arg(hash());
 		case TYPE_SEARCH:
-		break;
+			return QString("Search files");
 
 		case TYPE_MESSAGE:
 		return PeerDefs::rsidFromId(RsPeerId(hash().toStdString()));
@@ -1386,13 +1386,6 @@ static void processList(const QStringList &list, const QString &textSingular, co
 				++countUnknown;
 			break;
 
-			case TYPE_FILE:
-			{
-				col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ;
-				fileLinkFound = true;
-			}
-			break;
-
 			case TYPE_PERSON:
 			{
 #ifdef DEBUG_RSLINK
@@ -1547,6 +1540,22 @@ static void processList(const QStringList &list, const QString &textSingular, co
 			}
 			break ;
 
+			case TYPE_FILE:
+			{
+				FileInfo fi1;
+				if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1)
+					&& !link.name().endsWith(RsCollection::ExtensionString))
+				{
+					/* fallthrough */
+				}
+				else
+				{
+					col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ;
+					fileLinkFound = true;
+					break;
+				}
+			}
+			//break;
 			case TYPE_EXTRAFILE:
 			{
 #ifdef DEBUG_RSLINK
@@ -1621,14 +1630,14 @@ static void processList(const QStringList &list, const QString &textSingular, co
 						} else if (ret == QMessageBox::NoToAll) {
 							dontOpenNextFile = true;
 						}
+						needNotifySuccess = false;
 					}
 				}
 
 				if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) {
 					fileAdded.append(link.name());
 				} else {
-					if (!bFileOpened) fileExist.append(link.name());
+					if (!bFileOpened && links.size()>1) fileExist.append(link.name());}
-				}
 			}
 			break;
 

retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp

@@ -2,7 +2,7 @@
  *
  *  RetroShare is distributed under the following license:
  *
- *  Copyright (C) 2011, csoler  
+ *  Copyright (C) 2011, csoler
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
@@ -16,7 +16,7 @@
  *
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, 
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
  *  Boston, MA  02110-1301, USA.
  *
  ****************************************************************/
@@ -80,14 +80,14 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
 
 	connect(ui.filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterChanged(QString)));
 
-            int S = QFontMetricsF(font()).height() ;
+		int S = QFontMetricsF(font()).height() ;
     ui.participantsList->setIconSize(QSize(1.4*S,1.4*S));
 
     ui.participantsList->setColumnCount(COLUMN_COUNT);
     ui.participantsList->setColumnWidth(COLUMN_ICON, 1.7*S);
     ui.participantsList->setColumnHidden(COLUMN_ACTIVITY,true);
     ui.participantsList->setColumnHidden(COLUMN_ID,true);
-    
+
     /* Set header resize modes and initial section sizes */
 	QHeaderView * header = ui.participantsList->header();
 	QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Stretch);
@@ -99,16 +99,16 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
     distantChatAct = new QAction(QIcon(":/images/chat_24.png"), tr("Start private chat"), this);
     sendMessageAct = new QAction(QIcon(":/images/mail_new.png"), tr("Send Message"), this);
     showInPeopleAct = new QAction(QIcon(), tr("Show author in people tab"), this);
-	
+
     QActionGroup *sortgrp = new QActionGroup(this);
     actionSortByName = new QAction(QIcon(), tr("Sort by Name"), this);
     actionSortByName->setCheckable(true);
-    actionSortByName->setChecked(true); 
+    actionSortByName->setChecked(true);
     actionSortByName->setActionGroup(sortgrp);
 
     actionSortByActivity = new QAction(QIcon(), tr("Sort by Activity"), this);
     actionSortByActivity->setCheckable(true);
-    actionSortByActivity->setChecked(false); 
+    actionSortByActivity->setChecked(false);
     actionSortByActivity->setActionGroup(sortgrp);
 
 
@@ -122,29 +122,33 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
 
     connect(actionSortByName, SIGNAL(triggered()), this, SLOT(sortParcipants()));
     connect(actionSortByActivity, SIGNAL(triggered()), this, SLOT(sortParcipants()));
-    
+
     	/* Add filter actions */
 	QTreeWidgetItem *headerItem = ui.participantsList->headerItem();
 	QString headerText = headerItem->text(COLUMN_NAME );
 	ui.filterLineEdit->addFilter(QIcon(), headerText, COLUMN_NAME , QString("%1 %2").arg(tr("Search"), headerText));
 
+	// just empiric values
+	double scaler_factor = S > 25 ? 2.4 : 1.8;
+	QSize icon_size(scaler_factor * S, scaler_factor * S);
+
 	// Add a button to invite friends.
 	//
 	inviteFriendsButton = new QToolButton ;
-  inviteFriendsButton->setMinimumSize(QSize(2.4*S,2.4*S)) ;
+	inviteFriendsButton->setMinimumSize(icon_size);
-  inviteFriendsButton->setMaximumSize(QSize(2.4*S,2.4*S)) ;
+	inviteFriendsButton->setMaximumSize(icon_size);
 	inviteFriendsButton->setText(QString()) ;
 	inviteFriendsButton->setAutoRaise(true) ;
 	inviteFriendsButton->setToolTip(tr("Invite friends to this lobby"));
 
-        mParticipantCompareRole = new RSTreeWidgetItemCompareRole;
+	mParticipantCompareRole = new RSTreeWidgetItemCompareRole;
-        mParticipantCompareRole->setRole(COLUMN_ACTIVITY, ROLE_SORT);
+	mParticipantCompareRole->setRole(COLUMN_ACTIVITY, ROLE_SORT);
 
 	{
 	QIcon icon ;
 	icon.addPixmap(QPixmap(":/icons/png/invite.png")) ;
 	inviteFriendsButton->setIcon(icon) ;
-    inviteFriendsButton->setIconSize(QSize(2.4*S,2.4*S)) ;
+	inviteFriendsButton->setIconSize(icon_size);
 	}
 
 	connect(inviteFriendsButton, SIGNAL(clicked()), this , SLOT(inviteFriends()));
@@ -163,7 +167,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
     }
     ownIdChooser = new GxsIdChooser() ;
     ownIdChooser->loadIds(idChooserFlag, current_id) ;
-    
+
     QWidgetAction *checkableAction = new QWidgetAction(this);
     checkableAction->setDefaultWidget(ownIdChooser);
 
@@ -175,9 +179,9 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
 
     connect(ownIdChooser,SIGNAL(currentIndexChanged(int)),this,SLOT(changeNickname())) ;
 
-    unsubscribeButton = new QToolButton ;
+	unsubscribeButton = new QToolButton;
-    unsubscribeButton->setMinimumSize(QSize(2.4*S,2.4*S)) ;
+	unsubscribeButton->setMinimumSize(icon_size);
-    unsubscribeButton->setMaximumSize(QSize(2.4*S,2.4*S)) ;
+	unsubscribeButton->setMaximumSize(icon_size);
 	unsubscribeButton->setText(QString()) ;
 	unsubscribeButton->setAutoRaise(true) ;
 	unsubscribeButton->setToolTip(tr("Leave this chat room (Unsubscribe)"));
@@ -186,7 +190,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi
 	QIcon icon ;
 	icon.addPixmap(QPixmap(":/icons/png/leave.png")) ;
 	unsubscribeButton->setIcon(icon) ;
-    unsubscribeButton->setIconSize(QSize(2.4*S,2.4*S)) ;
+	unsubscribeButton->setIconSize(icon_size);
 	}
 
 	/* Initialize splitter */
@@ -246,8 +250,8 @@ void ChatLobbyDialog::participantsTreeWidgetCustomPopupMenu(QPoint)
 
 void ChatLobbyDialog::textBrowserAskContextMenu(QMenu* contextMnu, QString anchorForPosition, const QPoint /*point*/)
 {
-	if (anchorForPosition.startsWith(PERSONID)){
+	if (anchorForPosition.startsWith(PERSONID)) {
-		QString strId = anchorForPosition.replace(PERSONID,"");
+		QString strId = anchorForPosition.replace(PERSONID, "");
 		if (strId.contains(" "))
 			strId.truncate(strId.indexOf(" "));
 
@@ -263,6 +267,8 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QList<RsGxs
 {
 	if (!contextMnu)
 		return;
+	if (idList.isEmpty())
+		return;
 
 	contextMnu->addAction(distantChatAct);
 	contextMnu->addAction(sendMessageAct);
@@ -281,7 +287,7 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QList<RsGxs
 	votePositiveAct->setEnabled(false);
 	voteNeutralAct->setEnabled(false);
 	voteNegativeAct->setEnabled(false);
-	showInPeopleAct->setEnabled(idList.count()==1);
+	showInPeopleAct->setEnabled(idList.count() == 1);
 
 	distantChatAct->setData(QVariant::fromValue(idList));
 	sendMessageAct->setData(QVariant::fromValue(idList));
@@ -291,19 +297,16 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QList<RsGxs
 	voteNegativeAct->setData(QVariant::fromValue(idList));
 	showInPeopleAct->setData(QVariant::fromValue(idList));
 
-	if(idList.count()==1)
+	RsGxsId gxsid = idList.at(0);
-	{
-		RsGxsId gxsid = idList.at(0);
 
-		if(!gxsid.isNull() && !rsIdentity->isOwnId(gxsid))
+	if(!gxsid.isNull() && !rsIdentity->isOwnId(gxsid))
-		{
+	{
-			distantChatAct->setEnabled(true);
+		distantChatAct->setEnabled(true);
-			votePositiveAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_POSITIVE);
+		votePositiveAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_POSITIVE);
-			voteNeutralAct->setEnabled((rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_POSITIVE) || (rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_NEGATIVE) );
+		voteNeutralAct->setEnabled((rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_POSITIVE) || (rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_NEGATIVE) );
-			voteNegativeAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_NEGATIVE);
+		voteNegativeAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_NEGATIVE);
-			muteAct->setEnabled(true);
+		muteAct->setEnabled(true);
-			muteAct->setChecked(isParticipantMuted(gxsid));
+		muteAct->setChecked(isParticipantMuted(gxsid));
-		}
 	}
 }
 
@@ -443,17 +446,17 @@ void ChatLobbyDialog::processSettings(bool load)
 
 		// state of splitter
 		ui.splitter->restoreState(Settings->value("splitter").toByteArray());
-		
+
 		// load sorting
 		actionSortByActivity->setChecked(Settings->value("sortbyActivity", QVariant(false)).toBool());
 		actionSortByName->setChecked(Settings->value("sortbyName", QVariant(true)).toBool());
-		
+
 	} else {
 		// save settings
 
 		// state of splitter
 		Settings->setValue("splitter", ui.splitter->saveState());
-		
+
 		//save sorting
 		Settings->setValue("sortbyActivity", actionSortByActivity->isChecked());
 		Settings->setValue("sortbyName", actionSortByName->isChecked());
@@ -464,7 +467,7 @@ void ChatLobbyDialog::processSettings(bool load)
 
 /**
  * Change your Nickname
- * 
+ *
  * - send a Message to all Members => later: send hidden message to clients, so they can actualize there mutedParticipants list
  */
 void ChatLobbyDialog::setIdentity(const RsGxsId& gxs_id)
@@ -500,7 +503,7 @@ void ChatLobbyDialog::changeNickname()
 
 /**
  * We get a new Message from a chat participant
- * 
+ *
  * - Ignore Messages from muted chat participants
  */
 void ChatLobbyDialog::addChatMsg(const ChatMessage& msg)
@@ -509,7 +512,7 @@ void ChatLobbyDialog::addChatMsg(const ChatMessage& msg)
     QDateTime recvTime = QDateTime::fromTime_t(msg.recvTime);
     QString message = QString::fromUtf8(msg.msg.c_str());
     RsGxsId gxs_id = msg.lobby_peer_gxs_id ;
-	
+
     if(!isParticipantMuted(gxs_id))
     {
         // We could change addChatMsg to display the peers icon, passing a ChatId
@@ -551,7 +554,7 @@ void ChatLobbyDialog::addChatMsg(const ChatMessage& msg)
 
 /**
  * Regenerate the QTreeWidget participant list of a Chat Lobby
- * 
+ *
  * Show yellow icon for muted Participants
  */
 void ChatLobbyDialog::updateParticipantsList()
@@ -602,7 +605,7 @@ void ChatLobbyDialog::updateParticipantsList()
 
             time_t tLastAct=widgetitem->text(COLUMN_ACTIVITY).toInt();
             time_t now = time(NULL);
-            
+
                 widgetitem->setSizeHint(COLUMN_ICON, QSize(20,20));
 
 
@@ -619,7 +622,7 @@ void ChatLobbyDialog::updateParticipantsList()
             if (RsGxsId(participant.toStdString()) == gxs_id) widgetitem->setIcon(COLUMN_ICON, bullet_yellow_128);
 
 	    widgetitem->updateBannedState();
-        
+
             QTime qtLastAct=QTime(0,0,0).addSecs(now-tLastAct);
             widgetitem->setToolTip(COLUMN_ICON,tr("Right click to mute/unmute participants<br/>Double click to address this person<br/>")
                                    +tr("This participant is not active since:")
@@ -635,7 +638,7 @@ void ChatLobbyDialog::updateParticipantsList()
 
 /**
  * Called when a Participant get Clicked / Changed
- * 
+ *
  * Check if the Checkbox altered and Mute User
  */
 void ChatLobbyDialog::changeParticipationState()
@@ -787,15 +790,15 @@ bool ChatLobbyDialog::isNicknameInLobby(const RsGxsId& nickname)
     return clinfo.gxs_ids.find(nickname) != clinfo.gxs_ids.end() ;
 }
 
-/** 
+/**
  * Should Messages from this Nickname be muted?
- * 
+ *
  * At the moment it is not possible to 100% know which peer sendet the message, and only
- * the nickname is available. So this couldn't work for 100%. So, for example,  if a peer 
+ * the nickname is available. So this couldn't work for 100%. So, for example,  if a peer
- * change his name to the name of a other peer, we couldn't block him. A real implementation 
+ * change his name to the name of a other peer, we couldn't block him. A real implementation
  * will be possible if we transfer a temporary Session ID from the sending Retroshare client
  * version 0.6
- * 
+ *
  * @param QString nickname to check
  */
 bool ChatLobbyDialog::isParticipantMuted(const RsGxsId& participant)
@@ -901,7 +904,7 @@ bool ChatLobbyDialog::canClose()
 
 void ChatLobbyDialog::showDialog(uint chatflags)
 {
-	if (chatflags & RS_CHAT_FOCUS) 
+	if (chatflags & RS_CHAT_FOCUS)
 	{
 		MainWindow::showWindow(MainWindow::ChatLobby);
 		dynamic_cast<ChatLobbyWidget*>(MainWindow::getPage(MainWindow::ChatLobby))->setCurrentChatPage(this) ;
@@ -916,7 +919,7 @@ void ChatLobbyDialog::sortParcipants()
 	} else if (actionSortByName->isChecked()) {
         ui.participantsList->sortItems(COLUMN_NAME, Qt::AscendingOrder);
 	}
-  
+
 }
 
 void ChatLobbyDialog::filterChanged(const QString& /*text*/)

retroshare-gui/src/gui/chat/ChatWidget.cpp

@@ -65,6 +65,8 @@
 #include <time.h>
 
 #define FMM 2.5//fontMetricsMultiplicator
+#define FMM_SMALLER 1.8
+#define FMM_THRESHOLD 25
 
 /*****
  * #define CHAT_DEBUG 1
@@ -75,9 +77,12 @@ ChatWidget::ChatWidget(QWidget *parent) :
 {
 	ui->setupUi(this);
 
-	int iconHeight = FMM*QFontMetricsF(font()).height() ;
+	int iconHeight = QFontMetricsF(font()).height();
-	QSize iconSize = QSize(iconHeight,iconHeight);
+	double fmm = iconHeight > FMM_THRESHOLD ? FMM : FMM_SMALLER;
-	QSize buttonSize = QSize(iconSize + QSize((int)FMM,(int)FMM));
+	iconHeight *= fmm;
+	QSize iconSize = QSize(iconHeight, iconHeight);
+	int butt_size(iconSize.height() + fmm);
+	QSize buttonSize = QSize(butt_size, butt_size);
 
 	newMessages = false;
 	typing = false;
@@ -110,7 +115,7 @@ ChatWidget::ChatWidget(QWidget *parent) :
 	ui->searchButton->setIconSize(iconSize);
 	ui->sendButton->setFixedHeight(iconHeight);
 	ui->sendButton->setIconSize(iconSize);
-  
+
 	//Initialize search
 	iCharToStartSearch=Settings->getChatSearchCharToStartSearch();
 	bFindCaseSensitively=Settings->getChatSearchCaseSensitively();
@@ -177,7 +182,7 @@ ChatWidget::ChatWidget(QWidget *parent) :
 
 	ui->infoFrame->setVisible(false);
 	ui->statusMessageLabel->hide();
-	
+
 	setAcceptDrops(true);
 	ui->chatTextEdit->setAcceptDrops(false);
 	ui->hashBox->setDropWidget(this);
@@ -197,7 +202,7 @@ ChatWidget::ChatWidget(QWidget *parent) :
 	menu->addAction(ui->actionMessageHistory);
 	ui->pushtoolsButton->setMenu(menu);
 	menu->addMenu(fontmenu);
-	
+
 	ui->actionSendAsPlainText->setChecked(Settings->getChatSendAsPlainTextByDef());
 	ui->chatTextEdit->setOnlyPlainText(ui->actionSendAsPlainText->isChecked());
 	connect(ui->actionSendAsPlainText, SIGNAL(toggled(bool)), ui->chatTextEdit, SLOT(setOnlyPlainText(bool)) );
@@ -259,9 +264,12 @@ void ChatWidget::addChatHorizontalWidget(QWidget *w)
 
 void ChatWidget::addChatBarWidget(QWidget *w)
 {
-	int iconHeight = FMM*QFontMetricsF(font()).height() ;
+	int iconHeight = QFontMetricsF(font()).height();
-	QSize iconSize = QSize(iconHeight,iconHeight);
+	double fmm = iconHeight > FMM_THRESHOLD ? FMM : FMM_SMALLER;
-	QSize buttonSize = QSize(iconSize + QSize((int)FMM,(int)FMM));
+	iconHeight *= fmm;
+	QSize iconSize = QSize(iconHeight, iconHeight);
+	int butt_size(iconSize.height() + fmm);
+	QSize buttonSize = QSize(butt_size, butt_size);
 	w->setFixedSize(buttonSize);
 	ui->pluginButtonFrame->layout()->addWidget(w) ;
 }
@@ -357,12 +365,12 @@ void ChatWidget::init(const ChatId &chat_id, const QString &title)
         ui->titleBarFrame->setVisible(false);
     }
 
-	if (rsHistory->getEnable(hist_chat_type)) 
+	if (rsHistory->getEnable(hist_chat_type))
 	{
 		// get chat messages from history
 		std::list<HistoryMsg> historyMsgs;
 
-		if (messageCount > 0) 
+		if (messageCount > 0)
 		{
             rsHistory->getMessages(chatId, historyMsgs, messageCount);
 
@@ -376,7 +384,7 @@ void ChatWidget::init(const ChatId &chat_id, const QString &title)
                     continue;
 
                 QString name;
-                if (chatId.isLobbyId() || chatId.isDistantChatId()) 
+                if (chatId.isLobbyId() || chatId.isDistantChatId())
                 {
                     RsIdentityDetails details;
                     if (rsIdentity->getIdDetails(RsGxsId(historyIt->peerName), details))
@@ -962,7 +970,7 @@ void ChatWidget::addChatMsg(bool incoming, const QString &name, const RsGxsId gx
 	unsigned int formatFlag = 0;
 
     bool addDate = false;
-    if (QDate::currentDate()>lastMsgDate) 
+    if (QDate::currentDate()>lastMsgDate)
 	 {
 		 addDate=true;
     }
@@ -1095,7 +1103,8 @@ void ChatWidget::contextMenuTextBrowser(QPoint point)
 
 	contextMnu->addSeparator();
 	contextMnu->addAction(ui->actionClearChatHistory);
-	contextMnu->addAction(ui->actionQuote);
+	if (ui->textBrowser->textCursor().selection().toPlainText().length())
+		contextMnu->addAction(ui->actionQuote);
 	contextMnu->addAction(ui->actionDropPlacemark);
 
 	if(ui->textBrowser->checkImage(point))
@@ -1161,13 +1170,14 @@ void ChatWidget::resetStatusBar()
 
 void ChatWidget::updateStatusTyping()
 {
+	if(Settings->getChatDoNotSendIsTyping())
+		return;
 	if (time(NULL) - lastStatusSendTime > 5)	// limit 'peer is typing' packets to at most every 10 sec
 	{
 #ifdef ONLY_FOR_LINGUIST
 		tr("is typing...");
 #endif
-		if(!Settings->getChatDoNotSendIsTyping())
+		rsMsgs->sendStatusString(chatId, "is typing...");
-			rsMsgs->sendStatusString(chatId, "is typing...");
 		lastStatusSendTime = time(NULL) ;
 	}
 }
@@ -1673,7 +1683,7 @@ void ChatWidget::updateStatus(const QString &peer_id, int status)
         vpid = chatId.toPeerId();
 
 	/* set font size for status  */
-    if (peer_id.toStdString() == vpid.toStdString()) 
+    if (peer_id.toStdString() == vpid.toStdString())
     {
 	    // the peers status has changed
 
@@ -1816,13 +1826,10 @@ bool ChatWidget::setStyle()
 void ChatWidget::quote()
 {
 	QString text = ui->textBrowser->textCursor().selection().toPlainText();
-	if(text.length() > 0)
+	QStringList sl = text.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
-	{
+	text = sl.join("\n> ");
-		QStringList sl = text.split(QRegExp("[\r\n]"),QString::SkipEmptyParts);
+	text.replace(QChar(-4), " "); // Char used when image on text.
-		text = sl.join("\n> ");
+	emit ui->chatTextEdit->append(QString("> ") + text);
-		text.replace(QChar(-4)," ");//Char used when image on text.
-		emit ui->chatTextEdit->append(QString("> ") + text);
-	}
 }
 
 void ChatWidget::dropPlacemark()

retroshare-gui/src/gui/chat/CreateLobbyDialog.cpp

@@ -106,7 +106,7 @@ void CreateLobbyDialog::checkTextFields()
                     break ;
     }
     
-    RsIdentityDetails(idd) ;
+    RsIdentityDetails idd;
     
     rsIdentity->getIdDetails(id,idd) ;
     

retroshare-gui/src/gui/chat/CreateLobbyDialog.ui

@@ -11,17 +11,26 @@
    </rect>
   </property>
   <property name="windowTitle">
-   <string>Create Chat Lobby</string>
+   <string>Create Chat Room</string>
   </property>
   <property name="windowIcon">
    <iconset resource="../images.qrc">
     <normaloff>:/images/logo/logo_32.png</normaloff>:/images/logo/logo_32.png</iconset>
   </property>
-  <layout class="QVBoxLayout" name="verticalLayout_2">
+  <layout class="QVBoxLayout" name="CreateLobbyDialogVLayout">
    <property name="spacing">
     <number>0</number>
    </property>
-   <property name="margin">
+   <property name="leftMargin">
+    <number>0</number>
+   </property>
+   <property name="topMargin">
+    <number>0</number>
+   </property>
+   <property name="rightMargin">
+    <number>0</number>
+   </property>
+   <property name="bottomMargin">
     <number>0</number>
    </property>
    <item>
@@ -29,11 +38,11 @@
    </item>
    <item>
     <widget class="QFrame" name="frame">
-     <layout class="QVBoxLayout" name="verticalLayout_3">
+     <layout class="QVBoxLayout" name="frameVLayout">
       <item>
        <widget class="StyledLabel" name="lobbyInfoLabel">
         <property name="text">
-         <string>A chat lobby is a decentralized and anonymous chat group. All participants receive all messages. Once the lobby is created you can invite other friends from the Friends tab.</string>
+         <string>A chat room is a decentralized and anonymous chat group. All participants receive all messages. Once the room is created you can invite other friend nodes with invite button on top right.</string>
         </property>
         <property name="wordWrap">
          <bool>true</bool>
@@ -41,16 +50,16 @@
        </widget>
       </item>
       <item>
-       <layout class="QHBoxLayout" name="horizontalLayout_2">
+       <layout class="QHBoxLayout" name="mainHLayout">
         <item>
-         <widget class="QFrame" name="lobbyFrame">
+         <widget class="QFrame" name="roomFrame">
-          <layout class="QVBoxLayout" name="verticalLayout">
+          <layout class="QVBoxLayout" name="roomFrameVLayout">
            <item>
-            <layout class="QGridLayout" name="gridLayout_3">
+            <layout class="QGridLayout" name="roomGLayout">
              <item row="0" column="0">
-              <widget class="QLabel" name="label_3">
+              <widget class="QLabel" name="lobbyName_Lb">
                <property name="text">
-                <string>Lobby name:</string>
+                <string>Room name:</string>
                </property>
                <property name="alignment">
                 <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
@@ -61,9 +70,9 @@
               <widget class="QLineEdit" name="lobbyName_LE"/>
              </item>
              <item row="1" column="0">
-              <widget class="QLabel" name="label_7">
+              <widget class="QLabel" name="lobbyTopic_Lb">
                <property name="text">
-                <string>Lobby topic:</string>
+                <string>Room topic:</string>
                </property>
                <property name="alignment">
                 <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
@@ -74,7 +83,7 @@
               <widget class="QLineEdit" name="lobbyTopic_LE"/>
              </item>
              <item row="2" column="0">
-              <widget class="QLabel" name="label_4">
+              <widget class="QLabel" name="idChooser_Lb">
                <property name="text">
                 <string>Identity to use:</string>
                </property>
@@ -84,7 +93,7 @@
               </widget>
              </item>
              <item row="3" column="0">
-              <widget class="QLabel" name="label_6">
+              <widget class="QLabel" name="security_Lb">
                <property name="text">
                 <string>Visibility:</string>
                </property>
@@ -113,7 +122,7 @@
              <item row="4" column="2">
               <widget class="QCheckBox" name="pgp_signed_CB">
                <property name="toolTip">
-                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If you check this, only PGP-signed ids can be used to join and talk in this lobby. This limitation prevents anonymous spamming as it becomes possible for at least some people in the lobby to locate the spammer's node.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If you check this, only PGP-signed ids can be used to join and talk in this room. This limitation prevents anonymous spamming as it becomes possible for at least some people in the room to locate the spammer's node.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
                </property>
                <property name="text">
                 <string>require PGP-signed identities</string>
@@ -121,7 +130,7 @@
               </widget>
              </item>
              <item row="4" column="0">
-              <widget class="QLabel" name="label">
+              <widget class="QLabel" name="pgp_signed_Lb">
                <property name="layoutDirection">
                 <enum>Qt::LeftToRight</enum>
                </property>
@@ -136,7 +145,7 @@
             </layout>
            </item>
            <item>
-            <spacer name="verticalSpacer">
+            <spacer name="roomVSpacer">
              <property name="orientation">
               <enum>Qt::Vertical</enum>
              </property>
@@ -152,34 +161,37 @@
          </widget>
         </item>
         <item>
-         <layout class="QGridLayout" name="gridLayout">
+         <layout class="QGridLayout" name="friendListGLayout">
           <item row="0" column="0">
-           <widget class="QLabel" name="label_2">
+           <widget class="QLabel" name="friendListLabel">
             <property name="text">
              <string>Select the Friends with which you want to group chat.</string>
             </property>
            </widget>
           </item>
           <item row="1" column="0">
-           <widget class="QDockWidget" name="contactsdockWidget">
+           <widget class="FriendSelectionWidget" name="keyShareList" native="true">
-            <property name="enabled">
-             <bool>true</bool>
-            </property>
             <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+             <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
               <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
+              <verstretch>4</verstretch>
              </sizepolicy>
             </property>
+            <property name="minimumSize">
+             <size>
+              <width>20</width>
+              <height>0</height>
+             </size>
+            </property>
             <property name="maximumSize">
              <size>
-              <width>52487</width>
+              <width>1677215</width>
-              <height>524287</height>
+              <height>16777215</height>
              </size>
             </property>
             <property name="sizeIncrement">
              <size>
-              <width>220</width>
+              <width>0</width>
               <height>0</height>
              </size>
             </property>
@@ -189,59 +201,6 @@
               <height>0</height>
              </size>
             </property>
-            <property name="floating">
-             <bool>false</bool>
-            </property>
-            <property name="features">
-             <set>QDockWidget::NoDockWidgetFeatures</set>
-            </property>
-            <property name="windowTitle">
-             <string>Invited friends</string>
-            </property>
-            <widget class="QWidget" name="dockWidgetContents">
-             <layout class="QGridLayout" name="_2">
-              <property name="margin">
-               <number>0</number>
-              </property>
-              <property name="spacing">
-               <number>0</number>
-              </property>
-              <item row="0" column="0">
-               <widget class="FriendSelectionWidget" name="keyShareList" native="true">
-                <property name="sizePolicy">
-                 <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
-                  <horstretch>0</horstretch>
-                  <verstretch>4</verstretch>
-                 </sizepolicy>
-                </property>
-                <property name="minimumSize">
-                 <size>
-                  <width>20</width>
-                  <height>0</height>
-                 </size>
-                </property>
-                <property name="maximumSize">
-                 <size>
-                  <width>1677215</width>
-                  <height>16777215</height>
-                 </size>
-                </property>
-                <property name="sizeIncrement">
-                 <size>
-                  <width>0</width>
-                  <height>0</height>
-                 </size>
-                </property>
-                <property name="baseSize">
-                 <size>
-                  <width>0</width>
-                  <height>0</height>
-                 </size>
-                </property>
-               </widget>
-              </item>
-             </layout>
-            </widget>
            </widget>
           </item>
          </layout>
@@ -249,7 +208,7 @@
        </layout>
       </item>
       <item>
-       <layout class="QHBoxLayout" name="horizontalLayout">
+       <layout class="QHBoxLayout" name="toolBarHLayout">
         <property name="sizeConstraint">
          <enum>QLayout::SetDefaultConstraint</enum>
         </property>

retroshare-gui/src/gui/common/GroupTreeWidget.cpp

@@ -18,6 +18,8 @@
  *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
+#include <iostream>
+
 #include "GroupTreeWidget.h"
 #include "ui_GroupTreeWidget.h"
 
@@ -58,6 +60,8 @@
 #define ROLE_SUBSCRIBE_FLAGS Qt::UserRole + 8
 #define ROLE_COLOR           Qt::UserRole + 9
 #define ROLE_SAVED_ICON      Qt::UserRole + 10
+#define ROLE_SEARCH_STRING   Qt::UserRole + 11
+#define ROLE_REQUEST_ID      Qt::UserRole + 12
 
 #define FILTER_NAME_INDEX  0
 #define FILTER_DESC_INDEX  1
@@ -132,6 +136,10 @@ GroupTreeWidget::GroupTreeWidget(QWidget *parent) :
 	ui->filterLineEdit->addFilter(QIcon(), tr("Description"), FILTER_DESC_INDEX , tr("Search Description"));
 	ui->filterLineEdit->setCurrentFilter(FILTER_NAME_INDEX);
 
+    ui->distantSearchLineEdit->setPlaceholderText(tr("Search entire network...")) ;
+
+    connect(ui->distantSearchLineEdit,SIGNAL(returnPressed()),this,SLOT(distantSearch())) ;
+
 	/* Initialize display button */
 	initDisplayMenu(ui->displayButton);
 
@@ -407,6 +415,54 @@ QTreeWidgetItem *GroupTreeWidget::addCategoryItem(const QString &name, const QIc
 	return item;
 }
 
+void GroupTreeWidget::removeSearchItem(QTreeWidgetItem *item)
+{
+    ui->treeWidget->takeTopLevelItem(ui->treeWidget->indexOfTopLevelItem(item)) ;
+}
+
+QTreeWidgetItem *GroupTreeWidget::addSearchItem(const QString& search_string, uint32_t id, const QIcon& icon)
+{
+    QTreeWidgetItem *item = addCategoryItem(search_string,icon,true);
+
+    item->setData(COLUMN_DATA,ROLE_SEARCH_STRING,search_string) ;
+    item->setData(COLUMN_DATA,ROLE_REQUEST_ID   ,id) ;
+
+    return item;
+}
+
+void GroupTreeWidget::setDistSearchVisible(bool visible)
+{
+    ui->distantSearchLineEdit->setVisible(visible);
+}
+
+bool GroupTreeWidget::isSearchRequestResult(QPoint &point,QString& group_id,uint32_t& search_req_id)
+{
+    QTreeWidgetItem *item = ui->treeWidget->itemAt(point);
+	if (item == NULL)
+		return false;
+
+    QTreeWidgetItem *parent = item->parent();
+
+    if(parent == NULL)
+        return false ;
+
+	search_req_id = parent->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt();
+    group_id = itemId(item) ;
+
+    return search_req_id > 0;
+}
+
+bool GroupTreeWidget::isSearchRequestItem(QPoint &point,uint32_t& search_req_id)
+{
+    QTreeWidgetItem *item = ui->treeWidget->itemAt(point);
+	if (item == NULL)
+		return false;
+
+	search_req_id = item->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt();
+
+    return search_req_id > 0;
+}
+
 QString GroupTreeWidget::itemId(QTreeWidgetItem *item)
 {
 	if (item == NULL) {
@@ -770,6 +826,13 @@ void GroupTreeWidget::resort(QTreeWidgetItem *categoryItem)
 	}
 }
 
+void GroupTreeWidget::distantSearch()
+{
+    emit distantSearchRequested(ui->distantSearchLineEdit->text());
+
+    ui->distantSearchLineEdit->clear();
+}
+
 void GroupTreeWidget::sort()
 {
 	resort(NULL);

retroshare-gui/src/gui/common/GroupTreeWidget.h

@@ -82,6 +82,11 @@ public:
 
 	// Add a new category item
 	QTreeWidgetItem *addCategoryItem(const QString &name, const QIcon &icon, bool expand);
+    // Add a new search item
+    void setDistSearchVisible(bool) ; // shows/hides distant search UI parts.
+	QTreeWidgetItem *addSearchItem(const QString& search_string, uint32_t id, const QIcon &icon) ;
+	void removeSearchItem(QTreeWidgetItem *item);
+
 	// Get id of item
 	QString itemId(QTreeWidgetItem *item);
 	QString itemIdAt(QPoint &point);
@@ -90,6 +95,9 @@ public:
 	// Set the unread count of an item
 	void setUnreadCount(QTreeWidgetItem *item, int unreadCount);
 
+	bool isSearchRequestItem(QPoint &point,uint32_t& search_req_id);
+	bool isSearchRequestResult(QPoint &point, QString &group_id, uint32_t& search_req_id);
+
 	QTreeWidgetItem *getItemFromId(const QString &id);
 	QTreeWidgetItem *activateId(const QString &id, bool focus);
 
@@ -110,6 +118,7 @@ signals:
 	void treeCustomContextMenuRequested(const QPoint &pos);
 	void treeCurrentItemChanged(const QString &id);
 	void treeItemActivated(const QString &id);
+    void distantSearchRequested(const QString&) ;
 
 protected:
 	void changeEvent(QEvent *e);
@@ -119,6 +128,7 @@ private slots:
 	void currentItemChanged(QTreeWidgetItem *current, QTreeWidgetItem *previous);
 	void itemActivated(QTreeWidgetItem *item, int column);
 	void filterChanged();
+	void distantSearch();
 
 	void sort();
 

retroshare-gui/src/gui/common/GroupTreeWidget.ui

@@ -124,6 +124,13 @@
      </column>
     </widget>
    </item>
+   <item>
+    <widget class="QLineEdit" name="distantSearchLineEdit">
+     <property name="toolTip">
+      <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Searches a single keyword into the reachable network.&lt;/p&gt;&lt;p&gt;Objects already provided by friend nodes are not reported.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>

retroshare-gui/src/gui/common/RSTextBrowser.cpp

@@ -251,22 +251,53 @@ bool RSTextBrowser::checkImage(QPoint pos, QString &imageStr)
  */
 QString RSTextBrowser::anchorForPosition(const QPoint &pos) const
 {
+	// Many calls when time label shows up
 	QTextCursor cursor = cursorForPosition(pos);
 	cursor.select(QTextCursor::WordUnderCursor);
+	QString word = cursor.selectedText();
 	QString anchor = "";
-	if (!cursor.selectedText().isEmpty()){
+	if (word.isEmpty())
-		QRegExp rx("<a name=\"(.*)\"",Qt::CaseSensitive, QRegExp::RegExp2);
+		return anchor;
-		rx.setMinimal(true);
+
-		QString sel = cursor.selection().toHtml();
+	// For finding positions
-		QStringList anchors;
+	QTextCursor cursor_line = cursorForPosition(pos);
-		int pos=0;
+	cursor_line.select(QTextCursor::LineUnderCursor);
-		while ((pos = rx.indexIn(sel,pos)) != -1) {
+	QString line = cursor_line.selectedText();
-			anchors << rx.cap(1);
+	// End of nickname (more or less, of course, because some can has colon in
-			pos += rx.matchedLength();
+	// name)
-		}
+	int end_of_name = line.indexOf(": ") + 1;
-		if (!anchors.isEmpty()){
+	// Start of nickname (after time)
-			anchor = anchors.at(0);
+	int space_index = line.indexOf(' ') + 1;
+	int word_index = line.indexOf(word) + 1;
+	int once = 1;
+	cursor_line.movePosition(QTextCursor::StartOfLine, QTextCursor::MoveAnchor, once);
+	bool after_name = cursor_line.position() + end_of_name < cursor.position();
+	bool cursor_after_time = space_index < word_index;
+	bool cursor_within_name = word_index <= end_of_name;
+	if (!after_name && cursor_after_time && cursor_within_name) {
+		cursor.movePosition(QTextCursor::StartOfLine, QTextCursor::MoveAnchor, once);
+		cursor.movePosition(QTextCursor::NextCharacter, QTextCursor::MoveAnchor,
+			space_index);
+		cursor.movePosition(QTextCursor::NextCharacter, QTextCursor::KeepAnchor,
+			end_of_name - space_index);
+		if (cursor.selectedText().isEmpty()) {
+			return anchor;
 		}
+	} else {
+		return anchor;
+	}
+
+	QRegExp rx("<a name=\"(.*)\"",Qt::CaseSensitive, QRegExp::RegExp2);
+	rx.setMinimal(true);
+	QString sel = cursor.selection().toHtml();
+	QStringList anchors;
+	int position = 0;
+	while ((position = rx.indexIn(sel, position)) != -1) {
+		anchors << rx.cap(1);
+		position += rx.matchedLength();
+	}
+	if (!anchors.isEmpty()) {
+		anchor = anchors.at(0);
 	}
 	return anchor;
 }

retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp

@@ -46,6 +46,8 @@
 #define IMAGE_EDIT           ":/images/edit_16.png"
 #define IMAGE_SHARE          ":/images/share-icon-16.png"
 #define IMAGE_TABNEW         ":/images/tab-new.png"
+#define IMAGE_DELETE         ":/images/delete.png"
+#define IMAGE_RETRIEVE       ":/images/edit_add24.png"
 #define IMAGE_COMMENT        ""
 
 #define TOKEN_TYPE_GROUP_SUMMARY    1
@@ -65,7 +67,7 @@
  */
 
 /** Constructor */
-GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent)
+GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent,bool allow_dist_sync)
 : RsGxsUpdateBroadcastPage(ifaceImpl, parent)
 {
 	/* Invoke the Qt Designer generated object setup routine */
@@ -73,6 +75,7 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p
 	ui->setupUi(this);
 
 	mInitialized = false;
+	mDistSyncAllowed = allow_dist_sync;
 	mInFill = false;
 	mCountChildMsgs = false;
 	mYourGroups = NULL;
@@ -92,13 +95,18 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p
 	mStateHelper->addWidget(TOKEN_TYPE_GROUP_SUMMARY, ui->loadingLabel, UISTATE_LOADING_VISIBLE);
 
 	connect(ui->groupTreeWidget, SIGNAL(treeCustomContextMenuRequested(QPoint)), this, SLOT(groupTreeCustomPopupMenu(QPoint)));
-    connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedGroup(QString)));
+    connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedCurrentGroup(QString)));
 	connect(ui->groupTreeWidget->treeWidget(), SIGNAL(signalMouseMiddleButtonClicked(QTreeWidgetItem*)), this, SLOT(groupTreeMiddleButtonClicked(QTreeWidgetItem*)));
 	connect(ui->messageTabWidget, SIGNAL(tabCloseRequested(int)), this, SLOT(messageTabCloseRequested(int)));
 	connect(ui->messageTabWidget, SIGNAL(currentChanged(int)), this, SLOT(messageTabChanged(int)));
 
 	connect(ui->todoPushButton, SIGNAL(clicked()), this, SLOT(todo()));
 
+	ui->groupTreeWidget->setDistSearchVisible(allow_dist_sync) ;
+
+    if(allow_dist_sync)
+		connect(ui->groupTreeWidget, SIGNAL(distantSearchRequested(const QString&)), this, SLOT(searchNetwork(const QString&)));
+
 	/* Set initial size the splitter */
 	ui->splitter->setStretchFactor(0, 0);
 	ui->splitter->setStretchFactor(1, 1);
@@ -121,7 +129,7 @@ GxsGroupFrameDialog::~GxsGroupFrameDialog()
 	delete(ui);
 }
 
-void GxsGroupFrameDialog::getGroupList(std::list<RsGroupMetaData>& group_list)
+void GxsGroupFrameDialog::getGroupList(std::map<RsGxsGroupId, RsGroupMetaData> &group_list)
 {
 	group_list = mCachedGroupMetas ;
 
@@ -243,6 +251,55 @@ void GxsGroupFrameDialog::updateDisplay(bool complete)
 			updateMessageSummaryList(msgIt->first);
 		}
 	}
+
+    updateSearchResults() ;
+}
+
+void GxsGroupFrameDialog::updateSearchResults()
+{
+    const std::set<TurtleRequestId>& reqs = getSearchResults();
+
+    for(auto it(reqs.begin());it!=reqs.end();++it)
+    {
+		std::cerr << "updating search ID " << std::hex << *it << std::dec << std::endl;
+
+        std::map<RsGxsGroupId,RsGxsGroupSummary> group_infos;
+
+        getDistantSearchResults(*it,group_infos) ;
+
+        std::cerr << "retrieved " << std::endl;
+
+        auto it2 = mSearchGroupsItems.find(*it);
+
+        if(mSearchGroupsItems.end() == it2)
+        {
+            std::cerr << "GxsGroupFrameDialog::updateSearchResults(): received result notification for req " << std::hex << *it << std::dec << " but no item present!" << std::endl;
+            continue ;	// we could create the item just as well but since this situation is not supposed to happen, I prefer to make this a failure case.
+        }
+
+        QList<GroupItemInfo> group_items ;
+
+        for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
+            if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end())
+			{
+				std::cerr << "  adding new group " << it3->first << " " << it3->second.group_id << " \"" << it3->second.group_name << "\"" << std::endl;
+
+				GroupItemInfo i ;
+				i.id             = QString(it3->second.group_id.toStdString().c_str()) ;
+				i.name           = QString::fromUtf8(it3->second.group_name.c_str()) ;
+				i.description    = QString::fromUtf8(it3->second.group_description.c_str()) ;
+				i.popularity     = 0; // could be set to the number of hits
+				i.lastpost       = QDateTime::fromTime_t(it3->second.last_message_ts);
+				i.subscribeFlags = 0; // irrelevant here
+				i.publishKey     = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags) ;
+				i.adminKey       = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags) ;
+				i.max_visible_posts = it3->second.number_of_messages ;
+
+				group_items.push_back(i);
+			}
+
+		ui->groupTreeWidget->fillGroupItems(it2->second, group_items);
+    }
 }
 
 void GxsGroupFrameDialog::todo()
@@ -250,98 +307,149 @@ void GxsGroupFrameDialog::todo()
 	QMessageBox::information(this, "Todo", text(TEXT_TODO));
 }
 
+void GxsGroupFrameDialog::removeCurrentSearch()
+{
+    QAction *action = dynamic_cast<QAction*>(sender()) ;
+
+    if(!action)
+        return ;
+
+    TurtleRequestId search_request_id = action->data().toUInt();
+
+    auto it = mSearchGroupsItems.find(search_request_id) ;
+
+    if(it == mSearchGroupsItems.end())
+        return ;
+
+    ui->groupTreeWidget->removeSearchItem(it->second) ;
+    mSearchGroupsItems.erase(it);
+
+    mKnownGroups.erase(search_request_id);
+}
+
+void GxsGroupFrameDialog::removeAllSearches()
+{
+    for(auto it(mSearchGroupsItems.begin());it!=mSearchGroupsItems.end();++it)
+		ui->groupTreeWidget->removeSearchItem(it->second) ;
+
+    mSearchGroupsItems.clear();
+    mKnownGroups.clear();
+}
 void GxsGroupFrameDialog::groupTreeCustomPopupMenu(QPoint point)
 {
-	QMenu contextMnu(this);
+	// First separately handle the case of search top level items
+
+	TurtleRequestId search_request_id = 0 ;
+
+	if(ui->groupTreeWidget->isSearchRequestItem(point,search_request_id))
+	{
+		QMenu contextMnu(this);
+
+		contextMnu.addAction(QIcon(IMAGE_DELETE), tr("Remove this search"), this, SLOT(removeCurrentSearch()))->setData(search_request_id);
+		contextMnu.addAction(QIcon(IMAGE_DELETE), tr("Remove all searches"), this, SLOT(removeAllSearches()));
+		contextMnu.exec(QCursor::pos());
+		return ;
+	}
+
+    // Then check whether we have a searched item, or a normal group
+
+    QString group_id_s ;
+
+	if(ui->groupTreeWidget->isSearchRequestResult(point,group_id_s,search_request_id))
+    {
+		QMenu contextMnu(this);
+
+		contextMnu.addAction(QIcon(IMAGE_RETRIEVE), tr("Request data"), this, SLOT(distantRequestGroupData()))->setData(group_id_s);
+		contextMnu.exec(QCursor::pos());
+		return ;
+    }
 
 	QString id = ui->groupTreeWidget->itemIdAt(point);
-	if (!id.isEmpty())
+	if (id.isEmpty()) return;
-	{
 
-		mGroupId = RsGxsGroupId(id.toStdString());
+	mGroupId = RsGxsGroupId(id.toStdString());
-		int subscribeFlags = ui->groupTreeWidget->subscribeFlags(QString::fromStdString(mGroupId.toStdString()));
+	int subscribeFlags = ui->groupTreeWidget->subscribeFlags(QString::fromStdString(mGroupId.toStdString()));
 
-		bool isAdmin = IS_GROUP_ADMIN(subscribeFlags);
+	bool isAdmin      = IS_GROUP_ADMIN(subscribeFlags);
-		bool isPublisher = IS_GROUP_PUBLISHER(subscribeFlags);
+	bool isPublisher  = IS_GROUP_PUBLISHER(subscribeFlags);
-		bool isSubscribed = IS_GROUP_SUBSCRIBED(subscribeFlags);
+	bool isSubscribed = IS_GROUP_SUBSCRIBED(subscribeFlags);
 
+	QMenu contextMnu(this);
+	QAction *action;
 
-		QAction *action;
+	if (mMessageWidget) {
-
+		action = contextMnu.addAction(QIcon(IMAGE_TABNEW), tr("Open in new tab"), this, SLOT(openInNewTab()));
-		if (mMessageWidget) {
+		if (mGroupId.isNull() || messageWidget(mGroupId, true)) {
-			action = contextMnu.addAction(QIcon(IMAGE_TABNEW), tr("Open in new tab"), this, SLOT(openInNewTab()));
+			action->setEnabled(false);
-			if (mGroupId.isNull() || messageWidget(mGroupId, true)) {
-				action->setEnabled(false);
-			}
 		}
+	}
 
-		if (isSubscribed) {
+	if (isSubscribed) {
-			action = contextMnu.addAction(QIcon(IMAGE_UNSUBSCRIBE), tr("Unsubscribe"), this, SLOT(unsubscribeGroup()));
+		action = contextMnu.addAction(QIcon(IMAGE_UNSUBSCRIBE), tr("Unsubscribe"), this, SLOT(unsubscribeGroup()));
-			action->setEnabled (!mGroupId.isNull() && IS_GROUP_SUBSCRIBED(subscribeFlags));
+		action->setEnabled (!mGroupId.isNull() && IS_GROUP_SUBSCRIBED(subscribeFlags));
-		} else {
+	} else {
-			action = contextMnu.addAction(QIcon(IMAGE_SUBSCRIBE), tr("Subscribe"), this, SLOT(subscribeGroup()));
+		action = contextMnu.addAction(QIcon(IMAGE_SUBSCRIBE), tr("Subscribe"), this, SLOT(subscribeGroup()));
-			action->setDisabled (mGroupId.isNull() || IS_GROUP_SUBSCRIBED(subscribeFlags));
+		action->setDisabled (mGroupId.isNull() || IS_GROUP_SUBSCRIBED(subscribeFlags));
-		}
+	}
 
+	contextMnu.addSeparator();
+
+	contextMnu.addAction(QIcon(icon(ICON_NEW)), text(TEXT_NEW), this, SLOT(newGroup()));
+
+	action = contextMnu.addAction(QIcon(IMAGE_INFO), tr("Show Details"), this, SLOT(showGroupDetails()));
+	action->setEnabled (!mGroupId.isNull());
+
+	action = contextMnu.addAction(QIcon(IMAGE_EDIT), tr("Edit Details"), this, SLOT(editGroupDetails()));
+	action->setEnabled (!mGroupId.isNull() && isAdmin);
+
+	uint32_t current_store_time = mInterface->getStoragePeriod(mGroupId)/86400 ;
+	uint32_t current_sync_time  = mInterface->getSyncPeriod(mGroupId)/86400 ;
+
+	std::cerr << "Got sync=" << current_sync_time << ". store=" << current_store_time << std::endl;
+	QAction *actnn = NULL;
+
+	QMenu *ctxMenu2 = contextMnu.addMenu(tr("Synchronise posts of last...")) ;
+	actnn = ctxMenu2->addAction(tr(" 5 days"     ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(  5)) ; if(current_sync_time ==  5) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 2 weeks"    ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 15)) ; if(current_sync_time == 15) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 1 month"    ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 30)) ; if(current_sync_time == 30) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 3 months"   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 90)) ; if(current_sync_time == 90) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 6 months"   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(180)) ; if(current_sync_time ==180) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 1 year  "   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(372)) ; if(current_sync_time ==372) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" Indefinitly"),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(  0)) ; if(current_sync_time ==  0) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+
+	ctxMenu2 = contextMnu.addMenu(tr("Store posts for at most...")) ;
+	actnn = ctxMenu2->addAction(tr(" 5 days"     ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(  5)) ; if(current_store_time ==  5) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 2 weeks"    ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 15)) ; if(current_store_time == 15) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 1 month"    ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 30)) ; if(current_store_time == 30) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 3 months"   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 90)) ; if(current_store_time == 90) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 6 months"   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(180)) ; if(current_store_time ==180) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" 1 year  "   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(372)) ; if(current_store_time ==372) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+	actnn = ctxMenu2->addAction(tr(" Indefinitly"),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(  0)) ; if(current_store_time ==  0) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
+
+	if (shareKeyType()) {
+		action = contextMnu.addAction(QIcon(IMAGE_SHARE), tr("Share publish permissions"), this, SLOT(sharePublishKey()));
+		action->setEnabled(!mGroupId.isNull() && isPublisher);
+	}
+
+	if (getLinkType() != RetroShareLink::TYPE_UNKNOWN) {
+		action = contextMnu.addAction(QIcon(IMAGE_COPYLINK), tr("Copy RetroShare Link"), this, SLOT(copyGroupLink()));
+		action->setEnabled(!mGroupId.isNull());
+	}
+
+	contextMnu.addSeparator();
+
+	action = contextMnu.addAction(QIcon(":/images/message-mail-read.png"), tr("Mark all as read"), this, SLOT(markMsgAsRead()));
+	action->setEnabled (!mGroupId.isNull() && isSubscribed);
+
+	action = contextMnu.addAction(QIcon(":/images/message-mail.png"), tr("Mark all as unread"), this, SLOT(markMsgAsUnread()));
+	action->setEnabled (!mGroupId.isNull() && isSubscribed);
+
+	/* Add special actions */
+	QList<QAction*> actions;
+	groupTreeCustomActions(mGroupId, subscribeFlags, actions);
+	if (!actions.isEmpty()) {
 		contextMnu.addSeparator();
-
+		contextMnu.addActions(actions);
-		contextMnu.addAction(QIcon(icon(ICON_NEW)), text(TEXT_NEW), this, SLOT(newGroup()));
-
-		action = contextMnu.addAction(QIcon(IMAGE_INFO), tr("Show Details"), this, SLOT(showGroupDetails()));
-		action->setEnabled (!mGroupId.isNull());
-
-		action = contextMnu.addAction(QIcon(IMAGE_EDIT), tr("Edit Details"), this, SLOT(editGroupDetails()));
-		action->setEnabled (!mGroupId.isNull() && isAdmin);
-
-		uint32_t current_store_time = mInterface->getStoragePeriod(mGroupId)/86400 ;
-		uint32_t current_sync_time  = mInterface->getSyncPeriod(mGroupId)/86400 ;
-
-		std::cerr << "Got sync=" << current_sync_time << ". store=" << current_store_time << std::endl;
-		QAction *actnn = NULL;
-
-		QMenu *ctxMenu2 = contextMnu.addMenu(tr("Synchronise posts of last...")) ;
-		actnn = ctxMenu2->addAction(tr(" 5 days"     ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(  5)) ; if(current_sync_time ==  5) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 2 weeks"    ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 15)) ; if(current_sync_time == 15) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 1 month"    ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 30)) ; if(current_sync_time == 30) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 3 months"   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant( 90)) ; if(current_sync_time == 90) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 6 months"   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(180)) ; if(current_sync_time ==180) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 1 year  "   ),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(372)) ; if(current_sync_time ==372) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" Indefinitly"),this,SLOT(setSyncPostsDelay())) ; actnn->setData(QVariant(  0)) ; if(current_sync_time ==  0) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-
-		ctxMenu2 = contextMnu.addMenu(tr("Store posts for at most...")) ;
-		actnn = ctxMenu2->addAction(tr(" 5 days"     ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(  5)) ; if(current_store_time ==  5) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 2 weeks"    ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 15)) ; if(current_store_time == 15) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 1 month"    ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 30)) ; if(current_store_time == 30) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 3 months"   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant( 90)) ; if(current_store_time == 90) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 6 months"   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(180)) ; if(current_store_time ==180) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" 1 year  "   ),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(372)) ; if(current_store_time ==372) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-		actnn = ctxMenu2->addAction(tr(" Indefinitly"),this,SLOT(setStorePostsDelay())) ; actnn->setData(QVariant(  0)) ; if(current_store_time ==  0) { actnn->setEnabled(false);actnn->setIcon(QIcon(":/images/start.png"));}
-
-		if (shareKeyType()) {
-			action = contextMnu.addAction(QIcon(IMAGE_SHARE), tr("Share publish permissions"), this, SLOT(sharePublishKey()));
-			action->setEnabled(!mGroupId.isNull() && isPublisher);
-		}
-
-		if (getLinkType() != RetroShareLink::TYPE_UNKNOWN) {
-			action = contextMnu.addAction(QIcon(IMAGE_COPYLINK), tr("Copy RetroShare Link"), this, SLOT(copyGroupLink()));
-			action->setEnabled(!mGroupId.isNull());
-		}
-
-		contextMnu.addSeparator();
-
-		action = contextMnu.addAction(QIcon(":/images/message-mail-read.png"), tr("Mark all as read"), this, SLOT(markMsgAsRead()));
-		action->setEnabled (!mGroupId.isNull() && isSubscribed);
-
-		action = contextMnu.addAction(QIcon(":/images/message-mail.png"), tr("Mark all as unread"), this, SLOT(markMsgAsUnread()));
-		action->setEnabled (!mGroupId.isNull() && isSubscribed);
-
-		/* Add special actions */
-		QList<QAction*> actions;
-		groupTreeCustomActions(mGroupId, subscribeFlags, actions);
-		if (!actions.isEmpty()) {
-			contextMnu.addSeparator();
-			contextMnu.addActions(actions);
-		}
 	}
 
 	//Add Standard Menu
@@ -594,7 +702,7 @@ bool GxsGroupFrameDialog::navigate(const RsGxsGroupId &groupId, const RsGxsMessa
 		return false;
 	}
 
-	changedGroup(groupIdString);
+	changedCurrentGroup(groupIdString);
 
 	/* search exisiting tab */
 	GxsMessageFrameWidget *msgWidget = messageWidget(mGroupId, false);
@@ -655,7 +763,7 @@ GxsCommentDialog *GxsGroupFrameDialog::commentWidget(const RsGxsMessageId& msgId
 	return NULL;
 }
 
-void GxsGroupFrameDialog::changedGroup(const QString &groupId)
+void GxsGroupFrameDialog::changedCurrentGroup(const QString &groupId)
 {
 	if (mInFill) {
 		return;
@@ -798,7 +906,7 @@ void GxsGroupFrameDialog::groupInfoToGroupItemInfo(const RsGroupMetaData &groupI
 	}
 }
 
-void GxsGroupFrameDialog::insertGroupsData(const std::list<RsGroupMetaData> &groupList, const RsUserdata *userdata)
+void GxsGroupFrameDialog::insertGroupsData(const std::map<RsGxsGroupId,RsGroupMetaData> &groupList, const RsUserdata *userdata)
 {
 	if (!mInitialized) {
 		return;
@@ -806,20 +914,18 @@ void GxsGroupFrameDialog::insertGroupsData(const std::list<RsGroupMetaData> &gro
 
 	mInFill = true;
 
-	std::list<RsGroupMetaData>::const_iterator it;
-
 	QList<GroupItemInfo> adminList;
 	QList<GroupItemInfo> subList;
 	QList<GroupItemInfo> popList;
 	QList<GroupItemInfo> otherList;
 	std::multimap<uint32_t, GroupItemInfo> popMap;
 
-	for (it = groupList.begin(); it != groupList.end(); ++it) {
+	for (auto it = groupList.begin(); it != groupList.end(); ++it) {
 		/* sort it into Publish (Own), Subscribed, Popular and Other */
-		uint32_t flags = it->mSubscribeFlags;
+		uint32_t flags = it->second.mSubscribeFlags;
 
 		GroupItemInfo groupItemInfo;
-		groupInfoToGroupItemInfo(*it, groupItemInfo, userdata);
+		groupInfoToGroupItemInfo(it->second, groupItemInfo, userdata);
 
 		if (IS_GROUP_SUBSCRIBED(flags))
 		{
@@ -836,7 +942,7 @@ void GxsGroupFrameDialog::insertGroupsData(const std::list<RsGroupMetaData> &gro
 		else
 		{
 			//popMap.insert(std::make_pair(it->mPop, groupItemInfo)); /* rate the others by popularity */
-			popMap.insert(std::make_pair(it->mLastPost, groupItemInfo)); /* rate the others by time of last post */
+			popMap.insert(std::make_pair(it->second.mLastPost, groupItemInfo)); /* rate the others by time of last post */
 		}
 	}
 
@@ -947,9 +1053,12 @@ void GxsGroupFrameDialog::loadGroupSummary(const uint32_t &token)
 	RsUserdata *userdata = NULL;
 	loadGroupSummaryToken(token, groupInfo, userdata);
 
-	mCachedGroupMetas = groupInfo ;
+	mCachedGroupMetas.clear();
+    for(auto it(groupInfo.begin());it!=groupInfo.end();++it)
+        mCachedGroupMetas[(*it).mGroupId] = *it;
 
-	insertGroupsData(groupInfo, userdata);
+	insertGroupsData(mCachedGroupMetas, userdata);
+    updateSearchResults();
 
 	mStateHelper->setLoading(TOKEN_TYPE_GROUP_SUMMARY, false);
 
@@ -1083,3 +1192,41 @@ void GxsGroupFrameDialog::loadRequest(const TokenQueue *queue, const TokenReques
 		}
 	}
 }
+
+TurtleRequestId GxsGroupFrameDialog::distantSearch(const QString& search_string)   // this should be overloaded in the child class
+{
+    std::cerr << "Searching for \"" << search_string.toStdString() << "\". Function is not overloaded, so nothing will happen." << std::endl;
+    return 0;
+}
+
+void GxsGroupFrameDialog::searchNetwork(const QString& search_string)
+{
+    if(search_string.isNull())
+        return ;
+
+    uint32_t request_id = distantSearch(search_string);
+
+    if(request_id == 0)
+        return ;
+
+	mSearchGroupsItems[request_id] = ui->groupTreeWidget->addSearchItem(tr("Search for")+ " \"" + search_string + "\"",(uint32_t)request_id,QIcon(icon(ICON_SEARCH)));
+}
+
+void GxsGroupFrameDialog::distantRequestGroupData()
+{
+    QAction *action = dynamic_cast<QAction*>(sender()) ;
+
+    if(!action)
+        return ;
+
+    RsGxsGroupId group_id(action->data().toString().toStdString());
+
+    if(group_id.isNull())
+    {
+        std::cerr << "Cannot retrieve group! Group id is null!" << std::endl;
+    }
+
+	std::cerr << "Explicit request for group " << group_id << std::endl;
+    checkRequestGroup(group_id) ;
+}
+

retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h

@@ -67,11 +67,12 @@ public:
 		ICON_SUBSCRIBED_GROUP,
 		ICON_POPULAR_GROUP,
 		ICON_OTHER_GROUP,
+		ICON_SEARCH,
 		ICON_DEFAULT
 	};
 
 public:
-	GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent = 0);
+	GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent = 0,bool allow_dist_sync=false);
 	virtual ~GxsGroupFrameDialog();
 
 	bool navigate(const RsGxsGroupId &groupId, const RsGxsMessageId& msgId);
@@ -81,7 +82,7 @@ public:
 
 	virtual QString getHelpString() const =0;
 
-	virtual void getGroupList(std::list<RsGroupMetaData>& groups) ;
+	virtual void getGroupList(std::map<RsGxsGroupId,RsGroupMetaData> &groups) ;
 
 protected:
 	virtual void showEvent(QShowEvent *event);
@@ -94,6 +95,7 @@ protected:
 	virtual RetroShareLink::enumType getLinkType() = 0;
 	virtual GroupFrameSettings::Type groupFrameSettingsType() { return GroupFrameSettings::Nothing; }
 	virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata);
+    virtual void checkRequestGroup(const RsGxsGroupId& /* grpId */) {}	// overload this one in order to retrieve full group data when the group is browsed
 
 private slots:
 	void todo();
@@ -106,8 +108,9 @@ private slots:
 
 	void restoreGroupKeys();
 	void newGroup();
+    void distantRequestGroupData();
 
-	void changedGroup(const QString &groupId);
+	void changedCurrentGroup(const QString &groupId);
 	void groupTreeMiddleButtonClicked(QTreeWidgetItem *item);
 	void openInNewTab();
 	void messageTabCloseRequested(int index);
@@ -130,10 +133,15 @@ private slots:
 
 	void loadComment(const RsGxsGroupId &grpId, const QVector<RsGxsMessageId>& msg_versions,const RsGxsMessageId &most_recent_msgId, const QString &title);
 
+    void searchNetwork(const QString &search_string) ;
+	void removeAllSearches();
+	void removeCurrentSearch();
+
 private:
 	virtual QString text(TextType type) = 0;
 	virtual QString icon(IconType type) = 0;
 	virtual QString settingsGroupName() = 0;
+    virtual TurtleRequestId distantSearch(const QString& search_string) ;
 
 	virtual GxsGroupDialog *createNewGroupDialog(TokenQueue *tokenQueue) = 0;
 	virtual GxsGroupDialog *createGroupDialog(TokenQueue *tokenQueue, RsTokenService *tokenService, GxsGroupDialog::Mode mode, RsGxsGroupId groupId) = 0;
@@ -142,10 +150,12 @@ private:
 	virtual void groupTreeCustomActions(RsGxsGroupId /*grpId*/, int /*subscribeFlags*/, QList<QAction*> &/*actions*/) {}
 	virtual RsGxsCommentService *getCommentService() { return NULL; }
 	virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; }
+    virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map<RsGxsGroupId,RsGxsGroupSummary>& /* group_infos */){ return false ;}
 
 	void initUi();
 
 	void updateMessageSummaryList(RsGxsGroupId groupId);
+	void updateSearchResults();
 
 	void openGroupInNewTab(const RsGxsGroupId &groupId);
 	void groupSubscribe(bool subscribe);
@@ -153,7 +163,7 @@ private:
 	void processSettings(bool load);
 
 	// New Request/Response Loading Functions.
-	void insertGroupsData(const std::list<RsGroupMetaData> &groupList, const RsUserdata *userdata);
+	void insertGroupsData(const std::map<RsGxsGroupId, RsGroupMetaData> &groupList, const RsUserdata *userdata);
 
 	void requestGroupSummary();
 	void loadGroupSummary(const uint32_t &token);
@@ -180,6 +190,7 @@ protected:
 private:
 	bool mInitialized;
 	bool mInFill;
+    bool mDistSyncAllowed;
 	QString mSettingsName;
 	RsGxsGroupId mGroupId;
 	RsGxsIfaceHelper *mInterface;
@@ -200,7 +211,10 @@ private:
 	/** Qt Designer generated object */
 	Ui::GxsGroupFrameDialog *ui;
 
-	std::list<RsGroupMetaData> mCachedGroupMetas;
+	std::map<RsGxsGroupId,RsGroupMetaData> mCachedGroupMetas;
+
+    std::map<uint32_t,QTreeWidgetItem*> mSearchGroupsItems ;
+    std::map<uint32_t,std::set<RsGxsGroupId> > mKnownGroups;
 };
 
 #endif

retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>619</width>
-    <height>420</height>
+    <height>493</height>
    </rect>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
@@ -38,7 +38,16 @@
       <enum>QFrame::Sunken</enum>
      </property>
      <layout class="QHBoxLayout" name="horizontalLayout_2">
-      <property name="margin">
+      <property name="leftMargin">
+       <number>2</number>
+      </property>
+      <property name="topMargin">
+       <number>2</number>
+      </property>
+      <property name="rightMargin">
+       <number>2</number>
+      </property>
+      <property name="bottomMargin">
        <number>2</number>
       </property>
       <item>

retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.h

@@ -57,6 +57,7 @@ protected:
 	virtual void groupNameChanged(const QString &/*name*/) {}
 
 	virtual void clearPosts() = 0;
+	virtual void blank() = 0;
 	virtual bool navigatePostItem(const RsGxsMessageId& msgId) = 0;
 
 	/* Thread functions */

retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.cpp

@@ -72,21 +72,26 @@ bool GxsMessageFrameWidget::isWaiting()
 
 void GxsMessageFrameWidget::setGroupId(const RsGxsGroupId &groupId)
 {
-	if (mGroupId == groupId) {
+	if (mGroupId == groupId  && !groupId.isNull())
-		if (!groupId.isNull()) {
+		return;
-			return;
+
+    if(!groupId.isNull())
+	{
+		mAcknowledgeReadStatusToken = 0;
+		if (mStateHelper->isLoading(mTokenTypeAcknowledgeReadStatus)) {
+			mStateHelper->setLoading(mTokenTypeAcknowledgeReadStatus, false);
+
+			emit waitingChanged(this);
 		}
+
+		mGroupId = groupId;
+		groupIdChanged();
 	}
-
+    else
-	mAcknowledgeReadStatusToken = 0;
+    {
-	if (mStateHelper->isLoading(mTokenTypeAcknowledgeReadStatus)) {
+        mGroupId.clear();
-		mStateHelper->setLoading(mTokenTypeAcknowledgeReadStatus, false);
+        blank();	// clear the displayed data, because no group is selected.
-
+    }
-		emit waitingChanged(this);
-	}
-
-	mGroupId = groupId;
-	groupIdChanged();
 }
 
 void GxsMessageFrameWidget::setAllMessagesRead(bool read)

retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.h

@@ -43,6 +43,7 @@ public:
 	virtual void groupIdChanged() = 0;
 	virtual QString groupName(bool withUnreadCount) = 0;
 	virtual QIcon groupIcon() = 0;
+    virtual void blank() =0;
 	virtual bool navigate(const RsGxsMessageId& msgId) = 0;
 	virtual bool isLoading();
 	virtual bool isWaiting();

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.cpp

@@ -16,12 +16,19 @@ RsGxsUpdateBroadcastBase::RsGxsUpdateBroadcastBase(RsGxsIfaceHelper *ifaceImpl,
 	connect(mUpdateBroadcast, SIGNAL(changed()), this, SLOT(updateBroadcastChanged()));
 	connect(mUpdateBroadcast, SIGNAL(grpsChanged(std::list<RsGxsGroupId>, std::list<RsGxsGroupId>)), this, SLOT(updateBroadcastGrpsChanged(std::list<RsGxsGroupId>,std::list<RsGxsGroupId>)));
 	connect(mUpdateBroadcast, SIGNAL(msgsChanged(std::map<RsGxsGroupId,std::set<RsGxsMessageId> >, std::map<RsGxsGroupId,std::set<RsGxsMessageId> >)), this, SLOT(updateBroadcastMsgsChanged(std::map<RsGxsGroupId,std::set<RsGxsMessageId> >,std::map<RsGxsGroupId,std::set<RsGxsMessageId> >)));
+	connect(mUpdateBroadcast, SIGNAL(distantSearchResultsChanged(const std::list<TurtleRequestId>&)), this, SLOT(updateBroadcastDistantSearchResultsChanged(const std::list<TurtleRequestId>&)));
 }
 
 RsGxsUpdateBroadcastBase::~RsGxsUpdateBroadcastBase()
 {
 }
 
+void RsGxsUpdateBroadcastBase::updateBroadcastDistantSearchResultsChanged(const std::list<TurtleRequestId>& ids)
+{
+    for(auto it(ids.begin());it!=ids.end();++it)
+        mTurtleResults.insert(*it);
+}
+
 void RsGxsUpdateBroadcastBase::fillComplete()
 {
 	mFillComplete = true;
@@ -37,6 +44,9 @@ void RsGxsUpdateBroadcastBase::securedUpdateDisplay()
 		return;
 	}
 
+    // This is *bad* because if the connection is done asynchronously the client will call mGrpIds, mGrpIdsMeta, etc without the guarranty that the
+    // the structed havnt' been cleared in the mean time.
+
 	emit fillDisplay(mFillComplete);
 	mFillComplete = false;
 
@@ -75,7 +85,7 @@ void RsGxsUpdateBroadcastBase::updateBroadcastChanged()
         // The question to whether we should re=load when mGrpIds is not empty is still open. It's not harmful anyway.
         // This should probably be decided by the service itself.
 
-		if (!mGrpIds.empty() || !mGrpIdsMeta.empty() /*|| !mMsgIds.empty()*/ || !mMsgIdsMeta.empty())
+		if (!mGrpIds.empty() || !mGrpIdsMeta.empty() /*|| !mMsgIds.empty()*/ || !mMsgIdsMeta.empty() || !mTurtleResults.empty())
             mFillComplete = true ;
 
 		securedUpdateDisplay();

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.h

@@ -7,6 +7,8 @@ class QShowEvent;
 class RsGxsIfaceHelper;
 class RsGxsUpdateBroadcast;
 
+typedef uint32_t TurtleRequestId ;
+
 class RsGxsUpdateBroadcastBase : public QObject
 {
 	friend class RsGxsUpdateBroadcastPage;
@@ -25,6 +27,7 @@ public:
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIds() { return mMsgIds; }
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIdsMeta() { return mMsgIdsMeta; }
 	void getAllMsgIds(std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgIds);
+    const std::set<TurtleRequestId>& getSearchResults() { return mTurtleResults ; }
 
 protected:
 	void fillComplete();
@@ -39,6 +42,7 @@ private slots:
 	void updateBroadcastChanged();
 	void updateBroadcastGrpsChanged(const std::list<RsGxsGroupId>& grpIds, const std::list<RsGxsGroupId> &grpIdsMeta);
 	void updateBroadcastMsgsChanged(const std::map<RsGxsGroupId, std::set<RsGxsMessageId> >& msgIds, const std::map<RsGxsGroupId, std::set<RsGxsMessageId> >& msgIdsMeta);
+	void updateBroadcastDistantSearchResultsChanged(const std::list<TurtleRequestId>& ids);
 	void securedUpdateDisplay();
 
 private:
@@ -49,4 +53,5 @@ private:
 	std::set<RsGxsGroupId> mGrpIdsMeta;
 	std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mMsgIds;
 	std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mMsgIdsMeta;
+    std::set<TurtleRequestId> mTurtleResults;
 };

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.cpp

@@ -22,6 +22,11 @@ void RsGxsUpdateBroadcastPage::setUpdateWhenInvisible(bool update)
 	mBase->setUpdateWhenInvisible(update);
 }
 
+const std::set<TurtleRequestId>& RsGxsUpdateBroadcastPage::getSearchResults()
+{
+    return mBase->getSearchResults();
+}
+
 const std::set<RsGxsGroupId> &RsGxsUpdateBroadcastPage::getGrpIdsMeta()
 {
 	return mBase->getGrpIdsMeta();

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.h

@@ -13,6 +13,7 @@
 
 class RsGxsIfaceHelper;
 class RsGxsUpdateBroadcastBase;
+typedef uint32_t TurtleRequestId ;
 
 class RsGxsUpdateBroadcastPage : public MainPage
 {
@@ -30,6 +31,7 @@ public:
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIds();
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIdsMeta();
 	void getAllMsgIds(std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgIds);
+    const std::set<TurtleRequestId>& getSearchResults();
 
 protected:
 	virtual void showEvent(QShowEvent *event);

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp

@@ -29,6 +29,10 @@ const std::set<RsGxsGroupId> &RsGxsUpdateBroadcastWidget::getGrpIds()
     return mBase->getGrpIds();
 }
 
+const std::set<TurtleRequestId>& RsGxsUpdateBroadcastWidget::getSearchResults()
+{
+    return mBase->getSearchResults();
+}
 const std::set<RsGxsGroupId> &RsGxsUpdateBroadcastWidget::getGrpIdsMeta()
 {
     return mBase->getGrpIdsMeta();

retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.h

@@ -13,6 +13,7 @@
 
 class RsGxsIfaceHelper;
 class RsGxsUpdateBroadcastBase;
+typedef uint32_t TurtleRequestId;
 
 class RsGxsUpdateBroadcastWidget : public QWidget
 {
@@ -30,6 +31,7 @@ public:
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIds();
 	const std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &getMsgIdsMeta();
 	void getAllMsgIds(std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgIds);
+    const std::set<TurtleRequestId>& getSearchResults() ;
 
 	RsGxsIfaceHelper *interfaceHelper() { return mInterfaceHelper; }
 

retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp

@@ -47,7 +47,7 @@ public:
 
 /** Constructor */
 GxsChannelDialog::GxsChannelDialog(QWidget *parent)
-	: GxsGroupFrameDialog(rsGxsChannels, parent)
+	: GxsGroupFrameDialog(rsGxsChannels, parent,true)
 {
 }
 
@@ -137,6 +137,8 @@ QString GxsChannelDialog::icon(IconType type)
 		return ":/images/folder_green.png";
 	case ICON_OTHER_GROUP:
 		return ":/images/folder_yellow.png";
+    case ICON_SEARCH:
+        return ":/images/find.png";
 	case ICON_DEFAULT:
 		return ":/images/channels.png";
 	}
@@ -334,3 +336,24 @@ void GxsChannelDialog::groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo
 		groupItemInfo.icon = iconIt.value();
 	}
 }
+
+TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string)
+{
+    return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ;
+}
+
+bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
+{
+    return rsGxsChannels->retrieveDistantSearchResults(id,group_infos);
+}
+
+void GxsChannelDialog::checkRequestGroup(const RsGxsGroupId& grpId)
+{
+    RsGxsChannelGroup distant_group;
+
+	if( rsGxsChannels->retrieveDistantGroup(grpId,distant_group)) // normally we should also check that the group meta is not already here.
+    {
+        std::cerr << "GxsChannelDialog::checkRequestGroup() sending turtle request for group data for group " << grpId << std::endl;
+        rsGxsChannels->turtleGroupRequest(grpId);
+    }
+}

retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h

@@ -50,6 +50,10 @@ protected:
 	virtual GroupFrameSettings::Type groupFrameSettingsType() { return GroupFrameSettings::Channel; }
 	virtual QString getHelpString() const ;
 	virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata);
+    virtual bool getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos);
+
+	virtual TurtleRequestId distantSearch(const QString& search_string) ;
+    virtual void checkRequestGroup(const RsGxsGroupId& grpId) ;
 
 private slots:
 	void toggleAutoDownload();

retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp

@@ -596,6 +596,20 @@ void GxsChannelPostsWidget::clearPosts()
 	ui->fileWidget->clear();
 }
 
+void GxsChannelPostsWidget::blank()
+{
+	mStateHelper->setWidgetEnabled(ui->postButton, false);
+	mStateHelper->setWidgetEnabled(ui->subscribeToolButton, false);
+
+	clearPosts();
+
+    groupNameChanged(QString());
+
+	ui->infoWidget->hide();
+	ui->feedWidget->show();
+	ui->fileWidget->hide();
+}
+
 bool GxsChannelPostsWidget::navigatePostItem(const RsGxsMessageId &msgId)
 {
 	FeedItem *feedItem = ui->feedWidget->findGxsFeedItem(groupId(), msgId);
@@ -643,12 +657,22 @@ bool GxsChannelPostsWidget::insertGroupData(const uint32_t &token, RsGroupMetaDa
 	std::vector<RsGxsChannelGroup> groups;
 	rsGxsChannels->getGroupData(token, groups);
 
-	if (groups.size() == 1)
+	if(groups.size() == 1)
 	{
 		insertChannelDetails(groups[0]);
 		metaData = groups[0].mMeta;
 		return true;
 	}
+    else
+    {
+        RsGxsChannelGroup distant_group;
+        if(rsGxsChannels->retrieveDistantGroup(groupId(),distant_group))
+        {
+			insertChannelDetails(distant_group);
+			metaData = distant_group.mMeta;
+            return true ;
+        }
+    }
 
 	return false;
 }

retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.h

@@ -73,6 +73,7 @@ protected:
 	virtual bool useThread() { return mUseThread; }
 	virtual void fillThreadCreatePost(const QVariant &post, bool related, int current, int count);
 	virtual bool navigatePostItem(const RsGxsMessageId& msgId);
+    virtual void blank() ;
 
 	/* GxsMessageFrameWidget */
 	virtual void setAllMessagesReadDo(bool read, uint32_t &token);

retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp

@@ -299,6 +299,28 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget
 
 }
 
+void GxsForumThreadWidget::blank()
+{
+	ui->progressBar->hide();
+	ui->progressText->hide();
+	ui->postText->clear() ;
+	ui->by_label->setId(RsGxsId()) ;
+	ui->time_label->clear();
+	ui->lineRight->hide();
+	ui->lineLeft->hide();
+	ui->by_text_label->hide();
+	ui->by_label->hide();
+	ui->postText->setImageBlockWidget(ui->imageBlockWidget) ;
+	ui->postText->resetImagesStatus(Settings->getForumLoadEmbeddedImages());
+    ui->threadTreeWidget->clear();
+	ui->forumName->setText("");
+
+    mStateHelper->setWidgetEnabled(ui->newthreadButton, false);
+	mStateHelper->setWidgetEnabled(ui->previousButton, false);
+	mStateHelper->setWidgetEnabled(ui->nextButton, false);
+	ui->versions_CB->hide();
+}
+
 GxsForumThreadWidget::~GxsForumThreadWidget()
 {
 	if (mFillThread) {

retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.h

@@ -58,6 +58,7 @@ public:
 
 	// Callback for all Loads.
 	virtual void loadRequest(const TokenQueue *queue, const TokenRequest &req);
+    virtual void blank();
 
 protected:
 	bool eventFilter(QObject *obj, QEvent *ev);

retroshare-gui/src/gui/notifyqt.cpp

@@ -563,6 +563,11 @@ void NotifyQt::notifyChatCleared(const ChatId& chat_id)
 	emit chatCleared(chat_id);
 }
 
+void NotifyQt::notifyTurtleSearchResult(uint32_t search_id,const std::list<TurtleGxsInfo>& found_groups)
+{
+    std::cerr << "(EE) missing code to handle GXS turtle search result." << std::endl;
+}
+
 void NotifyQt::notifyTurtleSearchResult(uint32_t search_id,const std::list<TurtleFileInfo>& files)
 {
 	{

retroshare-gui/src/gui/notifyqt.h

@@ -23,7 +23,9 @@ class MessengerWindow;
 class ToasterItem;
 class ToasterNotify;
 class SignatureEventData ;
+
 struct TurtleFileInfo;
+struct TurtleGxsInfo;
 
 class NotifyQt: public QObject, public NotifyClient
 {
@@ -47,6 +49,7 @@ class NotifyQt: public QObject, public NotifyClient
 		virtual void notifyCustomState(const std::string& peer_id, const std::string& status_string);
 		virtual void notifyHashingInfo(uint32_t type, const std::string& fileinfo);
 		virtual void notifyTurtleSearchResult(uint32_t search_id,const std::list<TurtleFileInfo>& found_files);
+		virtual void notifyTurtleSearchResult(uint32_t search_id,const std::list<TurtleGxsInfo>& found_groups);
 		virtual void notifyPeerHasNewAvatar(std::string peer_id) ;
 		virtual void notifyOwnAvatarChanged() ;
         virtual void notifyChatLobbyEvent(uint64_t /* lobby id */, uint32_t /* event type */, const RsGxsId & /*nickname*/, const std::string& /* any string */) ;

retroshare-gui/src/gui/qss/stylesheet/Standard.qss

@@ -186,7 +186,7 @@ ChatLobbyDialog QListWidget#participantsList {
 	background: white;
 }
 
-CreateLobbyDialog QFrame#lobbyFrame {
+CreateLobbyDialog QFrame#roomFrame {
 	border: 2px solid #CCCCCC;
 	border-radius:6px;
 	background: white;

retroshare-gui/src/gui/settings/PluginsPage.cpp

@@ -33,7 +33,7 @@
 
 #include "../MainWindow.h"
 
-PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags)
+settings::PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags)
     : ConfigPage(parent, flags)
 {
     ui.setupUi(this);
@@ -123,7 +123,7 @@ PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags)
 
 			 if(plugin == NULL || plugin->qt_config_panel() == NULL)
                  item->_configure_PB->hide() ;
-				 
+
 
 			 if(plugin != NULL){
 				 item->enableButton->hide();
@@ -159,7 +159,7 @@ PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags)
 
 	QObject::connect(ui.enableAll,SIGNAL(toggled(bool)),this,SLOT(toggleEnableAll(bool))) ;
 }
-QString PluginsPage::helpText() const
+QString settings::PluginsPage::helpText() const
 {
    return tr("<h1><img width=\"24\" src=\":/icons/help_64.png\">&nbsp;&nbsp;Plugins</h1>     \
               <p>Plugins are loaded from the directories listed in the bottom list.</p>         \
@@ -171,11 +171,11 @@ QString PluginsPage::helpText() const
               <p>If you want to develop your own plugins, contact the developpers team          \
               they will be happy to help you out!</p>") ;
 }
-void PluginsPage::toggleEnableAll(bool b)
+void settings::PluginsPage::toggleEnableAll(bool b)
 {
 	rsPlugins->allowAllPlugins(b) ;
 }
-void PluginsPage::aboutPlugin(int i)
+void settings::PluginsPage::aboutPlugin(int i)
 {
 	std::cerr << "Launching about window for plugin " << i << std::endl;
 
@@ -183,7 +183,7 @@ void PluginsPage::aboutPlugin(int i)
 	if(rsPlugins->plugin(i) != NULL && (dialog = rsPlugins->plugin(i)->qt_about_page()) != NULL)
 		dialog->exec() ;
 }
-void PluginsPage::configurePlugin(int i)
+void settings::PluginsPage::configurePlugin(int i)
 {
 	std::cerr << "Launching configuration window for plugin " << i << std::endl;
 
@@ -191,14 +191,14 @@ void PluginsPage::configurePlugin(int i)
 		rsPlugins->plugin(i)->qt_config_panel()->show() ;
 }
 
-void PluginsPage::enablePlugin(const QString& hash)
+void settings::PluginsPage::enablePlugin(const QString& hash)
 {
 	std::cerr << "Switching status of plugin " << hash.toStdString() << " to  enable" << std::endl;
 
         rsPlugins->enablePlugin(RsFileHash(hash.toStdString()) );
 }
 
-void PluginsPage::disablePlugin(const QString& hash)
+void settings::PluginsPage::disablePlugin(const QString& hash)
 {
 	std::cerr << "Switching status of plugin " << hash.toStdString() << " to disable " << std::endl;
 
@@ -206,11 +206,11 @@ void PluginsPage::disablePlugin(const QString& hash)
 }
 
 
-PluginsPage::~PluginsPage()
+settings::PluginsPage::~PluginsPage()
 {
 }
 
 /** Loads the settings for this page */
-void PluginsPage::load()
+void settings::PluginsPage::load()
 {
 }

retroshare-gui/src/gui/settings/PluginsPage.h

@@ -24,6 +24,8 @@
 #include <retroshare-gui/configpage.h>
 #include "ui_PluginsPage.h"
 
+namespace settings {
+
 class PluginsPage : public ConfigPage
 {
 	Q_OBJECT
@@ -52,3 +54,4 @@ class PluginsPage : public ConfigPage
 			Ui::PluginsPage ui;
 };
 
+} // namespace settings

retroshare-gui/src/gui/settings/rsettingswin.cpp

@@ -158,7 +158,7 @@ SettingsPage::initStackedWidget()
     addPage(new ForumPage()); // FORUMS
     addPage(new PostedPage()); // POSTED RENAME TO LINKS
     addPage(new NotifyPage()); // NOTIFY
-    addPage(new PluginsPage() ); // PLUGINS
+    addPage(new settings::PluginsPage() ); // PLUGINS
     addPage(new AppearancePage()); // APPEARENCE
     addPage(new SoundPage() ); // SOUND
     addPage(new ServicePermissionsPage() ); // PERMISSIONS

retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp

@@ -1,7 +1,11 @@
 #include <QObject>
+#include <util/rsprint.h>
 #include <retroshare/rsturtle.h>
 #include <retroshare/rspeers.h>
 #include <retroshare/rsgxstunnel.h>
+#include <retroshare/rsservicecontrol.h>
+#include <retroshare/rsidentity.h>
+#include <retroshare/rsgxsdistsync.h>
 #include "TurtleRouterDialog.h"
 #include <QPainter>
 #include <QStylePainter>
@@ -240,19 +244,14 @@ QTreeWidgetItem *TurtleRouterDialog::findParentHashItem(const std::string& hash)
 	else
 		return items.front() ;
 }
+
 //=======================================================================================================================//
 
-
+TunnelStatisticsDialog::TunnelStatisticsDialog(QWidget *parent)
-GxsTunnelsDialog::GxsTunnelsDialog(QWidget *parent)
 	: RsAutoUpdatePage(2000,parent)
 {
-//	setupUi(this) ;
-	
 	m_bProcessSettings = false;
 
-        //float fontHeight = QFontMetricsF(font()).height();
-        //float fact = fontHeight/14.0;
-	
 	maxWidth = 200 ;
 	maxHeight = 200 ;
 
@@ -260,14 +259,13 @@ GxsTunnelsDialog::GxsTunnelsDialog(QWidget *parent)
     processSettings(true);
 }
 
-GxsTunnelsDialog::~GxsTunnelsDialog()
+TunnelStatisticsDialog::~TunnelStatisticsDialog()
 {
-
     // save settings
     processSettings(false);
 }
 
-void GxsTunnelsDialog::processSettings(bool bLoad)
+void TunnelStatisticsDialog::processSettings(bool bLoad)
 {
     m_bProcessSettings = true;
 
@@ -284,7 +282,77 @@ void GxsTunnelsDialog::processSettings(bool bLoad)
     m_bProcessSettings = false;
 }
 
-void GxsTunnelsDialog::updateDisplay()
+QString TunnelStatisticsDialog::getPeerName(const RsPeerId &peer_id)
+{
+    static std::map<RsPeerId, QString> names ;
+
+    std::map<RsPeerId,QString>::const_iterator it = names.find(peer_id) ;
+
+	if( it != names.end())
+		return it->second ;
+	else
+	{
+		RsPeerDetails detail ;
+		if(!rsPeers->getPeerDetails(peer_id,detail))
+			return tr("Unknown Peer");
+
+		return (names[peer_id] = QString::fromUtf8(detail.name.c_str())) ;
+	}
+}
+
+QString TunnelStatisticsDialog::getPeerName(const RsGxsId& gxs_id)
+{
+    static std::map<RsGxsId, QString> names ;
+
+    std::map<RsGxsId,QString>::const_iterator it = names.find(gxs_id) ;
+
+	if( it != names.end())
+		return it->second ;
+	else
+	{
+		RsIdentityDetails detail ;
+
+		if(!rsIdentity->getIdDetails(gxs_id,detail))
+			return tr("Unknown Peer");
+
+		return (names[gxs_id] = QString::fromUtf8(detail.mNickname.c_str())) ;
+	}
+}
+
+
+QString TunnelStatisticsDialog::speedString(float f)
+{
+	if(f < 1.0f)
+		return QString("0 B/s") ;
+	if(f < 1024.0f)
+		return QString::number((int)f)+" B/s" ;
+
+	return QString::number(f/1024.0,'f',2) + " KB/s";
+}
+
+void TunnelStatisticsDialog::paintEvent(QPaintEvent */*event*/)
+{
+    QStylePainter(this).drawPixmap(0, 0, pixmap);
+}
+
+void TunnelStatisticsDialog::resizeEvent(QResizeEvent *event)
+{
+    QRect TaskGraphRect = geometry();
+
+    maxWidth = TaskGraphRect.width();
+    maxHeight = TaskGraphRect.height() ;
+
+    QWidget::resizeEvent(event);
+    update();
+}
+//=======================================================================================================================//
+
+GxsAuthenticatedTunnelsDialog::GxsAuthenticatedTunnelsDialog(QWidget *parent)
+    : TunnelStatisticsDialog(parent)
+{
+}
+
+void GxsAuthenticatedTunnelsDialog::updateDisplay()
 {
     // Request info about ongoing tunnels
     
@@ -334,8 +402,8 @@ void GxsTunnelsDialog::updateDisplay()
 		// draw...
 
 		painter.drawText(ox+4*cellx,oy+celly,tr("Tunnel ID: %1").arg(QString::fromStdString(tunnel_infos[i].tunnel_id.toStdString()))) ; oy += celly ;
-		painter.drawText(ox+6*cellx,oy+celly,tr("from: %1").arg(QString::fromStdString(tunnel_infos[i].source_gxs_id.toStdString()))) ; oy += celly ;
+		painter.drawText(ox+6*cellx,oy+celly,tr("from: %1 (%2)").arg(QString::fromStdString(tunnel_infos[i].source_gxs_id.toStdString())).arg(getPeerName(tunnel_infos[i].source_gxs_id))) ; oy += celly ;
-		painter.drawText(ox+6*cellx,oy+celly,tr("to: %1").arg(QString::fromStdString(tunnel_infos[i].destination_gxs_id.toStdString()))) ; oy += celly ;
+		painter.drawText(ox+6*cellx,oy+celly,tr("to: %1 (%2)").arg(QString::fromStdString(tunnel_infos[i].destination_gxs_id.toStdString())).arg(getPeerName(tunnel_infos[i].destination_gxs_id))) ; oy += celly ;
 		painter.drawText(ox+6*cellx,oy+celly,tr("status: %1").arg(QString::number(tunnel_infos[i].tunnel_status))) ; oy += celly ;
 		painter.drawText(ox+6*cellx,oy+celly,tr("total sent: %1 bytes").arg(QString::number(tunnel_infos[i].total_size_sent))) ; oy += celly ;
 		painter.drawText(ox+6*cellx,oy+celly,tr("total recv: %1 bytes").arg(QString::number(tunnel_infos[i].total_size_received))) ; oy += celly ;
@@ -350,46 +418,197 @@ void GxsTunnelsDialog::updateDisplay()
 	maxHeight = std::max(oy,10*celly);
 }
 
-QString GxsTunnelsDialog::getPeerName(const RsPeerId &peer_id)
+//=======================================================================================================================//
+
+GxsNetTunnelsDialog::GxsNetTunnelsDialog(QWidget *parent)
+    : TunnelStatisticsDialog(parent)
 {
-    static std::map<RsPeerId, QString> names ;
+}
 
-    std::map<RsPeerId,QString>::const_iterator it = names.find(peer_id) ;
+static QString getGroupStatusString(RsGxsNetTunnelGroupInfo::GroupStatus group_status)
+{
+    switch(group_status)
+    {
+    default:
+	   	    case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN            : return QObject::tr("Unknown") ;
+	   	    case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE               : return QObject::tr("Idle");
+	   	    case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE    : return QObject::tr("Virtual peers available");
+    }
+    return QString();
+}
 
-	if( it != names.end())
+static QString getGroupPolicyString(RsGxsNetTunnelGroupInfo::GroupPolicy group_policy)
-		return it->second ;
+{
-	else
+    switch(group_policy)
+    {
+    default:
+	 		case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN            : return QObject::tr("Unknown") ;
+	 		case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE            : return QObject::tr("Passive") ;
+	 		case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE             : return QObject::tr("Active") ;
+	 		case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING         : return QObject::tr("Requesting peers") ;
+    }
+    return QString();
+}
+
+static QString getLastContactString(time_t last_contact)
+{
+    time_t now = time(NULL);
+
+    if(last_contact == 0)
+        return QObject::tr("Never");
+
+    return QString::number(now - last_contact) + " secs ago" ;
+}
+
+static QString getServiceNameString(uint16_t service_id)
+{
+	static RsPeerServiceInfo ownServices;
+
+    if(ownServices.mServiceList.find(service_id) == ownServices.mServiceList.end())
+		rsServiceControl->getOwnServices(ownServices);
+
+    return QString::fromUtf8(ownServices.mServiceList[service_id].mServiceName.c_str()) ;
+}
+
+static QString getVirtualPeerStatusString(uint8_t status)
+{
+	switch(status)
 	{
-		RsPeerDetails detail ;
+	default:
-		if(!rsPeers->getPeerDetails(peer_id,detail))
+	case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN   : return QObject::tr("Unknown") ;
-			return tr("Unknown Peer");
+	case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK : return QObject::tr("Tunnel OK") ;
-
+	case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE    : return QObject::tr("Tunnel active") ;
-		return (names[peer_id] = QString::fromUtf8(detail.name.c_str())) ;
 	}
+	return QString();
 }
 
-QString GxsTunnelsDialog::speedString(float f)
+static QString getSideString(uint8_t side)
 {
-	if(f < 1.0f) 
+    return side?QObject::tr("Client"):QObject::tr("Server") ;
-		return QString("0 B/s") ;
-	if(f < 1024.0f)
-		return QString::number((int)f)+" B/s" ;
-
-	return QString::number(f/1024.0,'f',2) + " KB/s";
 }
 
-void GxsTunnelsDialog::paintEvent(QPaintEvent */*event*/)
+static QString getMasterKeyString(const uint8_t *key,uint32_t size)
 {
-    QStylePainter(this).drawPixmap(0, 0, pixmap);
+    return QString::fromStdString(RsUtil::BinToHex(key,size,10));
 }
 
-void GxsTunnelsDialog::resizeEvent(QResizeEvent *event)
+void GxsNetTunnelsDialog::updateDisplay()
 {
-    QRect TaskGraphRect = geometry();
+	// Request info about ongoing tunnels
-    
-    maxWidth = TaskGraphRect.width();
-    maxHeight = TaskGraphRect.height() ;
 
-    QWidget::resizeEvent(event);
+	std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> groups;                                 // groups on the client and server side
-    update();
+    std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId> turtle2gxsnettunnel;         // convertion table from turtle to net tunnel virtual peer id
+	std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo> virtual_peers;    // current virtual peers, which group they provide, and how to talk to them through turtle
+	Bias20Bytes bias;
+
+	rsGxsDistSync->getStatistics(groups,virtual_peers,turtle2gxsnettunnel,bias) ;
+
+    // RsGxsNetTunnelGroupInfo:
+    //
+	//   enum GroupStatus {
+	//   	    RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN            = 0x00,	// unknown status
+	//   	    RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE               = 0x01,	// no virtual peers requested, just waiting
+	//   	    RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE    = 0x02	// some virtual peers are available. Data can be read/written
+	//   };
+	//   enum GroupPolicy {
+	//   	    RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN            = 0x00,	// nothing has been set
+	//   	    RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE            = 0x01,	// group is available for server side tunnels, but does not explicitely request tunnels
+	//   	    RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE             = 0x02,	// group will only explicitely request tunnels if none available
+	//   	    RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING         = 0x03,	// group explicitely requests tunnels
+	//      };
+    //
+	//   RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {}
+    //
+	//   GroupPolicy        group_policy ;
+	//   GroupStatus        group_status ;
+	//   time_t             last_contact ;
+	//   RsFileHash         hash ;
+	//   uint16_t           service_id ;
+	//   std::set<RsPeerId> virtual_peers ;
+    //
+ 	// struct RsGxsNetTunnelVirtualPeerInfo:
+	//
+	// 	enum {   RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN   = 0x00,		// unknown status.
+	// 		     RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01,		// tunnel has been established and we're waiting for virtual peer id
+	// 		     RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE    = 0x02		// virtual peer id is known. Data can transfer.
+	// 	     };
+	//
+	// 	RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; }
+	// 	virtual ~RsGxsNetTunnelVirtualPeerInfo(){}
+	//
+	// 	uint8_t vpid_status ;					// status of the peer
+	// 	time_t  last_contact ;					// last time some data was sent/recvd
+	// 	uint8_t side ;	                        // client/server
+	// 	uint8_t encryption_master_key[32];
+	//
+	// 	RsPeerId      turtle_virtual_peer_id ;  // turtle peer to use when sending data to this vpid.
+	//
+	// 	RsGxsGroupId group_id ;					// group that virtual peer is providing
+	// 	uint16_t service_id ; 					// this is used for checkng consistency of the incoming data
+
+	// update the pixmap
+	//
+
+    // now draw the shit
+	QPixmap tmppixmap(maxWidth, maxHeight);
+	tmppixmap.fill(Qt::transparent);
+	//setFixedHeight(maxHeight);
+
+	QPainter painter(&tmppixmap);
+	painter.initFrom(this);
+
+	// extracts the height of the fonts in pixels. This is used to calibrate the size of the objects to draw.
+
+	float fontHeight = QFontMetricsF(font()).height();
+	float fact = fontHeight/14.0;
+
+	int cellx = 6*fact ;
+	int celly = (10+4)*fact ;
+	int ox=5*fact,oy=5*fact ;
+
+	painter.setPen(QColor::fromRgb(0,0,0)) ;
+	painter.drawText(ox+2*cellx,oy+celly,tr("Random Bias: %1").arg(getMasterKeyString(bias.toByteArray(),20))) ; oy += celly ;
+	painter.drawText(ox+2*cellx,oy+celly,tr("GXS Groups:")) ; oy += celly ;
+
+	for(auto it(groups.begin());it!=groups.end();++it)
+    {
+		painter.drawText(ox+4*cellx,oy+celly,tr("Service: %1 (%2) - Group ID: %3,\t policy: %4, \tstatus: %5, \tlast contact: %6")
+                .arg(QString::number(it->second.service_id))
+                .arg(getServiceNameString(it->second.service_id))
+                .arg(QString::fromStdString(it->first.toStdString()))
+                .arg(getGroupPolicyString(it->second.group_policy))
+                .arg(getGroupStatusString(it->second.group_status))
+                .arg(getLastContactString(it->second.last_contact))
+		                 ),oy+=celly ;
+
+
+        for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2)
+        {
+            auto it4 = turtle2gxsnettunnel.find(*it2) ;
+
+            if(it4 != turtle2gxsnettunnel.end())
+            {
+				auto it3 = virtual_peers.find(it4->second) ;
+
+				if(virtual_peers.end() != it3)
+					painter.drawText(ox+6*cellx,oy+celly,tr("Peer: %1:\tstatus: %2/%3, \tlast contact: %4, \tMaster key: %5.")
+					                 .arg(QString::fromStdString((*it2).toStdString()))
+					                 .arg(getVirtualPeerStatusString(it3->second.vpid_status))
+					                 .arg(getSideString(it3->second.side))
+					                 .arg(getLastContactString(it3->second.last_contact))
+					                 .arg(getMasterKeyString(it3->second.encryption_master_key,32))
+					                 ),oy+=celly ;
+            }
+            else
+				painter.drawText(ox+6*cellx,oy+celly,tr("Peer: %1: no information available")
+				                 .arg(QString::fromStdString((*it2).toStdString()))
+                                 ),oy+=celly;
+
+        }
+    }
+
+	oy += celly ;
+
+	pixmap = tmppixmap;
+	maxHeight = std::max(oy,10*celly);
 }

retroshare-gui/src/gui/statistics/TurtleRouterDialog.h

@@ -37,29 +37,55 @@ class TurtleRouterDialog: public RsAutoUpdatePage, public Ui::TurtleRouterDialog
 
 } ;
 
-class GxsTunnelsDialog: public RsAutoUpdatePage
+class TunnelStatisticsDialog: public RsAutoUpdatePage
 {
     Q_OBJECT
 
 public:
-    GxsTunnelsDialog(QWidget *parent = NULL) ;
+    TunnelStatisticsDialog(QWidget *parent = NULL) ;
-    ~GxsTunnelsDialog();
+    ~TunnelStatisticsDialog();
 
     // Cache for peer names.
     static QString getPeerName(const RsPeerId &peer_id) ;
+	static QString getPeerName(const RsGxsId& gxs_id);
 
 protected:
     virtual void paintEvent(QPaintEvent *);
     virtual void resizeEvent(QResizeEvent *event);
+
+	int maxWidth ;
+    int maxHeight ;
+
+    QPixmap pixmap;
+
 private:
     void processSettings(bool bLoad);
     bool m_bProcessSettings;
     static QString speedString(float f);
 
-    virtual void updateDisplay() ;
+    virtual void updateDisplay() =0;
-    
+} ;
-    int maxWidth ;
+
-    int maxHeight ;
+class GxsAuthenticatedTunnelsDialog: public TunnelStatisticsDialog
-    
+{
-    QPixmap pixmap;
+    Q_OBJECT
+
+public:
+    GxsAuthenticatedTunnelsDialog(QWidget *parent = NULL) ;
+    ~GxsAuthenticatedTunnelsDialog() {}
+
+private:
+    virtual void updateDisplay() ;
+} ;
+
+class GxsNetTunnelsDialog: public TunnelStatisticsDialog
+{
+    Q_OBJECT
+
+public:
+    GxsNetTunnelsDialog(QWidget *parent = NULL) ;
+    ~GxsNetTunnelsDialog() {}
+
+private:
+    virtual void updateDisplay() ;
 } ;

retroshare-gui/src/gui/statistics/TurtleRouterStatistics.cpp

@@ -195,11 +195,13 @@ TurtleRouterStatistics::TurtleRouterStatistics(QWidget *parent)
 	_tunnel_statistics_F->setFrameStyle(QFrame::NoFrame);
 	_tunnel_statistics_F->setFocusPolicy(Qt::NoFocus);
 	
-	routertabWidget->addTab(new TurtleRouterDialog(),QString(tr("Anonymous tunnels")));
+	routertabWidget->addTab(new TurtleRouterDialog(),           QString(tr("File transfer tunnels")));
-	routertabWidget->addTab(new GxsTunnelsDialog(),QString(tr("Authenticated tunnels")));
+	routertabWidget->addTab(new GxsAuthenticatedTunnelsDialog(),QString(tr("Authenticated tunnels")));
+	routertabWidget->addTab(new GxsNetTunnelsDialog(),          QString(tr("GXS sync tunnels")     ));
+
+	float fontHeight = QFontMetricsF(font()).height();
+	float fact = fontHeight/14.0;
 
-        float fontHeight = QFontMetricsF(font()).height();
-        float fact = fontHeight/14.0;
     frmGraph->setMinimumHeight(200*fact);
 	
 	// load settings

retroshare-gui/src/rshare.cpp

@@ -360,7 +360,7 @@ QString Rshare::retroshareVersion(bool withRevision)
 {
 	QString version = QString("%1.%2.%3%4").arg(RS_MAJOR_VERSION).arg(RS_MINOR_VERSION).arg(RS_BUILD_NUMBER).arg(RS_BUILD_NUMBER_ADD);
 	if (withRevision) {
-		version += QString(" %1 %2").arg(tr("Revision")).arg(QString::number(RS_REVISION_NUMBER,16));
+		version += QString(" %1 %2").arg(tr("Revision")).arg(RS_REVISION_NUMBER,8,16,QChar('0'));
 	}
 
 	return version;

retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp

@@ -79,5 +79,8 @@ void RsGxsUpdateBroadcast::onChangesReceived(const RsGxsChanges& changes)
         emit grpsChanged(changes.mGrps, changes.mGrpsMeta);
     }
 
+    if(!changes.mDistantSearchReqs.empty())
+        emit distantSearchResultsChanged(changes.mDistantSearchReqs) ;
+
     emit changed();
 }