Merge pull request #861 from RetroShare/v0.6-GxsTransport

V0.6 gxs transport

libretroshare/src/chat/distantchat.cc

@@ -243,7 +243,9 @@ void DistantChatService::markDistantChatAsClosed(const DistantChatPeerId& dcpid)
         mDistantChatContacts.erase(it) ;
 }
 
-bool DistantChatService::initiateDistantChatConnexion(const RsGxsId& to_gxs_id, const RsGxsId& from_gxs_id, DistantChatPeerId& dcpid, uint32_t& error_code)
+bool DistantChatService::initiateDistantChatConnexion(
+        const RsGxsId& to_gxs_id, const RsGxsId& from_gxs_id,
+        DistantChatPeerId& dcpid, uint32_t& error_code, bool notify )
 {
     RsGxsTunnelId tunnel_id ;
 
@@ -259,17 +261,19 @@ bool DistantChatService::initiateDistantChatConnexion(const RsGxsId& to_gxs_id,
 
     error_code = RS_DISTANT_CHAT_ERROR_NO_ERROR ;
 
-    // Make a self message to raise the chat window
+	if(notify)
-    
+	{
-    RsChatMsgItem *item = new RsChatMsgItem;
+		// Make a self message to raise the chat window
-    item->message = "[Starting distant chat. Please wait for secure tunnel to be established]" ;
+		RsChatMsgItem *item = new RsChatMsgItem;
-    item->chatFlags = RS_CHAT_FLAG_PRIVATE ;
+		item->message = "[Starting distant chat. Please wait for secure tunnel";
-    item->sendTime = time(NULL) ;
+		item->message += " to be established]";
-    item->PeerId(RsPeerId(tunnel_id)) ;
+		item->chatFlags = RS_CHAT_FLAG_PRIVATE;
-    handleRecvChatMsgItem(item) ;
+		item->sendTime = time(NULL);
-    
+		item->PeerId(RsPeerId(tunnel_id));
-    delete item ;	// item is replaced by NULL if partial, but this is not the case here.
+		handleRecvChatMsgItem(item);
-    
+		delete item ;
+	}
+
     return true ;
 }
 

libretroshare/src/chat/distantchat.h

@@ -45,10 +45,15 @@ public:
     bool processLoadListItem(const RsItem *item) ;
     void addToSaveList(std::list<RsItem*>& list) const;
     
-    // Creates the invite if the public key of the distant peer is available.
+	/**
-    // Om success, stores the invite in the map above, so that we can respond to tunnel requests.
+	 * Creates the invite if the public key of the distant peer is available.
-    //
+	 * On success, stores the invite in the map above, so that we can respond
-    bool initiateDistantChatConnexion(const RsGxsId& to_gxs_id, const RsGxsId &from_gxs_id, DistantChatPeerId& dcpid, uint32_t &error_code) ;
+	 * to tunnel requests. */
+	bool initiateDistantChatConnexion( const RsGxsId& to_gxs_id,
+	                                   const RsGxsId &from_gxs_id,
+	                                   DistantChatPeerId& dcpid,
+	                                   uint32_t &error_code,
+	                                   bool notify = true );
     bool closeDistantChatConnexion(const DistantChatPeerId &tunnel_id) ;
     
     // Sets flags to only allow connexion from some people.

libretroshare/src/chat/p3chatservice.cc

@@ -40,6 +40,7 @@
 #include "pqi/p3historymgr.h"
 #include "rsserver/p3face.h"
 #include "services/p3idservice.h"
+#include "gxstrans/p3gxstrans.h"
 
 #include "chat/p3chatservice.h"
 #include "rsitems/rsconfigitems.h"
@@ -53,40 +54,29 @@ static const uint32_t MAX_AVATAR_JPEG_SIZE              = 32767; // Maximum size
                                                                  // don't transfer correctly and can kill the system.
 																					  // Images are 96x96, which makes approx. 27000 bytes uncompressed.
 
-p3ChatService::p3ChatService(p3ServiceControl *sc,p3IdService *pids, p3LinkMgr *lm, p3HistoryMgr *historyMgr)
+p3ChatService::p3ChatService( p3ServiceControl *sc, p3IdService *pids,
-    : DistributedChatService(getServiceInfo().mServiceType,sc,historyMgr,pids), mChatMtx("p3ChatService"),mServiceCtrl(sc), mLinkMgr(lm) , mHistoryMgr(historyMgr)
+                              p3LinkMgr *lm, p3HistoryMgr *historyMgr,
+                              p3GxsTrans& gxsTransService ) :
+    DistributedChatService(getServiceInfo().mServiceType, sc, historyMgr,pids),
+    mChatMtx("p3ChatService"), mServiceCtrl(sc), mLinkMgr(lm),
+    mHistoryMgr(historyMgr), _own_avatar(NULL),
+    _serializer(new RsChatSerialiser()),
+    mDGMutex("p3ChatService distant id - gxs id map mutex"),
+    mGxsTransport(gxsTransService)
 {
-	_serializer = new RsChatSerialiser() ;
+	addSerialType(_serializer);
-
+	mGxsTransport.registerGxsTransClient( GxsTransSubServices::P3_CHAT_SERVICE,
-	_own_avatar = NULL ;
+	                                      this );
-	_custom_status_string = "" ;
-
-	addSerialType(_serializer) ;
 }
 
-const std::string CHAT_APP_NAME = "chat";
-const uint16_t CHAT_APP_MAJOR_VERSION	= 	1;
-const uint16_t CHAT_APP_MINOR_VERSION  = 	0;
-const uint16_t CHAT_MIN_MAJOR_VERSION  = 	1;
-const uint16_t CHAT_MIN_MINOR_VERSION	=	0;
-
 RsServiceInfo p3ChatService::getServiceInfo()
-{
+{ return RsServiceInfo(RS_SERVICE_TYPE_CHAT, "chat", 1, 0, 1, 0); }
-	return RsServiceInfo(RS_SERVICE_TYPE_CHAT, 
-		CHAT_APP_NAME,
-		CHAT_APP_MAJOR_VERSION, 
-		CHAT_APP_MINOR_VERSION, 
-		CHAT_MIN_MAJOR_VERSION, 
-		CHAT_MIN_MINOR_VERSION);
-}
 
 int	p3ChatService::tick()
 {
-	if(receivedItems()) 
+	if(receivedItems()) receiveChatQueue();
-		receiveChatQueue();
 
-	DistributedChatService::flush() ;
+	DistributedChatService::flush();
-	//DistantChatService::flush() ;
 
 	return 0;
 }
@@ -213,37 +203,38 @@ void p3ChatService::sendGroupChatStatusString(const std::string& status_string)
 	}
 }
 
-void p3ChatService::sendStatusString(const ChatId& id , const std::string& status_string)
+void p3ChatService::sendStatusString( const ChatId& id,
+                                      const std::string& status_string )
 {
-    if(id.isLobbyId())
+	if(id.isLobbyId()) sendLobbyStatusString(id.toLobbyId(),status_string);
-        sendLobbyStatusString(id.toLobbyId(),status_string) ;
+	else if(id.isBroadcast()) sendGroupChatStatusString(status_string);
-    else if(id.isBroadcast())
+	else if(id.isPeerId() || id.isDistantChatId())
-        sendGroupChatStatusString(status_string);
+	{
-    else if(id.isPeerId() || id.isDistantChatId())
+		RsPeerId vpid;
-    {
+		if(id.isDistantChatId()) vpid = RsPeerId(id.toDistantChatId());
-	    RsChatStatusItem *cs = new RsChatStatusItem ;
+		else vpid = id.toPeerId();
 
-	    cs->status_string = status_string ;
+		if(isOnline(vpid))
-	    cs->flags = RS_CHAT_FLAG_PRIVATE ;
+		{
-	    RsPeerId vpid;
+			RsChatStatusItem *cs = new RsChatStatusItem;
-	    if(id.isDistantChatId())
+
-		    vpid = RsPeerId(id.toDistantChatId());
+			cs->status_string = status_string;
-	    else
+			cs->flags = RS_CHAT_FLAG_PRIVATE;
-		    vpid = id.toPeerId();
+			cs->PeerId(vpid);
-        
-	    cs->PeerId(vpid);
 
 #ifdef CHAT_DEBUG
-	    std::cerr  << "sending chat status packet:" << std::endl ;
+			std::cerr  << "sending chat status packet:" << std::endl;
-	    cs->print(std::cerr) ;
+			cs->print(std::cerr);
 #endif
-	    sendChatItem(cs);
+			sendChatItem(cs);
-    }
+		}
-    else
+	}
-    {
+	else
-        std::cerr << "p3ChatService::sendStatusString() Error: chat id of this type is not handled, is it empty?" << std::endl;
+	{
-        return;
+		std::cerr << "p3ChatService::sendStatusString() Error: chat id of this "
-    }
+		          << "type is not handled, is it empty?" << std::endl;
+		return;
+	}
 }
 
 void p3ChatService::clearChatLobby(const ChatId& id)
@@ -253,12 +244,12 @@ void p3ChatService::clearChatLobby(const ChatId& id)
 
 void p3ChatService::sendChatItem(RsChatItem *item)
 {
-	if(DistantChatService::handleOutgoingItem(item))
+	if(DistantChatService::handleOutgoingItem(item)) return;
-		return ;
 #ifdef CHAT_DEBUG
-	std::cerr << "p3ChatService::sendChatItem(): sending to " << item->PeerId() << ": interpreted as friend peer id." << std::endl;
+	std::cerr << "p3ChatService::sendChatItem(): sending to " << item->PeerId()
+	          << ": interpreted as friend peer id." << std::endl;
 #endif
-	sendItem(item) ;
+	sendItem(item);
 }
 
 void p3ChatService::checkSizeAndSendMessage(RsChatMsgItem *msg)
@@ -336,28 +327,50 @@ bool p3ChatService::sendChat(ChatId destination, std::string msg)
     message.incoming = false;
     message.online = true;
 
-    if(!isOnline(vpid))
+	if(!isOnline(vpid))
 	{
-		/* peer is offline, add to outgoing list */
+		message.online = false;
+		RsServer::notify()->notifyChatMessage(message);
+
+		// use the history to load pending messages to the gui
+		// this is not very nice, because the user may think the message was send, while it is still in the queue
+		mHistoryMgr->addMessage(message);
+
+		RsGxsTransId tId = RSRandom::random_u64();
+
+		if(destination.isDistantChatId())
 		{
-			RS_STACK_MUTEX(mChatMtx);
+			RS_STACK_MUTEX(mDGMutex);
-			privateOutgoingList.push_back(ci);
+			DIDEMap::const_iterator it =
+			        mDistantGxsMap.find(destination.toDistantChatId());
+			if(it != mDistantGxsMap.end())
+			{
+				const DistantEndpoints& de(it->second);
+				uint32_t sz = _serializer->size(ci);
+				std::vector<uint8_t> data; data.resize(sz);
+				_serializer->serialise(ci, &data[0], &sz);
+				mGxsTransport.sendData(tId, GxsTransSubServices::P3_CHAT_SERVICE,
+				                       de.from, de.to, &data[0], sz);
+			}
+			else
+				std::cout << "p3ChatService::sendChat(...) can't find distant"
+				          << "chat id in mDistantGxsMap this is unxpected!"
+				          << std::endl;
 		}
 
-        message.online = false;
+		// peer is offline, add to outgoing list
-        RsServer::notify()->notifyChatMessage(message);
+		{
+			RS_STACK_MUTEX(mChatMtx);
+			privateOutgoingMap.insert(outMP::value_type(tId, ci));
+		}
 
-        // use the history to load pending messages to the gui
+		IndicateConfigChanged();
-        // this is not very nice, because the user may think the message was send, while it is still in the queue
+		return false;
-        mHistoryMgr->addMessage(message);
+	}
 
-        IndicateConfigChanged();
+	{
-        return false;
+		RS_STACK_MUTEX(mChatMtx);
-    }
+		std::map<RsPeerId,AvatarInfo*>::iterator it = _avatars.find(vpid);
-
-    {
-        RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
-        std::map<RsPeerId,AvatarInfo*>::iterator it = _avatars.find(vpid) ;
 
         if(it == _avatars.end())
         {
@@ -505,50 +518,50 @@ class MsgCounter
 void p3ChatService::handleIncomingItem(RsItem *item)
 {
 #ifdef CHAT_DEBUG
-	std::cerr << "p3ChatService::receiveChatQueue() Item:" << (void*)item << std::endl ;
+	std::cerr << "p3ChatService::receiveChatQueue() Item:" << (void*)item
+	          << std::endl ;
 #endif
 
 	// RsChatMsgItems needs dynamic_cast, since they have derived siblings.
-	//
+	RsChatMsgItem* ci = dynamic_cast<RsChatMsgItem*>(item);
-	RsChatMsgItem *ci = dynamic_cast<RsChatMsgItem*>(item) ; 
+	if(ci)
-	if(ci != NULL) 
 	{
 		handleRecvChatMsgItem(ci);
-        
-        	if(ci)
-			delete ci ;
 
-		return ;	// don't delete! It's handled by handleRecvChatMsgItem in some specific cases only.
+		/* +ci+ deletion is handled by handleRecvChatMsgItem ONLY in some
+		 * specific cases, in case +ci+ has not been handled deleted it here */
+		delete ci ;
+
+		return;
 	}
 
-//	if(DistantChatService::handleRecvItem(dynamic_cast<RsChatItem*>(item)))
+	if(DistributedChatService::handleRecvItem(dynamic_cast<RsChatItem*>(item)))
-//	{
-//		delete item ;
-//		return ;
-//	}
-
-    if(DistributedChatService::handleRecvItem(dynamic_cast<RsChatItem*>(item)))
 	{
-		delete item ;
+		delete item;
-		return ;
+		return;
 	}
 
 	switch(item->PacketSubType())
 	{
-		case RS_PKT_SUBTYPE_CHAT_STATUS:                handleRecvChatStatusItem(dynamic_cast<RsChatStatusItem*>(item)) ; break ;
+	case RS_PKT_SUBTYPE_CHAT_STATUS:
-		case RS_PKT_SUBTYPE_CHAT_AVATAR:                handleRecvChatAvatarItem(dynamic_cast<RsChatAvatarItem*>(item)) ; break ;
+		handleRecvChatStatusItem(dynamic_cast<RsChatStatusItem*>(item));
-		default:
+		break;
-																		{
+	case RS_PKT_SUBTYPE_CHAT_AVATAR:
-																			static int already = false ;
+		handleRecvChatAvatarItem(dynamic_cast<RsChatAvatarItem*>(item));
-
+		break;
-																			if(!already)
+	default:
-																			{
+	{
-																				std::cerr << "Unhandled item subtype " << (int)item->PacketSubType() << " in p3ChatService: " << std::endl;
+		static int already = false;
-																				already = true ;
+		if(!already)
-																			}
+		{
-																		}
+			std::cerr << "Unhandled item subtype "
+			          << static_cast<int>(item->PacketSubType())
+			          << " in p3ChatService: " << std::endl;
+			already = true;
+		}
 	}
-	delete item ;
+	}
+	delete item;
 }
 
 void p3ChatService::handleRecvChatAvatarItem(RsChatAvatarItem *ca)
@@ -676,35 +689,103 @@ bool p3ChatService::checkForMessageSecurity(RsChatMsgItem *ci)
 	return true ;
 }
 
+bool p3ChatService::initiateDistantChatConnexion( const RsGxsId& to_gxs_id,
+                                                  const RsGxsId& from_gxs_id,
+                                                  DistantChatPeerId& pid,
+                                                  uint32_t& error_code,
+                                                  bool notify )
+{
+	if(DistantChatService::initiateDistantChatConnexion( to_gxs_id,
+	                                                     from_gxs_id, pid,
+	                                                     error_code, notify ))
+	{
+		RS_STACK_MUTEX(mDGMutex);
+		DistantEndpoints ep; ep.from = from_gxs_id; ep.to = to_gxs_id;
+		mDistantGxsMap.insert(DIDEMap::value_type(pid, ep));
+		return true;
+	}
+	return false;
+}
+
+bool p3ChatService::receiveGxsTransMail( const RsGxsId& authorId,
+                                         const RsGxsId& recipientId,
+                                         const uint8_t* data,
+                                         uint32_t dataSize )
+{
+	DistantChatPeerId pid;
+	uint32_t error_code;
+	if(initiateDistantChatConnexion(
+	            authorId, recipientId, pid, error_code, false ))
+	{
+		RsChatMsgItem* item = static_cast<RsChatMsgItem*>(
+		            _serializer->deserialise(
+		                const_cast<uint8_t*>(data), &dataSize ));
+		RsPeerId rd(p3GxsTunnelService::makeGxsTunnelId(authorId, recipientId));
+		item->PeerId(rd);
+		handleRecvChatMsgItem(item);
+		delete item;
+		return true;
+	}
+
+	std::cerr << __PRETTY_FUNCTION__ << " (EE) failed initiating"
+	          << " distant chat connection error: "<< error_code
+	          << std::endl;
+	return false;
+}
+
+bool p3ChatService::notifyGxsTransSendStatus(RsGxsTransId mailId,
+                                             GxsTransSendStatus status )
+{
+	if ( status != GxsTransSendStatus::RECEIPT_RECEIVED ) return true;
+
+	bool changed = false;
+
+	{
+		RS_STACK_MUTEX(mChatMtx);
+		auto it = privateOutgoingMap.find(mailId);
+		if( it != privateOutgoingMap.end() )
+		{
+			privateOutgoingMap.erase(it);
+			changed = true;
+		}
+	}
+
+	if(changed)
+	{
+		RsServer::notify()->notifyListChange(
+		            NOTIFY_LIST_PRIVATE_OUTGOING_CHAT, NOTIFY_TYPE_DEL );
+
+		IndicateConfigChanged();
+	}
+
+	return true;
+}
+
 bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *& ci)
 {
-	time_t now = time(NULL);
 	std::string name;
-    uint32_t popupChatFlag = RS_POPUP_CHAT;
+	uint32_t popupChatFlag = RS_POPUP_CHAT;
 
-    {
+	{
-        RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
+		RS_STACK_MUTEX(mChatMtx);
 
-        if(!locked_checkAndRebuildPartialMessage(ci))		// we make sure this call does not take control over the memory
+		// we make sure this call does not take control over the memory
-		return true ;					// message is a subpart of an existing message. So everything ok, but we need to return.
+		if(!locked_checkAndRebuildPartialMessage(ci)) return true;
-    }
+		    /* message is a subpart of an existing message.
+			 * So everything ok, but we need to return. */
+	}
 
-    // Check for security. This avoids bombing messages, and so on.
+	// Check for security. This avoids bombing messages, and so on.
+	if(!checkForMessageSecurity(ci)) return false;
 
-    if(!checkForMessageSecurity(ci))
+	/* If it's a lobby item, we need to bounce it and possibly check for timings
-        return false ;
+	 * etc. */
-
+	if(!DistributedChatService::handleRecvChatLobbyMsgItem(ci)) return false;
-    // If it's a lobby item, we need to bounce it and possibly check for timings etc.
-
-    if(!DistributedChatService::handleRecvChatLobbyMsgItem(ci))
-        return false ;
 
 #ifdef CHAT_DEBUG
-	std::cerr << "p3ChatService::receiveChatQueue() Item:";
+	std::cerr << "p3ChatService::receiveChatQueue() Item:" << std::endl;
-	std::cerr << std::endl;
 	ci->print(std::cerr);
-	std::cerr << std::endl;
+	std::cerr << std::endl << "Got msg. Flags = " << ci->chatFlags << std::endl;
-	std::cerr << "Got msg. Flags = " << ci->chatFlags << std::endl ;
 #endif
 
     // Now treat normal chat stuff such as avatar requests, except for chat lobbies.
@@ -759,7 +840,7 @@ bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *& ci)
         }
     }
 
-    ci->recvTime = now;
+	ci->recvTime = time(NULL);
 
     ChatMessage cm;
     initChatMessage(ci, cm);
@@ -1087,6 +1168,7 @@ RsChatStatusItem *p3ChatService::makeOwnCustomStateStringItem()
 
 	return ci ;
 }
+
 RsChatAvatarItem *p3ChatService::makeOwnAvatarItem()
 {
 	RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
@@ -1100,11 +1182,11 @@ RsChatAvatarItem *p3ChatService::makeOwnAvatarItem()
 
 void p3ChatService::sendAvatarJpegData(const RsPeerId& peer_id)
 {
-   #ifdef CHAT_DEBUG
+#ifdef CHAT_DEBUG
-   std::cerr << "p3chatservice: sending requested for peer " << peer_id << ", data=" << (void*)_own_avatar << std::endl ;
+	std::cerr << "p3chatservice: sending requested for peer " << peer_id << ", data=" << (void*)_own_avatar << std::endl ;
-   #endif
+#endif
 
-   if(_own_avatar != NULL)
+	if(_own_avatar != NULL)
 	{
 		RsChatAvatarItem *ci = makeOwnAvatarItem();
 		ci->PeerId(peer_id);
@@ -1144,19 +1226,34 @@ bool p3ChatService::loadList(std::list<RsItem*>& load)
 
 	for(std::list<RsItem*>::const_iterator it(load.begin());it!=load.end();++it)
 	{
+		if(PrivateOugoingMapItem* om=dynamic_cast<PrivateOugoingMapItem*>(*it))
+		{
+			RS_STACK_MUTEX(mChatMtx);
+			for( auto& pair : om->store )
+			{
+				privateOutgoingMap.insert(
+				            outMP::value_type(pair.first,
+				                              new RsChatMsgItem(pair.second)) );
+			}
+
+			delete om; continue;
+		}
+
+
 		RsChatAvatarItem *ai = NULL ;
 
 		if(NULL != (ai = dynamic_cast<RsChatAvatarItem *>(*it)))
 		{
-			RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
+			RS_STACK_MUTEX(mChatMtx);
 
 			if(ai->image_size <= MAX_AVATAR_JPEG_SIZE)
 				_own_avatar = new AvatarInfo(ai->image_data,ai->image_size) ;
 			else
-				std::cerr << "Dropping avatar image, because its size is " << ai->image_size << ", and the maximum allowed size is " << MAX_AVATAR_JPEG_SIZE << std::endl;
+				std::cerr << "Dropping avatar image, because its size is "
+				          << ai->image_size << ", and the maximum allowed size "
+				          << "is " << MAX_AVATAR_JPEG_SIZE << std::endl;
 
 			delete *it;
-
 			continue;
 		}
 
@@ -1164,40 +1261,42 @@ bool p3ChatService::loadList(std::list<RsItem*>& load)
 
 		if(NULL != (mitem = dynamic_cast<RsChatStatusItem *>(*it)))
 		{
-			RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
+			RS_STACK_MUTEX(mChatMtx);
 
 			_custom_status_string = mitem->status_string ;
 
 			delete *it;
-
 			continue;
 		}
 
-		RsPrivateChatMsgConfigItem *citem = NULL ;
+		/* TODO: G10h4ck 2017/02/27 this block is kept for retrocompatibility,
-
+		 * and will be used just first time, to load messages in the old format
-		if(NULL != (citem = dynamic_cast<RsPrivateChatMsgConfigItem *>(*it)))
+		 * should be removed in the following RS version */
+		if( RsPrivateChatMsgConfigItem *citem =
+		        dynamic_cast<RsPrivateChatMsgConfigItem *>(*it) )
 		{
-			RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
+			RS_STACK_MUTEX(mChatMtx);
 
-			if (citem->chatFlags & RS_CHAT_FLAG_PRIVATE) {
+			if ( citem->chatFlags & RS_CHAT_FLAG_PRIVATE )
-				if (std::find(ssl_peers.begin(), ssl_peers.end(), citem->configPeerId) != ssl_peers.end()) {
+			{
+				if ( std::find(ssl_peers.begin(), ssl_peers.end(),
+				               citem->configPeerId) != ssl_peers.end() )
+				{
 					RsChatMsgItem *ci = new RsChatMsgItem();
 					citem->get(ci);
 
-					if (citem->configFlags & RS_CHATMSG_CONFIGFLAG_INCOMING) {
+					if (citem->configFlags & RS_CHATMSG_CONFIGFLAG_INCOMING)
+					{
 						locked_storeIncomingMsg(ci);
-					} else {
-						privateOutgoingList.push_back(ci);
 					}
-				} else {
+					else privateOutgoingMap.insert(
-					// no friends
+					            outMP::value_type(RSRandom::random_u64(), ci) );
 				}
-			} else {
+				else { /* no friends */ }
-				// ignore all other items
 			}
+			else { /* ignore all other items */ }
 
 			delete *it;
-
 			continue;
 		}
 
@@ -1228,21 +1327,18 @@ bool p3ChatService::saveList(bool& cleanup, std::list<RsItem*>& list)
 
 	mChatMtx.lock(); /****** MUTEX LOCKED *******/
 
+	PrivateOugoingMapItem* om =  new PrivateOugoingMapItem;
+	typedef std::map<uint64_t, RsChatMsgItem>::value_type vT;
+	for( auto& pair : privateOutgoingMap )
+		om->store.insert(vT(pair.first, *pair.second));
+	list.push_back(om);
+
+
 	RsChatStatusItem *di = new RsChatStatusItem ;
 	di->status_string = _custom_status_string ;
 	di->flags = RS_CHAT_FLAG_CUSTOM_STATE ;
 
-	list.push_back(di) ;
+	list.push_back(di);
-
-	/* save outgoing private chat messages */
-    std::list<RsChatMsgItem *>::iterator it;
-	for (it = privateOutgoingList.begin(); it != privateOutgoingList.end(); it++) {
-		RsPrivateChatMsgConfigItem *ci = new RsPrivateChatMsgConfigItem;
-
-		ci->set(*it, (*it)->PeerId(), 0);
-
-		list.push_back(ci);
-	}
 
 	DistributedChatService::addToSaveList(list) ;
 	DistantChatService::addToSaveList(list) ;
@@ -1269,54 +1365,51 @@ RsSerialiser *p3ChatService::setupSerialiser()
 
 void p3ChatService::statusChange(const std::list<pqiServicePeer> &plist)
 {
-	std::list<pqiServicePeer>::const_iterator it;
+	for (auto it = plist.cbegin(); it != plist.cend(); ++it)
-	for (it = plist.begin(); it != plist.end(); ++it) {
+	{
 		if (it->actions & RS_SERVICE_PEER_CONNECTED) 
 		{
 			/* send the saved outgoing messages */
 			bool changed = false;
 
-			std::vector<RsChatMsgItem*> to_send ;
+			std::vector<RsChatMsgItem*> to_send;
 
-			if (privateOutgoingList.size()) 
 			{
-				RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/
+				RS_STACK_MUTEX(mChatMtx);
 
-				RsPeerId ownId = mServiceCtrl->getOwnId();
+				for( auto cit = privateOutgoingMap.begin();
-
+				     cit != privateOutgoingMap.end(); )
-				std::list<RsChatMsgItem *>::iterator cit = privateOutgoingList.begin();
+				{
-				while (cit != privateOutgoingList.end()) {
+					RsChatMsgItem *c = cit->second;
-					RsChatMsgItem *c = *cit;
+					if (c->PeerId() == it->id)
-
+					{
-					if (c->PeerId() == it->id) {
+						//mHistoryMgr->addMessage(false, c->PeerId(), ownId, c);
-                        //mHistoryMgr->addMessage(false, c->PeerId(), ownId, c);
 
 						to_send.push_back(c) ;
-
 						changed = true;
-
+						cit = privateOutgoingMap.erase(cit);
-						cit = privateOutgoingList.erase(cit);
-
 						continue;
 					}
 
 					++cit;
 				}
-			} /* UNLOCKED */
+			}
 
-			for(uint32_t i=0;i<to_send.size();++i)
+			for(auto toIt = to_send.begin(); toIt != to_send.end(); ++toIt)
-            {
+			{
-                ChatMessage message;
+				ChatMessage message;
-                initChatMessage(to_send[i], message);
+				initChatMessage(*toIt, message);
-                message.incoming = false;
+				message.incoming = false;
-                message.online = true;
+				message.online = true;
-                RsServer::notify()->notifyChatMessage(message);
+				RsServer::notify()->notifyChatMessage(message);
 
-                checkSizeAndSendMessage(to_send[i]); // delete item
+				checkSizeAndSendMessage(*toIt); // delete item
-            }
+			}
 
-			if (changed) {
+			if (changed)
-				RsServer::notify()->notifyListChange(NOTIFY_LIST_PRIVATE_OUTGOING_CHAT, NOTIFY_TYPE_DEL);
+			{
+				RsServer::notify()->notifyListChange(
+				            NOTIFY_LIST_PRIVATE_OUTGOING_CHAT, NOTIFY_TYPE_DEL);
 
 				IndicateConfigChanged();
 			}
@@ -1324,18 +1417,17 @@ void p3ChatService::statusChange(const std::list<pqiServicePeer> &plist)
 		else if (it->actions & RS_SERVICE_PEER_REMOVED) 
 		{
 			/* now handle remove */
-            mHistoryMgr->clear(ChatId(it->id));
+			mHistoryMgr->clear(ChatId(it->id));
 
-            std::list<RsChatMsgItem *>::iterator cit = privateOutgoingList.begin();
+			RS_STACK_MUTEX(mChatMtx);
-            while (cit != privateOutgoingList.end()) {
+			for ( auto cit = privateOutgoingMap.begin();
-                RsChatMsgItem *c = *cit;
+			      cit != privateOutgoingMap.end(); )
-                if (c->PeerId() == it->id) {
+			{
-                    cit = privateOutgoingList.erase(cit);
+				RsChatMsgItem *c = cit->second;
-                    continue;
+				if (c->PeerId() == it->id) cit = privateOutgoingMap.erase(cit);
-                }
+				else ++cit;
-                ++cit;
+			}
-            }
+			IndicateConfigChanged();
-            IndicateConfigChanged();
 		}
 	}
 }

libretroshare/src/chat/p3chatservice.h

@@ -37,6 +37,8 @@
 #include "chat/distantchat.h"
 #include "chat/distributedchat.h"
 #include "retroshare/rsmsgs.h"
+#include "gxstrans/p3gxstrans.h"
+#include "util/rsdeprecate.h"
 
 class p3ServiceControl;
 class p3LinkMgr;
@@ -51,10 +53,12 @@ typedef RsPeerId ChatLobbyVirtualPeerId ;
   * This service uses rsnotify (callbacks librs clients (e.g. rs-gui))
   * @see NotifyBase
   */
-class p3ChatService: public p3Service, public DistantChatService, public DistributedChatService, public p3Config, public pqiServiceMonitor
+struct p3ChatService :
+        p3Service, DistantChatService, DistributedChatService, p3Config,
+        pqiServiceMonitor, GxsTransClient
 {
-public:
+	p3ChatService(p3ServiceControl *cs, p3IdService *pids, p3LinkMgr *cm,
-	p3ChatService(p3ServiceControl *cs, p3IdService *pids,p3LinkMgr *cm, p3HistoryMgr *historyMgr);
+	               p3HistoryMgr *historyMgr, p3GxsTrans& gxsTransService );
 
 	virtual RsServiceInfo getServiceInfo();
 
@@ -66,7 +70,7 @@ public:
 		 * : notifyCustomState, notifyChatStatus, notifyPeerHasNewAvatar
 		 * @see NotifyBase
 		 */
-	virtual int   tick();
+	virtual int tick();
 
 	/*************** pqiMonitor callback ***********************/
 	virtual void statusChange(const std::list<pqiServicePeer> &plist);
@@ -91,11 +95,11 @@ public:
 		 */
 	bool	sendPrivateChat(const RsPeerId &id, const std::string &msg);
 
-	/*!
+	/**
-		 * can be used to send 'immediate' status msgs, these status updates are meant for immediate use by peer (not saved by rs)
+	 * can be used to send 'immediate' status msgs, these status updates are
-		 * e.g currently used to update user when a peer 'is typing' during a chat
+	 * meant for immediate use by peer (not saved by rs) e.g currently used to
-		 */
+	 * update user when a peer 'is typing' during a chat */
-	void  sendStatusString(const ChatId& id,const std::string& status_str) ;
+	void sendStatusString( const ChatId& id, const std::string& status_str );
 
 	/**
 	 * @brief clearChatLobby: Signal chat was cleared by GUI.
@@ -161,6 +165,22 @@ public:
 		 */
 	bool clearPrivateChatQueue(bool incoming, const RsPeerId &id);
 
+	virtual bool initiateDistantChatConnexion( const RsGxsId& to_gxs_id,
+	                                           const RsGxsId& from_gxs_id,
+	                                           DistantChatPeerId &pid,
+	                                           uint32_t& error_code,
+	                                           bool notify = true );
+
+	/// @see GxsTransClient::receiveGxsTransMail(...)
+	virtual bool receiveGxsTransMail( const RsGxsId& authorId,
+	                                  const RsGxsId& recipientId,
+	                                  const uint8_t* data, uint32_t dataSize );
+
+	/// @see GxsTransClient::notifySendMailStatus(...)
+	virtual bool notifyGxsTransSendStatus( RsGxsTransId mailId,
+	                                       GxsTransSendStatus status );
+
+
 protected:
 	/************* from p3Config *******************/
 	virtual RsSerialiser *setupSerialiser() ;
@@ -177,8 +197,9 @@ protected:
 
 	/// This is to be used by subclasses/parents to call IndicateConfigChanged()
 	virtual void triggerConfigSave()  { IndicateConfigChanged() ; }
+
 	/// Same, for storing messages in incoming list
-	virtual void locked_storeIncomingMsg(RsChatMsgItem *) ;
+	RS_DEPRECATED virtual void locked_storeIncomingMsg(RsChatMsgItem *) ;
 
 private:
 	RsMutex mChatMtx;
@@ -231,16 +252,25 @@ private:
 	p3LinkMgr *mLinkMgr;
 	p3HistoryMgr *mHistoryMgr;
 
-	std::list<RsChatMsgItem *> privateOutgoingList; // messages waiting to be send when peer comes online
+	/// messages waiting to be send when peer comes online
+	typedef std::map<uint64_t, RsChatMsgItem *> outMP;
+	outMP privateOutgoingMap;
 
 	AvatarInfo *_own_avatar ;
 	std::map<RsPeerId,AvatarInfo *> _avatars ;
-	std::map<RsPeerId,RsChatMsgItem *> _pendingPartialMessages ;	
+	std::map<RsPeerId,RsChatMsgItem *> _pendingPartialMessages;
 
 	std::string _custom_status_string ;
 	std::map<RsPeerId,StateStringInfo> _state_strings ;
 
-	RsChatSerialiser *_serializer ;
+	RsChatSerialiser *_serializer;
+
+	struct DistantEndpoints { RsGxsId from; RsGxsId to; };
+	typedef std::map<DistantChatPeerId, DistantEndpoints> DIDEMap;
+	DIDEMap mDistantGxsMap;
+	RsMutex mDGMutex;
+
+	p3GxsTrans& mGxsTransport;
 };
 
 class p3ChatService::StateStringInfo

libretroshare/src/chat/rschatitems.cc

@@ -39,27 +39,27 @@ static const uint32_t RS_CHAT_SERIALIZER_FLAGS_NO_SIGNATURE = 0x0001;
 
 RsItem *RsChatSerialiser::create_item(uint16_t service_id,uint8_t item_sub_id) const
 {
-	if (service_id != RS_SERVICE_TYPE_CHAT)
+	if(service_id != RS_SERVICE_TYPE_CHAT) return NULL;
-		return NULL;
 
 	switch(item_sub_id)
 	{
-		case RS_PKT_SUBTYPE_DEFAULT:                       return new RsChatMsgItem() ;
+	case RS_PKT_SUBTYPE_DEFAULT: return new RsChatMsgItem();
-		case RS_PKT_SUBTYPE_PRIVATECHATMSG_CONFIG:         return new RsPrivateChatMsgConfigItem() ;
+	case RS_PKT_SUBTYPE_PRIVATECHATMSG_CONFIG: return new RsPrivateChatMsgConfigItem();
-		case RS_PKT_SUBTYPE_CHAT_STATUS:                   return new RsChatStatusItem() ;
+	case RS_PKT_SUBTYPE_CHAT_STATUS: return new RsChatStatusItem();
-		case RS_PKT_SUBTYPE_CHAT_AVATAR:                   return new RsChatAvatarItem() ;
+	case RS_PKT_SUBTYPE_CHAT_AVATAR: return new RsChatAvatarItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_MSG:         return new RsChatLobbyMsgItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_MSG: return new RsChatLobbyMsgItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE_DEPRECATED:  return new RsChatLobbyInviteItem_Deprecated() ; // to be removed (deprecated since May 2017)
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE: return new RsChatLobbyInviteItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE:             return new RsChatLobbyInviteItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE_DEPRECATED: return new RsChatLobbyInviteItem_Deprecated(); // to be removed (deprecated since May 2017)
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_CHALLENGE:          return new RsChatLobbyConnectChallengeItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_CHALLENGE: return new RsChatLobbyConnectChallengeItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_UNSUBSCRIBE:        return new RsChatLobbyUnsubscribeItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_UNSUBSCRIBE: return new RsChatLobbyUnsubscribeItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_EVENT:       return new RsChatLobbyEventItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_EVENT: return new RsChatLobbyEventItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_LIST_REQUEST:       return new RsChatLobbyListRequestItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_LIST_REQUEST: return new RsChatLobbyListRequestItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_LIST:               return new RsChatLobbyListItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_LIST: return new RsChatLobbyListItem();
-		case RS_PKT_SUBTYPE_CHAT_LOBBY_CONFIG:             return new RsChatLobbyConfigItem() ;
+	case RS_PKT_SUBTYPE_CHAT_LOBBY_CONFIG: return new RsChatLobbyConfigItem();
-		default:
+	case RS_PKT_SUBTYPE_OUTGOING_MAP: return new PrivateOugoingMapItem();
-			std::cerr << "Unknown packet type in chat!" << std::endl ;
+	default:
-			return NULL ;
+		std::cerr << "Unknown packet type in chat!" << std::endl;
+		return NULL;
 	}
 }
 
@@ -205,4 +205,11 @@ void RsPrivateChatMsgConfigItem::get(RsChatMsgItem *ci)
 	ci->recvTime = recvTime;
 }
 
+/* Necessary to serialize `store` that is an STL container with RsChatMsgItem
+ * inside which is a subtype of RsItem */
+RS_REGISTER_ITEM_TYPE(RsChatMsgItem)
 
+void PrivateOugoingMapItem::serial_process(
+        RsGenericSerializer::SerializeJob j,
+        RsGenericSerializer::SerializeContext& ctx )
+{ RS_REGISTER_SERIAL_MEMBER(store); }

libretroshare/src/chat/rschatitems.h

@@ -33,6 +33,7 @@
 #include "rsitems/itempriorities.h"
 #include "rsitems/rsitem.h"
 #include "serialiser/rsserial.h"
+#include "util/rsdeprecate.h"
 
 #include "serialiser/rstlvidset.h"
 #include "serialiser/rstlvfileitem.h"
@@ -77,8 +78,12 @@ const uint8_t RS_PKT_SUBTYPE_DISTANT_CHAT_DH_PUBLIC_KEY   = 0x16 ;
 const uint8_t RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_MSG     	  = 0x17 ;
 const uint8_t RS_PKT_SUBTYPE_CHAT_LOBBY_SIGNED_EVENT      = 0x18 ;
 const uint8_t RS_PKT_SUBTYPE_CHAT_LOBBY_LIST              = 0x19 ;
+
+RS_DEPRECATED_FOR(RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE) \
 const uint8_t RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE_DEPRECATED = 0x1A ;	// to be removed (deprecated since May 2017)
+
 const uint8_t RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE            = 0x1B ;
+const uint8_t RS_PKT_SUBTYPE_OUTGOING_MAP                 = 0x1C ;
 
 typedef uint64_t 		ChatLobbyId ;
 typedef uint64_t 		ChatLobbyMsgId ;
@@ -239,8 +244,8 @@ class RsChatLobbyConnectChallengeItem: public RsChatItem
 		uint64_t challenge_code ;
 };
 
-// to be removed (deprecated since May 2017)
+/// @deprecated since May 2017, to be removed
-class RsChatLobbyInviteItem_Deprecated: public RsChatItem
+class RsChatLobbyInviteItem_Deprecated : public RsChatItem
 {
 	public:
 		RsChatLobbyInviteItem_Deprecated() :RsChatItem(RS_PKT_SUBTYPE_CHAT_LOBBY_INVITE_DEPRECATED) {}
@@ -272,27 +277,28 @@ class RsChatLobbyInviteItem: public RsChatItem
  * For saving incoming and outgoing chat msgs
  * @see p3ChatService
  */
-class RsPrivateChatMsgConfigItem: public RsChatItem
+struct RsPrivateChatMsgConfigItem : RsChatItem
 {
-	public:
+	RsPrivateChatMsgConfigItem() :
-		RsPrivateChatMsgConfigItem() :RsChatItem(RS_PKT_SUBTYPE_PRIVATECHATMSG_CONFIG) {}
+	    RsChatItem(RS_PKT_SUBTYPE_PRIVATECHATMSG_CONFIG) {}
 
-		virtual ~RsPrivateChatMsgConfigItem() {}
+	virtual ~RsPrivateChatMsgConfigItem() {}
-		virtual void clear() {}
+	virtual void clear() {}
 
-		virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+	virtual void serial_process( RsGenericSerializer::SerializeJob j,
+	                             RsGenericSerializer::SerializeContext& ctx );
 
-		/* set data from RsChatMsgItem to RsPrivateChatMsgConfigItem */
+	/** set data from RsChatMsgItem to RsPrivateChatMsgConfigItem */
-		void set(RsChatMsgItem *ci, const RsPeerId &peerId, uint32_t confFlags);
+	void set(RsChatMsgItem *ci, const RsPeerId &peerId, uint32_t confFlags);
-		/* get data from RsPrivateChatMsgConfigItem to RsChatMsgItem */
+	/** get data from RsPrivateChatMsgConfigItem to RsChatMsgItem */
-		void get(RsChatMsgItem *ci);
+	void get(RsChatMsgItem *ci);
 
-		RsPeerId configPeerId;
+	RsPeerId configPeerId;
-		uint32_t chatFlags;
+	uint32_t chatFlags;
-		uint32_t configFlags;
+	uint32_t configFlags;
-		uint32_t sendTime;
+	uint32_t sendTime;
-		std::string message;
+	std::string message;
-		uint32_t recvTime;
+	uint32_t recvTime;
 };
 
 class RsChatLobbyConfigItem: public RsChatItem
@@ -339,12 +345,23 @@ class RsChatAvatarItem: public RsChatItem
 		unsigned char *image_data ;		// image
 };
 
-class RsChatSerialiser: public RsServiceSerializer
-{
-	public:
-		RsChatSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE)
-            :RsServiceSerializer(RS_SERVICE_TYPE_CHAT,RsGenericSerializer::FORMAT_BINARY,flags) {}
 
-		virtual RsItem *create_item(uint16_t service_id,uint8_t item_sub_id) const ;
+struct PrivateOugoingMapItem : RsChatItem
+{
+	PrivateOugoingMapItem() : RsChatItem(RS_PKT_SUBTYPE_OUTGOING_MAP) {}
+
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx );
+
+	std::map<uint64_t, RsChatMsgItem> store;
+};
+
+struct RsChatSerialiser : RsServiceSerializer
+{
+	RsChatSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE) :
+	    RsServiceSerializer( RS_SERVICE_TYPE_CHAT,
+	                         RsGenericSerializer::FORMAT_BINARY, flags ) {}
+
+	virtual RsItem *create_item(uint16_t service_id,uint8_t item_sub_id) const;
 };
 

libretroshare/src/grouter/p3grouter.cc

@@ -1902,7 +1902,9 @@ bool p3GRouter::encryptDataItem(RsGRouterGenericDataItem *item,const RsGxsId& de
     uint32_t encrypted_size =0;
     uint32_t error_status ;
 
-    if(!mGixs->encryptData(item->data_bytes,item->data_size,encrypted_data,encrypted_size,destination_key,true,error_status))
+	if(!mGixs->encryptData( item->data_bytes, item->data_size, encrypted_data,
+	                        encrypted_size, destination_key, error_status,
+	                        true ))
     {
         std::cerr << "(EE) Cannot encrypt: " ;
         if(error_status == RsGixs::RS_GIXS_ERROR_KEY_NOT_AVAILABLE) std::cerr << " key not available for ID = " << destination_key << std::endl;
@@ -1936,7 +1938,9 @@ bool p3GRouter::decryptDataItem(RsGRouterGenericDataItem *item)
     uint32_t decrypted_size =0;
     uint32_t error_status ;
 
-    if(!mGixs->decryptData(item->data_bytes,item->data_size,decrypted_data,decrypted_size,item->destination_key,error_status))
+	if(!mGixs->decryptData( item->data_bytes, item->data_size, decrypted_data,
+	                        decrypted_size, item->destination_key, error_status,
+	                        true ))
     {
         if(error_status == RsGixs::RS_GIXS_ERROR_KEY_NOT_AVAILABLE)
             std::cerr << "(EE) Cannot decrypt incoming message. Key " << item->destination_key << " unknown." << std::endl;
@@ -1986,7 +1990,7 @@ bool p3GRouter::signDataItem(RsGRouterAbstractMsgItem *item,const RsGxsId& signi
         std::cerr << "Created    signature for data hash: " << RsDirUtil::sha1sum(data,data_size) << " and key id=" << signing_id << std::endl;
 //#endif
         // Check signature
-        RsIdentityUsage::UsageCode info;
+        RsIdentityUsage::UsageCode info = RsIdentityUsage::GLOBAL_ROUTER_SIGNATURE_CREATION;
         uint32_t error;
 
         if(verifySignedDataItem(item,info,error))

libretroshare/src/gxs/gxssecurity.cc

@@ -359,7 +359,7 @@ bool GxsSecurity::getSignature(const char *data, uint32_t data_len, const RsTlvP
 	ok &= EVP_SignUpdate(mdctx, data, data_len) == 1;
 
 	unsigned int siglen = EVP_PKEY_size(key_priv);
-	unsigned char sigbuf[siglen];
+    unsigned char sigbuf[siglen] = { 0 };
 	ok &= EVP_SignFinal(mdctx, sigbuf, &siglen, key_priv) == 1;
 
 	// clean up

libretroshare/src/gxs/gxssecurity.h

@@ -36,9 +36,10 @@
 
 
 /*!
- * This contains functionality for performing security
+ * This contains functionality for performing basic security operations needed
- * operations needed to validate data received in RsGenExchange
+ * in RsGenExchange operations.
  * Also has routine for creating security objects around msgs and groups
+ * TODO: Those functions doesn't do param checking!
  */
 class GxsSecurity 
 {

libretroshare/src/gxs/gxstokenqueue.h

@@ -29,41 +29,37 @@
 #include "util/rsthreads.h"
 
 
-/*
+struct GxsTokenQueueItem
- *
- * A little helper class, to manage callbacks from requests
- *
- */
-
-class GxsTokenQueueItem
 {
-	public:
-	GxsTokenQueueItem(const uint32_t token, const uint32_t req_type)
-	:mToken(token),mReqType(req_type) { return; }
 
-	GxsTokenQueueItem(): mToken(0), mReqType(0) { return; }
+	GxsTokenQueueItem(const uint32_t token, const uint32_t req_type) :
+	    mToken(token), mReqType(req_type) {}
+
+	GxsTokenQueueItem(): mToken(0), mReqType(0) {}
 
 	uint32_t mToken;
 	uint32_t mReqType;
 };
 
 
+/**
+ * A little helper class, to manage callbacks from requests
+ */
 class GxsTokenQueue
 {
-	public:
+public:
+	GxsTokenQueue(RsGenExchange *gxs) :
+	    mGenExchange(gxs), mQueueMtx("GxsTokenQueueMtx") {}
 
-	GxsTokenQueue(RsGenExchange *gxs)
+	bool queueRequest(uint32_t token, uint32_t req_type);
-	:mGenExchange(gxs), mQueueMtx("GxsTokenQueueMtx") { return; }
+	void checkRequests(); /// must be called by
-bool	queueRequest(uint32_t token, uint32_t req_type);
 
-void	checkRequests(); // must be called by
+protected:
 
-	protected:
+	/// This must be overloaded to complete the functionality.
+	virtual void handleResponse(uint32_t token, uint32_t req_type);
 
-	// This must be overloaded to complete the functionality.
+private:
-virtual void handleResponse(uint32_t token, uint32_t req_type);
-
-	private:
 	RsGenExchange *mGenExchange;
 	RsMutex mQueueMtx;
 	std::list<GxsTokenQueueItem> mQueue;

libretroshare/src/gxs/rsgenexchange.cc

@@ -85,9 +85,6 @@ RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService
   mChecking(false),
   mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD) + 120),	// this helps unsynchronising the checks for the different services, with 2 min security to avoid checking right away before statistics come up.
   mIntegrityCheck(NULL),
-  CREATE_FAIL(0),
-  CREATE_SUCCESS(1),
-  CREATE_FAIL_TRY_LATER(2),
   SIGN_MAX_WAITING_TIME(60),
   SIGN_FAIL(0),
   SIGN_SUCCESS(1),
@@ -1397,7 +1394,7 @@ bool RsGenExchange::getGroupData(const uint32_t &token, std::vector<RsGxsGrpItem
 	return ok;
 }
 
-bool RsGenExchange::getMsgData(const uint32_t &token, GxsMsgDataMap &msgItems)
+bool RsGenExchange::getMsgData(uint32_t token, GxsMsgDataMap &msgItems)
 {
 	RS_STACK_MUTEX(mGenMtx) ;
 	NxsMsgDataResult msgResult;
@@ -1447,7 +1444,8 @@ bool RsGenExchange::getMsgData(const uint32_t &token, GxsMsgDataMap &msgItems)
 	return ok;
 }
 
-bool RsGenExchange::getMsgRelatedData(const uint32_t &token, GxsMsgRelatedDataMap &msgItems)
+bool RsGenExchange::getMsgRelatedData( uint32_t token,
+                                       GxsMsgRelatedDataMap &msgItems )
 {
 	RS_STACK_MUTEX(mGenMtx) ;
     NxsMsgRelatedDataResult msgResult;

libretroshare/src/gxs/rsgenexchange.h

@@ -338,11 +338,11 @@ public:
      * @param token token to be redeemed for message item retrieval
      * @param msgItems
      */
-    bool getMsgData(const uint32_t &token, GxsMsgDataMap& msgItems);
+	bool getMsgData(uint32_t token, GxsMsgDataMap& msgItems);
 
     template <class MsgType>
-    bool getMsgDataT(const uint32_t &token, std::map<RsGxsGroupId,
+	bool getMsgDataT( uint32_t token, std::map<RsGxsGroupId,
-    		std::vector<MsgType*> >& msgItems)
+	                  std::vector<MsgType*> >& msgItems)
     {
     	GxsMsgDataMap msgData;
     	bool ok = getMsgData(token, msgData);
@@ -379,7 +379,7 @@ public:
      * @param token token to be redeemed for message item retrieval
      * @param msgItems
      */
-    bool getMsgRelatedData(const uint32_t &token, GxsMsgRelatedDataMap& msgItems);
+	bool getMsgRelatedData(uint32_t token, GxsMsgRelatedDataMap& msgItems);
 
 protected:
 
@@ -732,18 +732,19 @@ private:
      */
     uint8_t createGroup(RsNxsGrp* grp, RsTlvSecurityKeySet& keySet);
 
+protected:
     /*!
      * This completes the creation of an instance on RsNxsMsg
      * by assigning it a groupId and signature via SHA1 and EVP_sign respectively
      * What signatures are calculated are based on the authentication policy
      * of the service
      * @param msg the Nxs message to create
-     * CREATE_FAIL, CREATE_SUCCESS, CREATE_ID_SIGN_NOT_AVAIL
      * @return CREATE_SUCCESS for success, CREATE_FAIL for fail,
      * 		   CREATE_FAIL_TRY_LATER for Id sign key not avail (but requested)
      */
     int createMessage(RsNxsMsg* msg);
 
+private:
     /*!
      * convenience function to create sign
      * @param signSet signatures are stored here
@@ -890,10 +891,10 @@ private:
     time_t mLastCheck;
     RsGxsIntegrityCheck* mIntegrityCheck;
 
-private:
+protected:
-
+	enum CreateStatus { CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER };
+	const uint8_t SIGN_MAX_WAITING_TIME;
 	// TODO: cleanup this should be an enum!
-    const uint8_t CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER, SIGN_MAX_WAITING_TIME;
     const uint8_t SIGN_FAIL, SIGN_SUCCESS, SIGN_FAIL_TRY_LATER;
     const uint8_t VALIDATE_FAIL, VALIDATE_SUCCESS, VALIDATE_FAIL_TRY_LATER, VALIDATE_MAX_WAITING_TIME;
 

libretroshare/src/gxs/rsgixs.h

@@ -103,7 +103,7 @@ typedef PGPIdType RsPgpId;
 class RsGixs
 {
 public:
-
+	// TODO: cleanup this should be an enum!
     static const uint32_t RS_GIXS_ERROR_NO_ERROR           = 0x0000 ;
     static const uint32_t RS_GIXS_ERROR_UNKNOWN            = 0x0001 ;
     static const uint32_t RS_GIXS_ERROR_KEY_NOT_AVAILABLE  = 0x0002 ;
@@ -119,8 +119,17 @@ public:
     virtual bool signData(const uint8_t *data,uint32_t data_size,const RsGxsId& signer_id,RsTlvKeySignature& signature,uint32_t& signing_error) = 0 ;
     virtual bool validateData(const uint8_t *data,uint32_t data_size,const RsTlvKeySignature& signature,bool force_load,const RsIdentityUsage& info,uint32_t& signing_error) = 0 ;
 
-    virtual bool encryptData(const uint8_t *clear_data,uint32_t clear_data_size,uint8_t *& encrypted_data,uint32_t& encrypted_data_size,const RsGxsId& encryption_key_id,bool force_load,uint32_t& encryption_error) = 0 ;
+	virtual bool encryptData( const uint8_t *clear_data,
-    virtual bool decryptData(const uint8_t *encrypted_data,uint32_t encrypted_data_size,uint8_t *& clear_data,uint32_t& clear_data_size,const RsGxsId& encryption_key_id,uint32_t& encryption_error) = 0 ;
+	                          uint32_t clear_data_size,
+	                          uint8_t *& encrypted_data,
+	                          uint32_t& encrypted_data_size,
+	                          const RsGxsId& encryption_key_id,
+	                          uint32_t& encryption_error, bool force_load) = 0 ;
+	virtual bool decryptData( const uint8_t *encrypted_data,
+	                          uint32_t encrypted_data_size,
+	                          uint8_t *& clear_data, uint32_t& clear_data_size,
+	                          const RsGxsId& encryption_key_id,
+	                          uint32_t& encryption_error, bool force_load) = 0 ;
 
     virtual bool getOwnIds(std::list<RsGxsId>& ids) = 0;
     virtual bool isOwnId(const RsGxsId& key_id) = 0 ;

libretroshare/src/gxstrans/p3gxstrans.cc

@@ -0,0 +1,925 @@
+/*
+ * GXS Mailing Service
+ * Copyright (C) 2016-2017  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "util/rsdir.h"
+#include "gxstrans/p3gxstrans.h"
+#include "util/stacktrace.h"
+
+typedef unsigned int uint;
+
+RsGxsTrans *rsGxsTrans = NULL ;
+
+p3GxsTrans::~p3GxsTrans()
+{
+	p3Config::saveConfiguration();
+
+	{
+		RS_STACK_MUTEX(mIngoingMutex);
+		for ( auto& kv : mIngoingQueue ) delete kv.second;
+	}
+}
+
+bool p3GxsTrans::getStatistics(GxsTransStatistics& stats)
+{
+    stats.prefered_group_id = mPreferredGroupId;
+    stats.outgoing_records.clear();
+
+    {
+		RS_STACK_MUTEX(mOutgoingMutex);
+
+		for ( auto it = mOutgoingQueue.begin(); it != mOutgoingQueue.end(); ++it)
+		{
+			const OutgoingRecord& pr(it->second);
+
+			RsGxsTransOutgoingRecord rec ;
+            rec.status = pr.status ;
+            rec.send_TS = pr.mailItem.meta.mPublishTs ;
+            rec.group_id = pr.mailItem.meta.mGroupId ;
+            rec.trans_id = pr.mailItem.mailId ;
+            rec.recipient = pr.recipient ;
+            rec.data_size = pr.mailData.size();
+            rec.data_hash = RsDirUtil::sha1sum(pr.mailData.data(),pr.mailData.size());
+            rec.client_service = pr.clientService ;
+
+            stats.outgoing_records.push_back(rec) ;
+        }
+    }
+
+    return true;
+}
+
+bool p3GxsTrans::sendData( RsGxsTransId& mailId,
+                           GxsTransSubServices service,
+                           const RsGxsId& own_gxsid, const RsGxsId& recipient,
+                           const uint8_t* data, uint32_t size,
+                           RsGxsTransEncryptionMode cm )
+{
+	std::cout << "p3GxsTrans::sendEmail(...)" << std::endl;
+
+	if(!mIdService.isOwnId(own_gxsid))
+	{
+		std::cerr << "p3GxsTrans::sendEmail(...) isOwnId(own_gxsid) false!"
+		          << std::endl;
+		return false;
+	}
+
+	if(recipient.isNull())
+	{
+		std::cerr << "p3GxsTrans::sendEmail(...) got invalid recipient"
+		          << std::endl;
+		print_stacktrace();
+		return false;
+	}
+
+	OutgoingRecord pr( recipient, service, data, size );
+
+    pr.mailItem.clear();
+	pr.mailItem.meta.mAuthorId = own_gxsid;
+	pr.mailItem.meta.mMsgId.clear();
+	pr.mailItem.cryptoType = cm;
+	pr.mailItem.mailId = RSRandom::random_u64();
+
+	{
+		RS_STACK_MUTEX(mOutgoingMutex);
+		mOutgoingQueue.insert(prMap::value_type(pr.mailItem.mailId, pr));
+	}
+
+	mailId = pr.mailItem.mailId;
+	return true;
+}
+
+bool p3GxsTrans::querySendStatus(RsGxsTransId mailId, GxsTransSendStatus& st)
+{
+	RS_STACK_MUTEX(mOutgoingMutex);
+	auto it = mOutgoingQueue.find(mailId);
+	if( it != mOutgoingQueue.end() )
+	{
+		st = it->second.status;
+		return true;
+	}
+	return false;
+}
+
+void p3GxsTrans::registerGxsTransClient(
+        GxsTransSubServices serviceType, GxsTransClient* service)
+{
+	RS_STACK_MUTEX(mServClientsMutex);
+	mServClients[serviceType] = service;
+}
+
+void p3GxsTrans::handleResponse(uint32_t token, uint32_t req_type)
+{
+	std::cout << "p3GxsTrans::handleResponse(" << token << ", " << req_type
+	          << ")" << std::endl;
+	switch (req_type)
+	{
+	case GROUPS_LIST:
+	{
+		std::vector<RsGxsGrpItem*> groups;
+		getGroupData(token, groups);
+
+		for( auto grp : groups )
+		{
+			/* For each group check if it is better candidate then
+			 * preferredGroupId, if it is supplant it and subscribe if it is not
+			 * subscribed yet.
+			 * Otherwise if it has recent messages subscribe.
+			 * If the group was already subscribed has no recent messages
+			 * unsubscribe.
+			 */
+
+			const RsGroupMetaData& meta = grp->meta;
+			bool subscribed = IS_GROUP_SUBSCRIBED(meta.mSubscribeFlags);
+			bool old = olderThen( meta.mLastPost,
+			                      UNUSED_GROUP_UNSUBSCRIBE_INTERVAL );
+			bool supersede = supersedePreferredGroup(meta.mGroupId);
+			uint32_t token;
+
+			bool shoudlSubscribe = !subscribed && ( !old || supersede );
+			bool shoudlUnSubscribe = subscribed && old
+			        && meta.mGroupId != mPreferredGroupId;
+
+			if(shoudlSubscribe)
+				RsGenExchange::subscribeToGroup(token, meta.mGroupId, true);
+			else if(shoudlUnSubscribe)
+				RsGenExchange::subscribeToGroup(token, meta.mGroupId, false);
+
+#ifdef GXS_MAIL_GRP_DEBUG
+			char buff[30];
+			struct tm* timeinfo;
+			timeinfo = localtime(&meta.mLastPost);
+			strftime(buff, sizeof(buff), "%Y %b %d %H:%M", timeinfo);
+
+			std::cout << "p3GxsTrans::handleResponse(...) GROUPS_LIST "
+			          << "meta.mGroupId: " << meta.mGroupId
+			          << " meta.mLastPost: " << buff
+			          << " subscribed: " << subscribed
+			          << " old: " << old
+			          << " shoudlSubscribe: " << shoudlSubscribe
+			          << " shoudlUnSubscribe: " << shoudlUnSubscribe
+			          << std::endl;
+#endif // GXS_MAIL_GRP_DEBUG
+
+			delete grp;
+		}
+
+		if(mPreferredGroupId.isNull())
+		{
+			/* This is true only at first run when we haven't received mail
+			 * distribuition groups from friends
+			 * TODO: We should check if we have some connected friend too, to
+			 * avoid to create yet another never used mail distribution group.
+			 */
+
+			std::cerr << "p3GxsTrans::handleResponse(...) preferredGroupId.isNu"
+			          << "ll() let's create a new group." << std::endl;
+			uint32_t token;
+			publishGroup(token, new RsGxsTransGroupItem());
+			queueRequest(token, GROUP_CREATE);
+		}
+
+		break;
+	}
+	case GROUP_CREATE:
+	{
+		std::cerr << "p3GxsTrans::handleResponse(...) GROUP_CREATE" << std::endl;
+		RsGxsGroupId grpId;
+		acknowledgeTokenGrp(token, grpId);
+		supersedePreferredGroup(grpId);
+		break;
+	}
+	case MAILS_UPDATE:
+	{
+		std::cout << "p3GxsTrans::handleResponse(...) MAILS_UPDATE" << std::endl;
+		typedef std::map<RsGxsGroupId, std::vector<RsGxsMsgItem*> > GxsMsgDataMap;
+		GxsMsgDataMap gpMsgMap;
+		getMsgData(token, gpMsgMap);
+		for ( GxsMsgDataMap::iterator gIt = gpMsgMap.begin();
+		      gIt != gpMsgMap.end(); ++gIt )
+		{
+			typedef std::vector<RsGxsMsgItem*> vT;
+			vT& mv(gIt->second);
+			for( vT::const_iterator mIt = mv.begin(); mIt != mv.end(); ++mIt )
+			{
+				RsGxsMsgItem* gIt = *mIt;
+				switch(static_cast<GxsTransItemsSubtypes>(gIt->PacketSubType()))
+				{
+				case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_MAIL:
+				case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_RECEIPT:
+				{
+					RsGxsTransBaseItem* mb =
+					        dynamic_cast<RsGxsTransBaseItem*>(*mIt);
+					if(mb)
+					{
+						RS_STACK_MUTEX(mIngoingMutex);
+						mIngoingQueue.insert(inMap::value_type(mb->mailId, mb));
+					}
+					else
+						std::cerr << "p3GxsTrans::handleResponse(...) "
+						          << "GXS_MAIL_SUBTYPE_MAIL cast error, "
+						          << "something really wrong is happening"
+						          << std::endl;
+					break;
+				}
+				default:
+					std::cerr << "p3GxsTrans::handleResponse(...) MAILS_UPDATE "
+					          << "Unknown mail subtype : "
+					          << static_cast<uint>(gIt->PacketSubType())
+					          << std::endl;
+					delete gIt;
+					break;
+				}
+			}
+		}
+		break;
+	}
+	default:
+		std::cerr << "p3GxsTrans::handleResponse(...) Unknown req_type: "
+		          << req_type << std::endl;
+		break;
+	}
+}
+
+void p3GxsTrans::GxsTransIntegrityCleanupThread::run()
+{
+    // first take out all the groups
+    std::map<RsGxsGroupId, RsNxsGrp*> grp;
+    mDs->retrieveNxsGrps(grp, true, true);
+
+    std::cerr << "GxsTransIntegrityCleanupThread::run()" << std::endl;
+
+    // compute hash and compare to stored value, if it fails then simply add it
+    // to list
+
+    GxsMsgReq grps;
+    for(std::map<RsGxsGroupId, RsNxsGrp*>::iterator git = grp.begin(); git != grp.end(); ++git)
+    {
+	    RsNxsGrp* grp = git->second;
+
+		// store the group for retrieveNxsMsgs
+		grps[grp->grpId];
+
+	    delete grp;
+    }
+
+    // now messages
+
+    std::map<RsGxsTransId,std::pair<RsGxsGroupId,RsGxsMessageId> > stored_msgs ;
+    std::list<RsGxsTransId> received_msgs ;
+
+    GxsMsgResult msgs;
+    mDs->retrieveNxsMsgs(grps, msgs, false, true);
+
+    for(GxsMsgResult::iterator mit = msgs.begin();mit != msgs.end(); ++mit)
+    {
+	    std::vector<RsNxsMsg*>& msgV = mit->second;
+	    std::vector<RsNxsMsg*>::iterator vit = msgV.begin();
+
+	    for(; vit != msgV.end(); ++vit)
+	    {
+		    RsNxsMsg* msg = *vit;
+
+            RsGxsTransSerializer s ;
+            uint32_t size = msg->msg.bin_len;
+            RsItem *item = s.deserialise(msg->msg.bin_data,&size);
+            RsGxsTransMailItem *mitem ;
+            RsGxsTransPresignedReceipt *pitem ;
+
+            if(item == NULL)
+                std::cerr << "  Unrecocognised item type!" << std::endl;
+            else if(NULL != (mitem = dynamic_cast<RsGxsTransMailItem*>(item)))
+            {
+                std::cerr << "  " << msg->metaData->mMsgId << ": Mail data with ID " << std::hex << mitem->mailId << std::dec << " from " << msg->metaData->mAuthorId << " size: " << msg->msg.bin_len << std::endl;
+
+                stored_msgs[mitem->mailId] = std::make_pair(msg->metaData->mGroupId,msg->metaData->mMsgId) ;
+            }
+            else if(NULL != (pitem = dynamic_cast<RsGxsTransPresignedReceipt*>(item)))
+            {
+                std::cerr << "  " << msg->metaData->mMsgId << ": Signed rcpt of ID " << std::hex << pitem->mailId << std::dec << " from " << msg->metaData->mAuthorId << " size: " << msg->msg.bin_len << std::endl;
+
+                received_msgs.push_back(pitem->mailId) ;
+            }
+            else
+                std::cerr << "  Unknown item type!" << std::endl;
+
+		    delete msg;
+	    }
+    }
+
+    GxsMsgReq msgsToDel ;
+
+    std::cerr << "Msg removal report:" << std::endl;
+
+    for(std::list<RsGxsTransId>::const_iterator it(received_msgs.begin());it!=received_msgs.end();++it)
+    {
+		std::map<RsGxsTransId,std::pair<RsGxsGroupId,RsGxsMessageId> >::const_iterator it2 = stored_msgs.find(*it) ;
+
+        if(stored_msgs.end() != it2)
+        {
+            msgsToDel[it2->second.first].push_back(it2->second.second);
+
+            std::cerr << "  scheduling msg " << std::hex << it2->second.first << "," << it2->second.second << " for deletion." << std::endl;
+        }
+    }
+
+    mDs->removeMsgs(msgsToDel);
+}
+
+void p3GxsTrans::service_tick()
+{
+	GxsTokenQueue::checkRequests();
+
+    time_t now = time(NULL);
+
+    if(mLastMsgCleanup + MAX_DELAY_BETWEEN_CLEANUPS < now)
+    {
+        if(!mCleanupThread)
+            mCleanupThread = new GxsTransIntegrityCleanupThread(getDataStore());
+
+        if(mCleanupThread->isRunning())
+            std::cerr << "Cleanup thread is already running. Not running it again!" << std::endl;
+        else
+        {
+            std::cerr << "Starting GxsIntegrity cleanup thread." << std::endl;
+
+			mCleanupThread->start() ;
+            mLastMsgCleanup = now ;
+        }
+    }
+
+	{
+		RS_STACK_MUTEX(mOutgoingMutex);
+		for ( auto it = mOutgoingQueue.begin(); it != mOutgoingQueue.end(); )
+		{
+			OutgoingRecord& pr(it->second);
+			GxsTransSendStatus oldStatus = pr.status;
+			processOutgoingRecord(pr);
+			if (oldStatus != pr.status) notifyClientService(pr);
+			if( pr.status >= GxsTransSendStatus::RECEIPT_RECEIVED )
+				it = mOutgoingQueue.erase(it);
+			else ++it;
+		}
+	}
+
+
+	{
+		RS_STACK_MUTEX(mIngoingMutex);
+		for( auto it = mIngoingQueue.begin(); it != mIngoingQueue.end(); )
+		{
+			switch(static_cast<GxsTransItemsSubtypes>(
+			           it->second->PacketSubType()))
+			{
+			case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_MAIL:
+			{
+				RsGxsTransMailItem* msg =
+				        dynamic_cast<RsGxsTransMailItem*>(it->second);
+				if(!msg)
+				{
+					std::cerr << "p3GxsTrans::service_tick() (EE) "
+					          << "GXS_MAIL_SUBTYPE_MAIL dynamic_cast failed, "
+					          << "something really wrong is happening!"
+					          << std::endl;
+				}
+				else
+				{
+					std::cout << "p3GxsTrans::service_tick() "
+					          << "GXS_MAIL_SUBTYPE_MAIL handling: "
+					          << msg->meta.mMsgId
+					          << " with cryptoType: "
+					          << static_cast<uint32_t>(msg->cryptoType)
+					          << " recipientHint: " << msg->recipientHint
+					          << " mailId: "<< msg->mailId
+					          << " payload.size(): " << msg->payload.size()
+					          << std::endl;
+					handleEncryptedMail(msg);
+				}
+				break;
+			}
+			case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_RECEIPT:
+			{
+				RsGxsTransPresignedReceipt* rcpt =
+				        dynamic_cast<RsGxsTransPresignedReceipt*>(it->second);
+				if(!rcpt)
+				{
+					std::cerr << "p3GxsTrans::service_tick() (EE) "
+					          << "GXS_MAIL_SUBTYPE_RECEIPT dynamic_cast failed,"
+					          << " something really wrong is happening!"
+					          << std::endl;
+				}
+				else if(mIdService.isOwnId(rcpt->meta.mAuthorId))
+				{
+					/* It is a receipt for a mail sent by this node live it in
+					 * ingoingQueue so processOutgoingRecord(...) will take care
+					 * of it at next tick */
+					++it;
+					continue;
+				}
+				else
+				{
+					/* TODO: It is a receipt for a message sent by someone else
+					 * we can delete original mail from our GXS DB without
+					 * waiting for GXS_STORAGE_PERIOD */
+				}
+				break;
+			}
+			default:
+				std::cerr << "p3GxsTrans::service_tick() (EE) got something "
+				          << "really unknown into ingoingQueue!!" << std::endl;
+				break;
+			}
+
+			delete it->second; it = mIngoingQueue.erase(it);
+		}
+	}
+}
+
+RsGenExchange::ServiceCreate_Return p3GxsTrans::service_CreateGroup(
+        RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& /*keySet*/ )
+{
+	std::cout << "p3GxsTrans::service_CreateGroup(...) "
+	          << grpItem->meta.mGroupId << std::endl;
+	return SERVICE_CREATE_SUCCESS;
+}
+
+void p3GxsTrans::notifyChanges(std::vector<RsGxsNotify*>& changes)
+{
+	std::cout << "p3GxsTrans::notifyChanges(...)" << std::endl;
+	for( std::vector<RsGxsNotify*>::const_iterator it = changes.begin();
+	     it != changes.end(); ++it )
+	{
+		RsGxsGroupChange* grpChange = dynamic_cast<RsGxsGroupChange *>(*it);
+		RsGxsMsgChange* msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
+
+		if (grpChange)
+		{
+			std::cout << "p3GxsTrans::notifyChanges(...) grpChange" << std::endl;
+			requestGroupsData(&(grpChange->mGrpIdList));
+		}
+		else if(msgChange)
+		{
+			std::cout << "p3GxsTrans::notifyChanges(...) msgChange" << std::endl;
+			uint32_t token;
+			RsTokReqOptions opts; opts.mReqType = GXS_REQUEST_TYPE_MSG_DATA;
+			RsGenExchange::getTokenService()->requestMsgInfo( token, 0xcaca,
+			                                   opts, msgChange->msgChangeMap );
+			GxsTokenQueue::queueRequest(token, MAILS_UPDATE);
+
+			for( GxsMsgReq::const_iterator it = msgChange->msgChangeMap.begin();
+			     it != msgChange->msgChangeMap.end(); ++it )
+			{
+				const RsGxsGroupId& grpId = it->first;
+				const std::vector<RsGxsMessageId>& msgsIds = it->second;
+				typedef std::vector<RsGxsMessageId>::const_iterator itT;
+				for(itT vit = msgsIds.begin(); vit != msgsIds.end(); ++vit)
+				{
+					const RsGxsMessageId& msgId = *vit;
+					std::cout << "p3GxsTrans::notifyChanges(...) got "
+					          << "notification for message " << msgId
+					          << " in group " << grpId << std::endl;
+				}
+			}
+		}
+	}
+}
+
+uint32_t p3GxsTrans::AuthenPolicy()
+{
+	uint32_t policy = 0;
+	uint32_t flag = 0;
+
+	// This ensure propagated message have valid author signature
+	flag = GXS_SERV::MSG_AUTHEN_ROOT_AUTHOR_SIGN |
+	        GXS_SERV::MSG_AUTHEN_CHILD_AUTHOR_SIGN;
+	RsGenExchange::setAuthenPolicyFlag( flag, policy,
+	                                    RsGenExchange::PUBLIC_GRP_BITS );
+
+	/* This ensure that in for restricted and private groups only authorized
+	 * authors get the messages. Really not used ATM but don't hurts. */
+	flag |= GXS_SERV::MSG_AUTHEN_ROOT_PUBLISH_SIGN |
+	        GXS_SERV::MSG_AUTHEN_CHILD_PUBLISH_SIGN;
+	RsGenExchange::setAuthenPolicyFlag( flag, policy,
+	                                    RsGenExchange::RESTRICTED_GRP_BITS );
+	RsGenExchange::setAuthenPolicyFlag( flag, policy,
+	                                    RsGenExchange::PRIVATE_GRP_BITS );
+
+	/* This seems never used RetroShare wide but we should investigate it
+	 * more before considering this conclusive */
+	flag = 0;
+	RsGenExchange::setAuthenPolicyFlag( flag, policy,
+	                                    RsGenExchange::GRP_OPTION_BITS );
+
+	return policy;
+}
+
+bool p3GxsTrans::requestGroupsData(const std::list<RsGxsGroupId>* groupIds)
+{
+	//	std::cout << "p3GxsTrans::requestGroupsList()" << std::endl;
+	uint32_t token;
+	RsTokReqOptions opts; opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA;
+	if(!groupIds) RsGenExchange::getTokenService()->requestGroupInfo(token, 0xcaca, opts);
+	else RsGenExchange::getTokenService()->requestGroupInfo(token, 0xcaca, opts, *groupIds);
+	GxsTokenQueue::queueRequest(token, GROUPS_LIST);
+	return true;
+}
+
+bool p3GxsTrans::handleEncryptedMail(const RsGxsTransMailItem* mail)
+{
+	std::cout << "p3GxsTrans::handleEcryptedMail(...)" << std::endl;
+
+	std::set<RsGxsId> decryptIds;
+	std::list<RsGxsId> ownIds;
+	mIdService.getOwnIds(ownIds);
+	for(auto it = ownIds.begin(); it != ownIds.end(); ++it)
+		if(mail->maybeRecipient(*it)) decryptIds.insert(*it);
+
+	// Hint match none of our own ids
+	if(decryptIds.empty())
+	{
+		std::cout << "p3GxsTrans::handleEcryptedMail(...) hint doesn't match"
+		          << std::endl;
+		return true;
+	}
+
+	switch (mail->cryptoType)
+	{
+	case RsGxsTransEncryptionMode::CLEAR_TEXT:
+	{
+		uint16_t csri = 0;
+		uint32_t off = 0;
+		getRawUInt16(&mail->payload[0], mail->payload.size(), &off, &csri);
+		std::cerr << "service: " << csri << " got CLEAR_TEXT mail!"
+		          << std::endl;
+		/* As we cannot verify recipient without encryption, just pass the hint
+		 * as recipient */
+		return dispatchDecryptedMail( mail->meta.mAuthorId, mail->recipientHint,
+		                              &mail->payload[0], mail->payload.size() );
+	}
+	case RsGxsTransEncryptionMode::RSA:
+	{
+		bool ok = true;
+		for( std::set<RsGxsId>::const_iterator it = decryptIds.begin();
+		     it != decryptIds.end(); ++it )
+		{
+			const RsGxsId& decryptId(*it);
+			uint8_t* decrypted_data = NULL;
+			uint32_t decrypted_data_size = 0;
+			uint32_t decryption_error;
+			if( mIdService.decryptData( &mail->payload[0],
+			                            mail->payload.size(), decrypted_data,
+			                            decrypted_data_size, decryptId,
+			                            decryption_error ) )
+				ok = ok && dispatchDecryptedMail( mail->meta.mAuthorId,
+				                                  decryptId, decrypted_data,
+				                                  decrypted_data_size );
+			free(decrypted_data);
+		}
+		return ok;
+	}
+	default:
+		std::cout << "Unknown encryption type:"
+		          << static_cast<uint32_t>(mail->cryptoType) << std::endl;
+		return false;
+	}
+}
+
+bool p3GxsTrans::dispatchDecryptedMail( const RsGxsId& authorId,
+                                        const RsGxsId& decryptId,
+                                        const uint8_t* decrypted_data,
+                                        uint32_t decrypted_data_size )
+{
+	std::cout << "p3GxsTrans::dispatchDecryptedMail(, , " << decrypted_data_size
+	          << ")" << std::endl;
+
+	uint16_t csri = 0;
+	uint32_t offset = 0;
+	if(!getRawUInt16( decrypted_data, decrypted_data_size, &offset, &csri))
+	{
+		std::cerr << "p3GxsTrans::dispatchDecryptedMail(...) (EE) fatal error "
+		          << "deserializing service type, something really wrong is "
+		          << "happening!" << std::endl;
+		return false;
+	}
+	GxsTransSubServices rsrvc = static_cast<GxsTransSubServices>(csri);
+
+	uint32_t rcptsize = decrypted_data_size - offset;
+	RsNxsTransPresignedReceipt* receipt =
+	        static_cast<RsNxsTransPresignedReceipt*>(
+	            RsNxsSerialiser(RS_SERVICE_TYPE_GXS_TRANS).deserialise(
+	                const_cast<uint8_t*>(&decrypted_data[offset]), &rcptsize ));
+	offset += rcptsize;
+	if(!receipt)
+	{
+		std::cerr << "p3GxsTrans::dispatchDecryptedMail(...) (EE) fatal error "
+		          << "deserializing presigned return receipt , something really"
+		          << " wrong is happening!" << std::endl;
+		return false;
+	}
+	std::cout << "p3GxsTrans::dispatchDecryptedMail(...) dispatching receipt "
+	          << "with: msgId: " << receipt->msgId << std::endl;
+
+	std::vector<RsNxsMsg*> rcct; rcct.push_back(receipt);
+	RsGenExchange::notifyNewMessages(rcct);
+
+	GxsTransClient* recipientService = NULL;
+	{
+		RS_STACK_MUTEX(mServClientsMutex);
+		recipientService = mServClients[rsrvc];
+	}
+
+	if(recipientService)
+		return recipientService->receiveGxsTransMail(
+		            authorId, decryptId, &decrypted_data[offset],
+		            decrypted_data_size-offset );
+	else
+	{
+		std::cerr << "p3GxsTrans::dispatchReceivedMail(...) "
+		          << "got message for unknown service: "
+		          << csri << std::endl;
+		return false;
+	}
+}
+
+void p3GxsTrans::processOutgoingRecord(OutgoingRecord& pr)
+{
+	//std::cout << "p3GxsTrans::processRecord(...)" << std::endl;
+
+	switch (pr.status)
+	{
+	case GxsTransSendStatus::PENDING_PROCESSING:
+	{
+		pr.mailItem.saltRecipientHint(pr.recipient);
+		pr.mailItem.saltRecipientHint(RsGxsId::random());
+		pr.mailItem.meta.mPublishTs = time(NULL);
+	}
+	case GxsTransSendStatus::PENDING_PREFERRED_GROUP:
+	{
+		if(mPreferredGroupId.isNull())
+		{
+			requestGroupsData();
+			pr.status = GxsTransSendStatus::PENDING_PREFERRED_GROUP;
+			break;
+		}
+
+		pr.mailItem.meta.mGroupId = mPreferredGroupId;
+	}
+	case GxsTransSendStatus::PENDING_RECEIPT_CREATE:
+	{
+		RsGxsTransPresignedReceipt grcpt;
+		grcpt.meta = pr.mailItem.meta;
+		grcpt.meta.mPublishTs = time(NULL);
+		grcpt.mailId = pr.mailItem.mailId;
+		uint32_t grsz = RsGxsTransSerializer().size(&grcpt);
+		std::vector<uint8_t> grsrz;
+		grsrz.resize(grsz);
+		RsGxsTransSerializer().serialise(&grcpt, &grsrz[0], &grsz);
+
+		pr.presignedReceipt.grpId = mPreferredGroupId;
+		pr.presignedReceipt.metaData = new RsGxsMsgMetaData();
+		*pr.presignedReceipt.metaData = grcpt.meta;
+		pr.presignedReceipt.msg.setBinData(&grsrz[0], grsz);
+	}
+	case GxsTransSendStatus::PENDING_RECEIPT_SIGNATURE:
+	{
+		switch (RsGenExchange::createMessage(&pr.presignedReceipt))
+		{
+		case CREATE_SUCCESS: break;
+		case CREATE_FAIL_TRY_LATER:
+			pr.status = GxsTransSendStatus::PENDING_RECEIPT_CREATE;
+			return;
+		default:
+			pr.status = GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE;
+			goto processingFailed;
+		}
+
+		uint32_t metaSize = pr.presignedReceipt.metaData->serial_size();
+		std::vector<uint8_t> srx; srx.resize(metaSize);
+		pr.presignedReceipt.metaData->serialise(&srx[0], &metaSize);
+		pr.presignedReceipt.meta.setBinData(&srx[0], metaSize);
+	}
+	case GxsTransSendStatus::PENDING_PAYLOAD_CREATE:
+	{
+		uint16_t serv = static_cast<uint16_t>(pr.clientService);
+		uint32_t rcptsize = RsGxsTransSerializer().size(&pr.presignedReceipt);
+		uint32_t datasize = pr.mailData.size();
+		pr.mailItem.payload.resize(2 + rcptsize + datasize);
+		uint32_t offset = 0;
+		setRawUInt16(&pr.mailItem.payload[0], 2, &offset, serv);
+		RsGxsTransSerializer().serialise(&pr.presignedReceipt,
+		                                 &pr.mailItem.payload[offset],
+		                                 &rcptsize);
+		offset += rcptsize;
+		memcpy(&pr.mailItem.payload[offset], &pr.mailData[0], datasize);
+	}
+	case GxsTransSendStatus::PENDING_PAYLOAD_ENCRYPT:
+	{
+		switch (pr.mailItem.cryptoType)
+		{
+		case RsGxsTransEncryptionMode::CLEAR_TEXT:
+		{
+			std::cerr << "p3GxsTrans::sendMail(...) you are sending a mail "
+			          << "without encryption, everyone can read it!"
+			          << std::endl;
+			break;
+		}
+		case RsGxsTransEncryptionMode::RSA:
+		{
+			uint8_t* encryptedData = NULL;
+			uint32_t encryptedSize = 0;
+			uint32_t encryptError = 0;
+			if( mIdService.encryptData( &pr.mailItem.payload[0],
+			                           pr.mailItem.payload.size(),
+			                           encryptedData, encryptedSize,
+			                           pr.recipient, encryptError, true ) )
+			{
+				pr.mailItem.payload.resize(encryptedSize);
+				memcpy( &pr.mailItem.payload[0], encryptedData, encryptedSize );
+				free(encryptedData);
+				break;
+			}
+			else
+			{
+				std::cerr << "p3GxsTrans::sendMail(...) RSA encryption failed! "
+				          << "error_status: " << encryptError << std::endl;
+				pr.status = GxsTransSendStatus::FAILED_ENCRYPTION;
+				goto processingFailed;
+			}
+		}
+		case RsGxsTransEncryptionMode::UNDEFINED_ENCRYPTION:
+		default:
+			std::cerr << "p3GxsTrans::sendMail(...) attempt to send mail with "
+			          << "wrong EncryptionMode: "
+			          << static_cast<uint>(pr.mailItem.cryptoType)
+			          << " dropping mail!" << std::endl;
+			pr.status = GxsTransSendStatus::FAILED_ENCRYPTION;
+			goto processingFailed;
+		}
+	}
+	case GxsTransSendStatus::PENDING_PUBLISH:
+	{
+		std::cout << "p3GxsTrans::sendEmail(...) sending mail to: "
+		          << pr.recipient
+		          << " with cryptoType: "
+		          << static_cast<uint>(pr.mailItem.cryptoType)
+		          << " recipientHint: " << pr.mailItem.recipientHint
+		          << " receiptId: " << pr.mailItem.mailId
+		          << " payload size: " << pr.mailItem.payload.size()
+		          << std::endl;
+
+		uint32_t token;
+		publishMsg(token, new RsGxsTransMailItem(pr.mailItem));
+		pr.status = GxsTransSendStatus::PENDING_RECEIPT_RECEIVE;
+		break;
+	}
+	//case GxsTransSendStatus::PENDING_TRANSFER:
+	case GxsTransSendStatus::PENDING_RECEIPT_RECEIVE:
+	{
+		RS_STACK_MUTEX(mIngoingMutex);
+		auto range = mIngoingQueue.equal_range(pr.mailItem.mailId);
+		for( auto it = range.first; it != range.second; ++it)
+		{
+			RsGxsTransPresignedReceipt* rt =
+			        dynamic_cast<RsGxsTransPresignedReceipt*>(it->second);
+			if(rt && mIdService.isOwnId(rt->meta.mAuthorId))
+			{
+				mIngoingQueue.erase(it); delete rt;
+				pr.status = GxsTransSendStatus::RECEIPT_RECEIVED;
+				break;
+			}
+		}
+		// TODO: Resend message if older then treshold
+		break;
+	}
+	case GxsTransSendStatus::RECEIPT_RECEIVED:
+		break;
+
+processingFailed:
+	case GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE:
+	case GxsTransSendStatus::FAILED_ENCRYPTION:
+	default:
+	{
+		std::cout << "p3GxsTrans::processRecord(" << pr.mailItem.mailId
+		          << ") failed with: " << static_cast<uint>(pr.status)
+		          << std::endl;
+		break;
+	}
+	}
+}
+
+void p3GxsTrans::notifyClientService(const OutgoingRecord& pr)
+{
+	RS_STACK_MUTEX(mServClientsMutex);
+	auto it = mServClients.find(pr.clientService);
+	if( it != mServClients.end())
+	{
+		GxsTransClient* serv(it->second);
+		if(serv)
+		{
+			serv->notifyGxsTransSendStatus(pr.mailItem.mailId, pr.status);
+			return;
+		}
+	}
+
+	std::cerr << "p3GxsTrans::processRecord(...) (EE) processed"
+	          << " mail for unkown service: "
+	          << static_cast<uint32_t>(pr.clientService)
+	          << " fatally failed with: "
+	          << static_cast<uint32_t>(pr.status) << std::endl;
+	print_stacktrace();
+}
+
+RsSerialiser* p3GxsTrans::setupSerialiser()
+{
+	RsSerialiser* rss = new RsSerialiser;
+	rss->addSerialType(new RsGxsTransSerializer);
+	return rss;
+}
+
+bool p3GxsTrans::saveList(bool &cleanup, std::list<RsItem *>& saveList)
+{
+	std::cout << "p3GxsTrans::saveList(...)" << saveList.size() << " "
+	          << mIngoingQueue.size() << " " << mOutgoingQueue.size()
+	          << std::endl;
+
+	mOutgoingMutex.lock();
+	mIngoingMutex.lock();
+
+	for ( auto& kv : mOutgoingQueue ) saveList.push_back(&kv.second);
+	for ( auto& kv : mIngoingQueue ) saveList.push_back(kv.second);
+
+	std::cout << "p3GxsTrans::saveList(...)" << saveList.size() << " "
+	          << mIngoingQueue.size() << " " << mOutgoingQueue.size()
+	          << std::endl;
+
+	cleanup = false;
+	return true;
+}
+
+void p3GxsTrans::saveDone()
+{
+	mOutgoingMutex.unlock();
+	mIngoingMutex.unlock();
+}
+
+bool p3GxsTrans::loadList(std::list<RsItem *>&loadList)
+{
+	std::cout << "p3GxsTrans::loadList(...) " << loadList.size() << " "
+	          << mIngoingQueue.size() << " " << mOutgoingQueue.size()
+	          << std::endl;
+
+	for(auto& v : loadList)
+		switch(static_cast<GxsTransItemsSubtypes>(v->PacketSubType()))
+		{
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_MAIL:
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_RECEIPT:
+		{
+			RsGxsTransBaseItem* mi = dynamic_cast<RsGxsTransBaseItem*>(v);
+			if(mi)
+			{
+				RS_STACK_MUTEX(mIngoingMutex);
+				mIngoingQueue.insert(inMap::value_type(mi->mailId, mi));
+			}
+			break;
+		}
+		case GxsTransItemsSubtypes::OUTGOING_RECORD_ITEM:
+		{
+			OutgoingRecord* ot = dynamic_cast<OutgoingRecord*>(v);
+			if(ot)
+			{
+				RS_STACK_MUTEX(mOutgoingMutex);
+				mOutgoingQueue.insert(
+				            prMap::value_type(ot->mailItem.mailId, *ot));
+			}
+			delete v;
+			break;
+		}
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_GROUP:
+		default:
+			std::cerr << "p3GxsTrans::loadList(...) (EE) got item with "
+			          << "unhandled type: "
+			          << static_cast<uint>(v->PacketSubType())
+			          << std::endl;
+			delete v;
+			break;
+		}
+
+	std::cout << "p3GxsTrans::loadList(...) " << loadList.size() << " "
+	          << mIngoingQueue.size() << " " << mOutgoingQueue.size()
+	          << std::endl;
+
+	return true;
+}

libretroshare/src/gxstrans/p3gxstrans.h

@@ -0,0 +1,299 @@
+#pragma once
+/*
+ * GXS Mailing Service
+ * Copyright (C) 2016-2017  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdint.h>
+#include <unordered_map>
+#include <map>
+
+#include "retroshare/rsgxsifacetypes.h" // For RsGxsId, RsGxsCircleId
+#include "gxs/gxstokenqueue.h" // For GxsTokenQueue
+#include "gxstrans/p3gxstransitems.h"
+#include "services/p3idservice.h" // For p3IdService
+#include "util/rsthreads.h"
+#include "retroshare/rsgxstrans.h"
+
+struct p3GxsTrans;
+
+/// Services who want to make use of p3GxsTrans should inherit this struct
+struct GxsTransClient
+{
+	/**
+	 * This will be called by p3GxsTrans to dispatch mails to the subservice
+	 * @param authorId message sender
+	 * @param decryptId recipient id
+	 * @param data buffer containing the decrypted data
+	 * @param dataSize size of the buffer
+	 * @return true if dispatching goes fine, false otherwise
+	 */
+	virtual bool receiveGxsTransMail( const RsGxsId& authorId,
+	                                  const RsGxsId& recipientId,
+	                                  const uint8_t* data, uint32_t dataSize
+	                                  ) = 0;
+
+	/**
+	 * This will be called by p3GxsTrans to notify the subservice about the
+	 * status of a sent email.
+	 * @param originalMessage message for with the notification is made
+	 * @param status the new status of the message
+	 * @return true if notification goes fine, false otherwise (ignored ATM)
+	 */
+	virtual bool notifyGxsTransSendStatus( RsGxsTransId mailId,
+	                                       GxsTransSendStatus status ) = 0;
+};
+
+/**
+ * @brief p3GxsTrans is a mail delivery service based on GXS.
+ * p3GxsTrans is capable of asynchronous mail delivery and acknowledgement.
+ * p3GxsTrans is meant to be capable of multiple encryption options,
+ * @see RsGxsTransEncryptionMode at moment messages are encrypted using RSA
+ * unless the user ask for them being sent in clear text ( this is not supposed
+ * to happen in non testing environment so warnings and stack traces are printed
+ * in the log if an attempt to send something in clear text is made ).
+ * p3GxsTrans try to hide metadata so the travelling message signed by the author
+ * but the recipient is not disclosed, instead to avoid everyone trying to
+ * decrypt every message a hint has been introduced, the hint is calculated in a
+ * way that one can easily prove that a message is not destined to someone, but
+ * cannot prove the message is destined to someone
+ * @see RsGxsTransMailItem::recipientHint for more details.
+ * p3GxsTrans expose a simple API to send and receive mails, the API also
+ * provide notification for the sending mail status @see sendMail(...),
+ * @see querySendStatus(...), @see registerGxsTransClient(...),
+ * @see GxsTransClient::receiveGxsTransMail(...),
+ * @see GxsTransClient::notifyGxsTransSendStatus(...).
+ */
+class p3GxsTrans : public RsGenExchange, public GxsTokenQueue, public p3Config, public RsGxsTrans
+{
+public:
+	p3GxsTrans( RsGeneralDataService* gds, RsNetworkExchangeService* nes,
+	            p3IdService& identities ) :
+	    RsGenExchange( gds, nes, new RsGxsTransSerializer(),
+	                   RS_SERVICE_TYPE_GXS_TRANS, &identities,
+	                   AuthenPolicy(), GXS_STORAGE_PERIOD ),
+	    GxsTokenQueue(this),
+        RsGxsTrans(this),
+        mIdService(identities),
+	    mServClientsMutex("p3GxsTrans client services map mutex"),
+	    mOutgoingMutex("p3GxsTrans outgoing queue map mutex"),
+	    mIngoingMutex("p3GxsTrans ingoing queue map mutex")
+    {
+        mLastMsgCleanup = time(NULL) - 60;	// to be changed into 0
+        mCleanupThread = NULL ;
+    }
+
+	virtual ~p3GxsTrans();
+
+    /*!
+     * \brief getStatistics
+     * 				Gathers all sorts of statistics about the internals of p3GxsTrans, in order to display info about the running status,
+     * 			message transport, etc.
+     * \param stats This structure contains all statistics information.
+     * \return 		true is the call succeeds.
+     */
+
+	virtual bool getStatistics(GxsTransStatistics& stats);
+
+	/**
+	 * Send an email to recipient, in the process author of the email is
+	 * disclosed to the network (because the sent GXS item is signed), while
+	 * recipient is not @see RsGxsTransMailItem::recipientHint for details on
+	 * recipient protection.
+	 * This method is part of the public interface of this service.
+	 * @return true if the mail will be sent, false if not
+	 */
+	bool sendData( RsGxsTransId& mailId,
+	               GxsTransSubServices service,
+	               const RsGxsId& own_gxsid, const RsGxsId& recipient,
+	               const uint8_t* data, uint32_t size,
+	               RsGxsTransEncryptionMode cm = RsGxsTransEncryptionMode::RSA
+	        );
+
+	/**
+	 * This method is part of the public interface of this service.
+	 * @return false if mail is not found in outgoing queue, true otherwise
+	 */
+	bool querySendStatus( RsGxsTransId mailId, GxsTransSendStatus& st );
+
+	/**
+	 * Register a client service to p3GxsTrans to receive mails via
+	 * GxsTransClient::receiveGxsTransMail(...) callback
+	 * This method is part of the public interface of this service.
+	 */
+	void registerGxsTransClient( GxsTransSubServices serviceType,
+	                             GxsTransClient* service );
+
+	/// @see RsGenExchange::getServiceInfo()
+	virtual RsServiceInfo getServiceInfo() { return RsServiceInfo( RS_SERVICE_TYPE_GXS_TRANS, "GXS Mails", 0, 1, 0, 1 ); }
+
+private:
+	/** Time interval of inactivity before a distribution group is unsubscribed.
+	 * Approximatively 3 months seems ok ATM. */
+	const static int32_t UNUSED_GROUP_UNSUBSCRIBE_INTERVAL = 0x76A700;
+
+	/**
+	 * This should be as little as possible as the size of the database can grow
+	 * very fast taking in account we are handling mails for the whole network.
+	 * We do prefer to resend a not acknowledged yet mail after
+	 * GXS_STORAGE_PERIOD has passed and keep it little.
+	 * Tought it can't be too little as this may cause signed receipts to
+	 * get lost thus causing resend and fastly grow perceived async latency, in
+	 * case two sporadically connected users sends mails each other.
+	 * While it is ok for signed acknowledged to stays in the DB for a
+	 * full GXS_STORAGE_PERIOD, mails should be removed as soon as a valid
+	 * signed acknowledged is received for each of them.
+	 * Two weeks seems fair ATM.
+	 */
+	static const uint32_t GXS_STORAGE_PERIOD = 0x127500;
+	static const uint32_t MAX_DELAY_BETWEEN_CLEANUPS = 1203; // every 20 mins. Could be less.
+
+    time_t mLastMsgCleanup ;
+
+	/// Define how the backend should handle authentication based on signatures
+	static uint32_t AuthenPolicy();
+
+	/// Types to mark queries in tokens queue
+	enum GxsReqResTypes
+	{
+		GROUPS_LIST        = 1,
+		GROUP_CREATE       = 2,
+		MAILS_UPDATE       = 3
+	};
+
+	/// Store the id of the preferred GXS group to send emails
+	RsGxsGroupId mPreferredGroupId;
+
+	/// Used for items {de,en}cryption
+	p3IdService& mIdService;
+
+	/// Stores pointers to client services to notify them about new mails
+	std::map<GxsTransSubServices, GxsTransClient*> mServClients;
+	RsMutex mServClientsMutex;
+
+	/**
+	 * @brief Keep track of outgoing mails.
+	 * Records enter the queue when a mail is sent, and are removed when a
+	 * receipt has been received or sending is considered definetly failed.
+	 * Items are saved in config for consistence accross RetroShare shutdowns.
+	 */
+	typedef std::map<RsGxsTransId, OutgoingRecord> prMap;
+	prMap mOutgoingQueue;
+	RsMutex mOutgoingMutex;
+	void processOutgoingRecord(OutgoingRecord& r);
+
+	/**
+	 * @brief Ingoing mail and receipt processing queue.
+	 * At shutdown remaining items are saved in config and then deleted in
+	 * destructor for consistence accross RetroShare instances.
+	 * In order to avoid malicious messages ( non malicious collision has 1/2^64
+	 * probablity ) to smash items in the queue thus causing previous incoming
+	 * item to not being processed and memleaked multimap is used instead of map
+	 * for incoming queue.
+	 */
+	typedef std::unordered_multimap<RsGxsTransId, RsGxsTransBaseItem*> inMap;
+	inMap mIngoingQueue;
+	RsMutex mIngoingMutex;
+
+	/// @see GxsTokenQueue::handleResponse(uint32_t token, uint32_t req_type)
+	virtual void handleResponse(uint32_t token, uint32_t req_type);
+
+	/// @see RsGenExchange::service_tick()
+	virtual void service_tick();
+
+	/// @see RsGenExchange::service_CreateGroup(...)
+	RsGenExchange::ServiceCreate_Return service_CreateGroup(
+	        RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& );
+
+	/// @see RsGenExchange::notifyChanges(std::vector<RsGxsNotify *> &changes)
+	void notifyChanges(std::vector<RsGxsNotify *> &changes);
+
+	/// @see p3Config::setupSerialiser()
+	virtual RsSerialiser* setupSerialiser();
+
+	/// @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
+	virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList);
+
+	/// @see p3Config::saveDone()
+	void saveDone();
+
+	/// @see p3Config::loadList(std::list<RsItem *>&)
+	virtual bool loadList(std::list<RsItem *>& loadList);
+
+	/// Request groups list to GXS backend. Async method.
+	bool requestGroupsData(const std::list<RsGxsGroupId>* groupIds = NULL);
+
+	/**
+	 * Check if current preferredGroupId is the best against potentialGrId, if
+	 * the passed one is better update it.
+	 * Useful when GXS backend notifies groups changes, or when a reponse to an
+	 * async grop request (@see GXS_REQUEST_TYPE_GROUP_*) is received.
+	 * @return true if preferredGroupId has been supeseded by potentialGrId
+	 *   false otherwise.
+	 */
+	bool inline supersedePreferredGroup(const RsGxsGroupId& potentialGrId)
+	{
+		if(mPreferredGroupId < potentialGrId)
+		{
+			std::cerr << "supersedePreferredGroup(...) " << potentialGrId
+			          << " supersed " << mPreferredGroupId << std::endl;
+			mPreferredGroupId = potentialGrId;
+			return true;
+		}
+		return false;
+	}
+
+	/** @return true if has passed more then interval seconds between timeStamp
+	 * and ref. @param ref by default now is taked as reference. */
+	bool static inline olderThen(time_t timeStamp, int32_t interval,
+	                             time_t ref = time(NULL))
+	{ return (timeStamp + interval) < ref; }
+
+
+	/// Decrypt email content and pass it to dispatchDecryptedMail(...)
+	bool handleEncryptedMail(const RsGxsTransMailItem* mail);
+
+	/// Dispatch the message to the recipient service
+	bool dispatchDecryptedMail( const RsGxsId& authorId,
+	                            const RsGxsId& decryptId,
+	                            const uint8_t* decrypted_data,
+	                            uint32_t decrypted_data_size );
+
+	void notifyClientService(const OutgoingRecord& pr);
+
+	/*!
+	 * Checks the integrity message and groups
+	 */
+	class GxsTransIntegrityCleanupThread : public RsSingleJobThread
+	{
+		enum CheckState { CheckStart, CheckChecking };
+
+	public:
+        GxsTransIntegrityCleanupThread(RsGeneralDataService *const dataService): mDs(dataService) {}
+
+		bool isDone();
+		void run();
+
+		void getDeletedIds(std::list<RsGxsGroupId>& grpIds, std::map<RsGxsGroupId, std::vector<RsGxsMessageId> >& msgIds);
+
+	private:
+
+		RsGeneralDataService* const mDs;
+	};
+
+    GxsTransIntegrityCleanupThread *mCleanupThread ;
+};
+

libretroshare/src/gxstrans/p3gxstransitems.cc

@@ -0,0 +1,53 @@
+/*
+ * GXS Mailing Service
+ * Copyright (C) 2016-2017  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "gxstrans/p3gxstransitems.h"
+#include "serialiser/rstypeserializer.h"
+
+const RsGxsId RsGxsTransMailItem::allRecipientsHint("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF");
+
+OutgoingRecord::OutgoingRecord() :
+    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_TRANS,
+            static_cast<uint8_t>(GxsTransItemsSubtypes::OUTGOING_RECORD_ITEM) )
+{ clear();}
+
+OutgoingRecord::OutgoingRecord( RsGxsId rec, GxsTransSubServices cs,
+                                const uint8_t* data, uint32_t size ) :
+    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_TRANS,
+            static_cast<uint8_t>(GxsTransItemsSubtypes::OUTGOING_RECORD_ITEM) ),
+    status(GxsTransSendStatus::PENDING_PROCESSING), recipient(rec),
+    clientService(cs)
+{
+	mailData.resize(size);
+	memcpy(&mailData[0], data, size);
+}
+
+
+RS_REGISTER_ITEM_TYPE(RsGxsTransMailItem)         // for mailItem
+RS_REGISTER_ITEM_TYPE(RsNxsTransPresignedReceipt) // for presignedReceipt
+
+void OutgoingRecord::serial_process(RsGenericSerializer::SerializeJob j,
+                                    RsGenericSerializer::SerializeContext& ctx)
+{
+	RS_REGISTER_SERIAL_MEMBER_TYPED(status, uint8_t);
+	RS_REGISTER_SERIAL_MEMBER(recipient);
+	RS_REGISTER_SERIAL_MEMBER(mailItem);
+	RS_REGISTER_SERIAL_MEMBER(mailData);
+	RS_REGISTER_SERIAL_MEMBER_TYPED(clientService, uint16_t);
+	RS_REGISTER_SERIAL_MEMBER(presignedReceipt);
+}

libretroshare/src/gxstrans/p3gxstransitems.h

@@ -0,0 +1,233 @@
+#pragma once
+/*
+ * GXS Mailing Service
+ * Copyright (C) 2016-2017  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string>
+
+#include "rsitems/rsgxsitems.h"
+#include "serialiser/rsbaseserial.h"
+#include "serialiser/rstlvidset.h"
+#include "retroshare/rsgxsflags.h"
+#include "retroshare/rsgxstrans.h"
+#include "retroshare/rsgxscircles.h" // For: GXS_CIRCLE_TYPE_PUBLIC
+#include "services/p3idservice.h"
+#include "serialiser/rstypeserializer.h"
+
+class RsNxsTransPresignedReceipt : public RsNxsMsg
+{
+public:
+	RsNxsTransPresignedReceipt() : RsNxsMsg(RS_SERVICE_TYPE_GXS_TRANS) {}
+
+    virtual ~RsNxsTransPresignedReceipt() {}
+};
+
+class RsGxsTransBaseItem : public RsGxsMsgItem
+{
+public:
+	RsGxsTransBaseItem(GxsTransItemsSubtypes subtype) :
+	    RsGxsMsgItem( RS_SERVICE_TYPE_GXS_TRANS,
+	                  static_cast<uint8_t>(subtype) ), mailId(0) {}
+
+    virtual ~RsGxsTransBaseItem() {}
+
+	RsGxsTransId mailId;
+
+	void inline clear()
+	{
+		mailId = 0;
+		meta = RsMsgMetaData();
+	}
+
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{ RS_REGISTER_SERIAL_MEMBER_TYPED(mailId, uint64_t); }
+};
+
+class RsGxsTransPresignedReceipt : public RsGxsTransBaseItem
+{
+public:
+	RsGxsTransPresignedReceipt() : RsGxsTransBaseItem(GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_RECEIPT) {}
+    virtual ~RsGxsTransPresignedReceipt() {}
+};
+
+enum class RsGxsTransEncryptionMode : uint8_t
+{
+	CLEAR_TEXT                = 1,
+	RSA                       = 2,
+	UNDEFINED_ENCRYPTION      = 250
+};
+
+class RsGxsTransMailItem : public RsGxsTransBaseItem
+{
+public:
+	RsGxsTransMailItem() :
+	    RsGxsTransBaseItem(GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_MAIL),
+	    cryptoType(RsGxsTransEncryptionMode::UNDEFINED_ENCRYPTION) {}
+
+    virtual ~RsGxsTransMailItem() {}
+
+	RsGxsTransEncryptionMode cryptoType;
+
+	/**
+	 * @brief recipientHint used instead of plain recipient id, so sender can
+	 *  decide the equilibrium between exposing the recipient and the cost of
+	 *  completely anonymize it. So a bunch of luky non recipient can conclude
+	 *  rapidly that they are not the recipient without trying to decrypt the
+	 *  message.
+	 *
+	 * To be able to decide how much metadata we disclose sending a message we
+	 * send an hint instead of the recipient id in clear, the hint cannot be
+	 * false (the recipient would discard the mail) but may be arbitrarly
+	 * obscure like 0xFF...FF so potentially everyone could be the recipient, or
+	 * may expose the complete recipient id or be a middle ground.
+	 * To calculate arbitrary precise hint one do a bitwise OR of the recipients
+	 * keys and an arbitrary salt, the more recipients has the mail and the more
+	 * 1 bits has the salt the less accurate is the hint.
+	 * This way the sender is able to adjust the metadata privacy needed for the
+	 * message, in the more private case (recipientHint == 0xFFF...FFF) no one
+	 * has a clue about who is the actual recipient, while this imply the cost
+	 * that every potencial recipient has to try to decrypt it to know if it is
+	 * for herself. This way a bunch of non recipients can rapidly discover that
+	 * the message is not directed to them without attempting it's decryption.
+	 *
+	 * To check if one id may be the recipient of the mail or not one need to
+	 * bitwise compare the hint with the id, if at least one bit of the hint is
+	 * 0 while the corresponding bit in the id is 1 then the id cannot be the
+	 * recipient of the mail.
+	 *
+	 * Note that by design one can prove that an id is not recipient of the mail
+	 * but cannot prove it is.
+	 * Also in the extreme case of using 0x00...00 as salt that is equivalent
+	 * to not salting at all (aka the plain recipient id is used as hint) a
+	 * malicious observer could not demostrate in a conclusive manner that the
+	 * mail is directed to the actual recipient as the "apparently"
+	 * corresponding hint may be fruit of a "luky" salting of another id.
+	 */
+	RsGxsId recipientHint;
+	void inline saltRecipientHint(const RsGxsId& salt)
+	{ recipientHint = recipientHint | salt; }
+
+	/**
+	 * @brief maybeRecipient given an id and an hint check if they match
+	 * @see recipientHint
+	 * @return true if the id may be recipient of the hint, false otherwise
+	 */
+	bool inline maybeRecipient(const RsGxsId& id) const
+	{ return (~id|recipientHint) == allRecipientsHint; }
+
+	const static RsGxsId allRecipientsHint;
+
+	/** This should travel encrypted, unless EncryptionMode::CLEAR_TEXT
+	 * is specified */
+	std::vector<uint8_t> payload;
+
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RsGxsTransBaseItem::serial_process(j, ctx);
+		RS_REGISTER_SERIAL_MEMBER_TYPED(cryptoType, uint8_t);
+		RS_REGISTER_SERIAL_MEMBER(recipientHint);
+		RS_REGISTER_SERIAL_MEMBER(payload);
+	}
+
+	void clear()
+	{
+		RsGxsTransBaseItem::clear();
+		cryptoType = RsGxsTransEncryptionMode::UNDEFINED_ENCRYPTION;
+		recipientHint.clear();
+		payload.clear();
+	}
+
+	/// Maximum mail size in bytes 10 MiB is more than anything sane can need
+	const static uint32_t MAX_SIZE = 10*8*1024*1024;
+};
+
+class RsGxsTransGroupItem : public RsGxsGrpItem
+{
+public:
+	RsGxsTransGroupItem() :
+	    RsGxsGrpItem( RS_SERVICE_TYPE_GXS_TRANS,
+	                  static_cast<uint8_t>(
+	                      GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_GROUP) )
+	{
+		meta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
+		meta.mGroupName = "Mail";
+		meta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC;
+	}
+    virtual ~RsGxsTransGroupItem() {}
+
+	// TODO: Talk with Cyril why there is no RsGxsGrpItem::serial_process
+	virtual void serial_process(RsGenericSerializer::SerializeJob /*j*/,
+	                            RsGenericSerializer::SerializeContext& /*ctx*/)
+	{}
+
+	void clear() {}
+	std::ostream &print(std::ostream &out, uint16_t /*indent = 0*/)
+	{ return out; }
+};
+
+class RsGxsTransSerializer;
+
+class OutgoingRecord : public RsItem
+{
+public:
+	OutgoingRecord( RsGxsId rec, GxsTransSubServices cs,
+	                const uint8_t* data, uint32_t size );
+
+    virtual ~OutgoingRecord() {}
+
+	GxsTransSendStatus status;
+	RsGxsId recipient;
+	/// Don't use a pointer would be invalid after publish
+	RsGxsTransMailItem mailItem;
+	std::vector<uint8_t> mailData;
+	GxsTransSubServices clientService;
+	RsNxsTransPresignedReceipt presignedReceipt;
+
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx );
+
+	void clear() {}
+
+private:
+	friend class RsGxsTransSerializer;
+	OutgoingRecord();
+};
+
+
+class RsGxsTransSerializer : public RsServiceSerializer
+{
+public:
+	RsGxsTransSerializer() : RsServiceSerializer(RS_SERVICE_TYPE_GXS_TRANS) {}
+	virtual ~RsGxsTransSerializer() {}
+
+	RsItem* create_item(uint16_t service_id, uint8_t item_sub_id) const
+	{
+		if(service_id != RS_SERVICE_TYPE_GXS_TRANS) return NULL;
+
+		switch(static_cast<GxsTransItemsSubtypes>(item_sub_id))
+		{
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_MAIL: return new RsGxsTransMailItem();
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_RECEIPT: return new RsGxsTransPresignedReceipt();
+		case GxsTransItemsSubtypes::GXS_TRANS_SUBTYPE_GROUP: return new RsGxsTransGroupItem();
+		case GxsTransItemsSubtypes::OUTGOING_RECORD_ITEM: return new OutgoingRecord();
+		default: return NULL;
+		}
+	}
+};
+

libretroshare/src/gxstunnel/p3gxstunnel.cc

@@ -1023,7 +1023,8 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item)
 
 // Note: for some obscure reason, the typedef does not work here. Looks like a compiler error. So I use the primary type.
 
-GXSTunnelId p3GxsTunnelService::makeGxsTunnelId(const RsGxsId &own_id, const RsGxsId &distant_id) const	// creates a unique ID from two GXS ids.
+/*static*/ GXSTunnelId p3GxsTunnelService::makeGxsTunnelId(
+        const RsGxsId &own_id, const RsGxsId &distant_id )
 {
     unsigned char mem[RsGxsId::SIZE_IN_BYTES * 2] ;
     
@@ -1244,18 +1245,20 @@ bool p3GxsTunnelService::locked_sendEncryptedTunnelData(RsGxsTunnelItem *item)
     
     std::map<RsGxsTunnelId,GxsTunnelPeerInfo>::iterator it = _gxs_tunnel_contacts.find(tunnel_id) ;
 
-    if(it == _gxs_tunnel_contacts.end())
+	if(it == _gxs_tunnel_contacts.end())
-    {
+	{
 #ifdef DEBUG_GXS_TUNNEL
-        std::cerr << "  Cannot find contact key info for tunnel id " << tunnel_id << ". Cannot send message!" << std::endl;
+		std::cerr << "  Cannot find contact key info for tunnel id "
+		          << tunnel_id << ". Cannot send message!" << std::endl;
 #endif
-        return false;
+		return false;
-    }
+	}
-    if(it->second.status != RS_GXS_TUNNEL_STATUS_CAN_TALK)
+	if(it->second.status != RS_GXS_TUNNEL_STATUS_CAN_TALK)
-    {
+	{
-        std::cerr << "(EE) Cannot talk to tunnel id " << tunnel_id << ". Tunnel status is: " << it->second.status << std::endl;
+		std::cerr << "(EE) Cannot talk to tunnel id " << tunnel_id
-        return false;
+		          << ". Tunnel status is: " << it->second.status << std::endl;
-    }
+		return false;
+	}
 
     it->second.total_sent += rssize ;	// counts the size of clear data that is sent
     

libretroshare/src/gxstunnel/p3gxstunnel.h

@@ -232,7 +232,6 @@ private:
     bool locked_initDHSessionKey(DH *&dh);
     
     TurtleVirtualPeerId virtualPeerIdFromHash(const TurtleFileHash& hash) ;	// ... and to a hash for p3turtle
-    RsGxsTunnelId makeGxsTunnelId(const RsGxsId &own_id, const RsGxsId &distant_id) const;	// creates a unique ID from two GXS ids.
 
     // item handling
     
@@ -256,5 +255,10 @@ private:
     std::map<uint32_t,RsGxsTunnelClientService*> mRegisteredServices ;
     
     void debug_dump();
+
+public:
+	/// creates a unique tunnel ID from two GXS ids.
+	static RsGxsTunnelId makeGxsTunnelId( const RsGxsId &own_id,
+	                                      const RsGxsId &distant_id );
 };
 

libretroshare/src/libretroshare.pro

@@ -868,7 +868,10 @@ gxsphotoshare {
 		rsitems/rsphotoitems.cc \
 }
 
-
+rs_gxs_trans {
+    HEADERS += gxstrans/p3gxstransitems.h gxstrans/p3gxstrans.h
+    SOURCES += gxstrans/p3gxstransitems.cc gxstrans/p3gxstrans.cc
+}
 
 
 

libretroshare/src/pqi/authssl.cc

@@ -599,7 +599,7 @@ bool AuthSSLimpl::SignData(const void *data, const uint32_t len, std::string &si
 
 	EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
         unsigned int signlen = EVP_PKEY_size(mOwnPrivateKey);
-	unsigned char signature[signlen];
+	unsigned char signature[signlen] = { 0 };
 
 	if (0 == EVP_SignInit(mdctx, EVP_sha1()))
 	{

libretroshare/src/pqi/p3cfgmgr.h

@@ -144,7 +144,6 @@ void	setHash(const RsFileHash& h);
  * Class data is protected by mutex's so that anyone can call these
  * functions, at any time.
  */
-
 class p3ConfigMgr
 {
 	public:
@@ -177,7 +176,7 @@ class p3ConfigMgr
          */
         void	addConfiguration(std::string file, pqiConfig *conf);
 
-		/* saves config, and disables further saving
+		/** saves config, and disables further saving
 		 * used for exiting the system
 		 */
 		void	completeConfiguration();
@@ -209,55 +208,53 @@ class p3ConfigMgr
 /***************************************************************************************************/
 
 
-//! abstract class for configuration saving, aimed at rs services that uses RsItem config data
+/**
-/*!
+ * @brief Abstract class for configuration saving.
- * The aim of this class is to provide a way for rs services and object to save particular
+ * Aimed at rs services that uses RsItem config data, provide a way for RS
- * configurations an items (and load them up as well).
+ * services to save and load particular configurations as items.
  */
-class p3Config: public pqiConfig
+struct p3Config : pqiConfig
 {
-	public:
-
 	p3Config();
 
-virtual bool	loadConfiguration(RsFileHash &loadHash);
+	virtual bool loadConfiguration(RsFileHash &loadHash);
-virtual bool	saveConfiguration();
+	virtual bool saveConfiguration();
 
+protected:
 
-	protected:
+	/// Key Functions to be overloaded for Full Configuration
+	virtual RsSerialiser *setupSerialiser() = 0;
 
-	/* Key Functions to be overloaded for Full Configuration */
+	/**
-virtual RsSerialiser *setupSerialiser() = 0;
+	 * saves list of derived object
+	 * @param cleanup this inform you if you need to call saveDone() to
+	 *   unlock/allow access to resources pointed to by handles (list)
+	 *   returned by function: thus false, call saveDone after returned list
+	 *   finished with and vice versa
+	 * @return list of config items derived object wants to saves
+	 */
+	virtual bool saveList(bool &cleanup, std::list<RsItem *>&) = 0;
 
-/**
+	/**
- * saves list of derived object
+	 * loads up list of configs items for derived object
- * @param cleanup this inform you if you need to call saveDone() to unlock/allow
+	 * @param load list of config items to load up
- * access to resources pointed to by handles (list)  returned by function: thus false, call saveDone after returned list finished with
+	 */
- * and vice versa
+	virtual bool loadList(std::list<RsItem *>& load) = 0;
- * @return list of config items derived object wants to saves
- */
-virtual bool saveList(bool &cleanup, std::list<RsItem *>&) = 0;
 
-/**
+	/**
- * loads up list of configs items for derived object
+	 * callback for mutex unlocking
- * @param load list of config items to load up
+	 * in derived classes (should only be needed if cleanup = false)
- */
+	 */
-virtual bool	loadList(std::list<RsItem *>& load) = 0;
+	virtual void saveDone() {}
-
-/**
- * callback for mutex unlocking
- * in derived classes (should only be needed if cleanup = false)
- */
-virtual void    saveDone() { return; }
 
 private:
 
-bool loadConfig();
+	bool loadConfig();
-bool saveConfig();
+	bool saveConfig();
 
-bool loadAttempt(const std::string&,const std::string&, std::list<RsItem *>& load);
+	bool loadAttempt( const std::string&, const std::string&,
-
+	                  std::list<RsItem *>& load );
-}; /* end of p3Config */
+}; // end of p3Config
 
 
 class p3GeneralConfig: public p3Config

libretroshare/src/pqi/pqistore.cc

@@ -396,10 +396,9 @@ bool pqiSSLstore::encryptedSendItems(const std::list<RsItem*>& rsItemList)
 	uint32_t offset = 0;
 
 	for(it = rsItemList.begin(); it != rsItemList.end(); ++it)
-        	if(*it != NULL)
+		if(*it != NULL) sizeItems += rsSerialiser->size(*it);
-			sizeItems += rsSerialiser->size(*it);
 
-    	RsTemporaryMemory data(sizeItems) ;
+	RsTemporaryMemory data( sizeItems ? sizeItems : 1 );
 
 	for(it = rsItemList.begin(); it != rsItemList.end(); ++it)
 	    if(*it != NULL)

libretroshare/src/retroshare/rsgxstrans.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "retroshare/rstokenservice.h"
+#include "retroshare/rsgxsifacehelper.h"
+#include "retroshare/rsgxscommon.h"
+
+/// Subservices identifiers (like port for TCP)
+enum class GxsTransSubServices : uint16_t
+{
+	UNKNOWN         = 0x00,
+	TEST_SERVICE    = 0x01,
+	P3_MSG_SERVICE  = 0x02,
+	P3_CHAT_SERVICE = 0x03
+};
+
+/// Values must fit into uint8_t
+enum class GxsTransItemsSubtypes : uint8_t
+{
+	GXS_TRANS_SUBTYPE_MAIL    = 0x01,
+	GXS_TRANS_SUBTYPE_RECEIPT = 0x02,
+	GXS_TRANS_SUBTYPE_GROUP   = 0x03,
+	OUTGOING_RECORD_ITEM      = 0x04
+};
+
+enum class GxsTransSendStatus : uint8_t
+{
+	UNKNOWN                       = 0x00,
+	PENDING_PROCESSING            = 0x01,
+	PENDING_PREFERRED_GROUP       = 0x02,
+	PENDING_RECEIPT_CREATE        = 0x03,
+	PENDING_RECEIPT_SIGNATURE     = 0x04,
+	PENDING_SERIALIZATION         = 0x05,
+	PENDING_PAYLOAD_CREATE        = 0x06,
+	PENDING_PAYLOAD_ENCRYPT       = 0x07,
+	PENDING_PUBLISH               = 0x08,
+	/** This will be useful so the user can know if the mail reached at least
+	 * some friend node, in case of internet connection interruption */
+	//PENDING_TRANSFER,
+	PENDING_RECEIPT_RECEIVE       = 0x09,
+	/// Records with status >= RECEIPT_RECEIVED get deleted
+	RECEIPT_RECEIVED              = 0x0a,
+	FAILED_RECEIPT_SIGNATURE      = 0xf0,
+	FAILED_ENCRYPTION             = 0xf1
+};
+
+typedef uint64_t RsGxsTransId;
+
+class RsGxsTransGroup
+{
+	public:
+	RsGroupMetaData mMeta;
+};
+
+class RsGxsTransMsg
+{
+public:
+	RsGxsTransMsg() : size(0),data(NULL) {}
+	virtual ~RsGxsTransMsg() { free(data) ; }
+
+public:
+	RsMsgMetaData mMeta;
+
+	uint32_t size ;
+	uint8_t *data ;
+};
+
+struct RsGxsTransOutgoingRecord
+{
+	GxsTransSendStatus status;
+	RsGxsId recipient;
+	RsGxsTransId trans_id;
+
+	GxsTransSubServices client_service;
+
+	uint32_t data_size ;
+	Sha1CheckSum data_hash ;
+	uint32_t send_TS ;
+	RsGxsGroupId group_id ;
+};
+
+class RsGxsTrans: public RsGxsIfaceHelper
+{
+public:
+	class GxsTransStatistics
+	{
+	public:
+		GxsTransStatistics() {}
+
+		RsGxsGroupId prefered_group_id ;
+		std::vector<RsGxsTransOutgoingRecord> outgoing_records;
+	};
+
+	RsGxsTrans(RsGxsIface *gxs) : RsGxsIfaceHelper(gxs) {}
+
+	virtual ~RsGxsTrans() {}
+
+	virtual bool getStatistics(GxsTransStatistics& stats)=0;
+
+//	virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsTransGroup> &groups) = 0;
+//	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsTransMsg> &posts) = 0;
+};
+
+extern RsGxsTrans *rsGxsTrans ;

libretroshare/src/retroshare/rsids.h

@@ -94,6 +94,22 @@ template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER> c
 		inline bool operator==(const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& fp) const { return !memcmp(bytes,fp.bytes,ID_SIZE_IN_BYTES) ; }
 		inline bool operator!=(const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& fp) const { return !!memcmp(bytes,fp.bytes,ID_SIZE_IN_BYTES); }
 		inline bool operator< (const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& fp) const { return (memcmp(bytes,fp.bytes,ID_SIZE_IN_BYTES) < 0) ; }
+		inline t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>
+		operator~ () const
+		{
+			t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER> ret;
+			for(uint32_t i=0; i < ID_SIZE_IN_BYTES; ++i)
+				ret.bytes[i] = ~bytes[i];
+			return ret;
+		}
+		inline t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>
+		operator| (const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& fp) const
+		{
+			t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER> ret;
+			for(uint32_t i=0; i < ID_SIZE_IN_BYTES; ++i)
+				ret.bytes[i] = bytes[i] | fp.bytes[i];
+			return ret;
+		}
 
 		inline bool isNull() const 
 		{ 

libretroshare/src/retroshare/rsiface.h

@@ -53,9 +53,10 @@ int InitRetroShare(int argc, char **argv, RsInit *config);
 
 class RsControl /* The Main Interface Class - for controlling the server */
 {
-	public:
+public:
-		static RsControl *instance() ;
+	/// TODO: This should return a reference instead of a pointer!
-		static void earlyInitNotificationSystem() { instance() ; }
+	static RsControl *instance();
+	static void earlyInitNotificationSystem() { instance(); }
 
 		/* Real Startup Fn */
 		virtual int StartupRetroShare() = 0;

libretroshare/src/retroshare/rstokenservice.h

@@ -138,7 +138,7 @@ public:
 
     /*!
      * Use this to request group related information
-     * @param token The token returned for the request, store this value to pool for request completion
+	 * @param token The token returned for the request, store this value to poll for request completion
      * @param ansType The type of result (e.g. group data, meta, ids)
      * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
      * @param groupIds group id to request info for
@@ -148,7 +148,7 @@ public:
 
     /*!
      * Use this to request all group related info
-     * @param token The token returned for the request, store this value to pool for request completion
+	 * @param token The token returned for the request, store this value to poll for request completion
      * @param ansType The type of result (e.g. group data, meta, ids)
      * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
      * @return
@@ -156,7 +156,7 @@ public:
     virtual bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts) = 0;
 
     /*!
-     * Use this to get msg related information, store this value to pole for request completion
+	 * Use this to get msg related information, store this value to poll for request completion
      * @param token The token returned for the request
      * @param ansType The type of result wanted
      * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
@@ -166,7 +166,7 @@ public:
     virtual bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const GxsMsgReq& msgIds) = 0;
 
     /*!
-     * Use this to get msg related information, store this value to pole for request completion
+	 * Use this to get msg related information, store this value to poll for request completion
      * @param token The token returned for the request
      * @param ansType The type of result wanted
      * @param opts Additional option that affect outcome of request. Please see specific services, for valid values

libretroshare/src/rsitems/rsitem.h

@@ -1,8 +1,11 @@
 #pragma once
 
+#include <typeinfo> // for typeid
+
 #include "util/smallobject.h"
 #include "retroshare/rstypes.h"
 #include "serialiser/rsserializer.h"
+#include "util/stacktrace.h"
 
 #include <typeinfo>
 
@@ -17,6 +20,8 @@ class RsItem: public RsMemoryManagement::SmallObject
 #endif
 
 		virtual ~RsItem();
+
+		/// TODO: Do this make sense with the new serialization system?
 		virtual void clear() = 0;
 
         virtual std::ostream &print(std::ostream &out, uint16_t /* indent */ = 0)
@@ -50,21 +55,16 @@ class RsItem: public RsMemoryManagement::SmallObject
 		inline void setPriorityLevel(uint8_t l) { _priority_level = l ;}
 
 		/**
-		 * @brief serialize this object to the given buffer
+		 * TODO: This should be made pure virtual as soon as all the codebase
-		 * @param Job to do: serialise or deserialize.
+		 * is ported to the new serialization system
-		 * @param data Chunk of memory were to dump the serialized data
-		 * @param size Size of memory chunk
-		 * @param offset Readed to determine at witch offset start writing data,
-		 *        written to inform caller were written data ends, the updated value
-		 *        is usually passed by the caller to serialize of another
-		 *        RsSerializable so it can write on the same chunk of memory just
-		 *        after where this RsSerializable has been serialized.
-		 * @return true if serialization successed, false otherwise
 		 */
-
+		virtual void serial_process(RsGenericSerializer::SerializeJob,
-		virtual void serial_process(RsGenericSerializer::SerializeJob /* j */,RsGenericSerializer::SerializeContext& /* ctx */)
+		                            RsGenericSerializer::SerializeContext&)// = 0;
 		{
-			std::cerr << "(EE) RsItem::serial_process() called by an item using new serialization classes, but not derived! Class is " << typeid(*this).name() << std::endl;
+			std::cerr << "(EE) RsItem::serial_process() called by an item using"
+			          << "new serialization classes, but not derived! Class is "
+			          << typeid(*this).name() << std::endl;
+			print_stacktrace();
 		}
 
 	protected:
@@ -73,6 +73,7 @@ class RsItem: public RsMemoryManagement::SmallObject
 		uint8_t _priority_level ;
 };
 
+/// TODO: Do this make sense with the new serialization system?
 class RsRawItem: public RsItem
 {
 public:

libretroshare/src/rsitems/rsnxsitems.cc

@@ -69,15 +69,18 @@ void RsNxsSyncMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGene
     RsTypeSerializer::serial_process          (j,ctx,msgId            ,"msgId") ;
     RsTypeSerializer::serial_process          (j,ctx,authorId         ,"authorId") ;
 }
-void RsNxsMsg::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+
+void RsNxsMsg::serial_process( RsGenericSerializer::SerializeJob j,
+                               RsGenericSerializer::SerializeContext& ctx )
 {
-    RsTypeSerializer::serial_process<uint32_t> (j,ctx,transactionNumber,"transactionNumber") ;
+	RS_REGISTER_SERIAL_MEMBER_TYPED(transactionNumber, uint32_t);
-    RsTypeSerializer::serial_process<uint8_t>  (j,ctx,pos              ,"pos") ;
+	RS_REGISTER_SERIAL_MEMBER_TYPED(pos, uint8_t);
-    RsTypeSerializer::serial_process           (j,ctx,msgId            ,"msgId") ;
+	RS_REGISTER_SERIAL_MEMBER(msgId);
-    RsTypeSerializer::serial_process           (j,ctx,grpId            ,"grpId") ;
+	RS_REGISTER_SERIAL_MEMBER(grpId);
-    RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,msg              ,"msg") ;
+	RS_REGISTER_SERIAL_MEMBER_TYPED(msg, RsTlvItem);
-    RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,meta             ,"meta") ;
+	RS_REGISTER_SERIAL_MEMBER_TYPED(meta, RsTlvItem);
 }
+
 void RsNxsGrp::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t> (j,ctx,transactionNumber,"transactionNumber") ;
@@ -149,11 +152,14 @@ void RsNxsMsg::clear()
     meta.TlvClear();
 }
 
+std::ostream&RsNxsMsg::print(std::ostream& out, uint16_t /*indent*/)
+{ return out; }
+
 void RsNxsGrp::clear()
 {
-    grpId.clear();
+	grpId.clear();
-    grp.TlvClear();
+	grp.TlvClear();
-    meta.TlvClear();
+	meta.TlvClear();
 }
 
 RsNxsGrp* RsNxsGrp::clone() const {

libretroshare/src/rsitems/rsnxsitems.h

@@ -365,38 +365,38 @@ public:
  * Used to respond to a RsGrpMsgsReq
  * with message items satisfying request
  */
-class RsNxsMsg : public RsNxsItem
+struct RsNxsMsg : RsNxsItem
 {
-public:
+	RsNxsMsg(uint16_t servtype) :
+	    RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_MSG_ITEM), meta(servtype),
+	    msg(servtype), metaData(NULL) { clear(); }
+	virtual ~RsNxsMsg() { delete metaData; }
 
-    RsNxsMsg(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_MSG_ITEM), meta(servtype), msg(servtype),
+	virtual void serial_process( RsGenericSerializer::SerializeJob j,
-    metaData(NULL) { clear(); }
+	                             RsGenericSerializer::SerializeContext& ctx );
-    virtual ~RsNxsMsg() { if(metaData) delete metaData; }
 
-    virtual void clear();
+	virtual void clear();
+	virtual std::ostream &print(std::ostream& out, uint16_t indent);
 
-	virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+	uint8_t pos; /// used for splitting up msg
+	uint8_t count; /// number of split up messages
+	RsGxsGroupId grpId; /// group id, forms part of version id
+	RsGxsMessageId msgId; /// msg id
+	static int refcount;
 
-    uint8_t pos; /// used for splitting up msg
+	/*!
-    uint8_t count; /// number of split up messages
+	 * This should contains all the data
-    RsGxsGroupId grpId; /// group id, forms part of version id
+	 * which is not specific to the Gxs service data
-    RsGxsMessageId msgId; /// msg id
+	 */
-    static int refcount;
+	RsTlvBinaryData meta;
 
-    /*!
+	/*!
-     * This should contains all the data
+	 * This contains Gxs specific data
-     * which is not specific to the Gxs service data
+	 * only client of API knows how to decode this
-     */
+	 */
-    RsTlvBinaryData meta;
+	RsTlvBinaryData msg;
-
-    /*!
-     * This contains Gxs specific data
-     * only client of API knows who to decode this
-     */
-    RsTlvBinaryData msg;
-
-    RsGxsMsgMetaData* metaData;
 
+	RsGxsMsgMetaData* metaData;
 };
 
 /*!

libretroshare/src/rsitems/rsserviceids.h

@@ -82,6 +82,7 @@ const uint16_t RS_SERVICE_GXS_TYPE_GXSCIRCLE   = 0x0218;
 // not gxs, but used with identities.
 const uint16_t RS_SERVICE_GXS_TYPE_REPUTATION  = 0x0219;
 const uint16_t RS_SERVICE_TYPE_GXS_RECOGN      = 0x0220;
+const uint16_t RS_SERVICE_TYPE_GXS_TRANS       = 0x0230;
 
 // Experimental Services.
 /* DSDV Testing at the moment - Service Only */

libretroshare/src/rsserver/p3face-server.cc

@@ -122,7 +122,7 @@ RsServer::RsServer()
 
 RsServer::~RsServer()
 {
-	return;
+	delete mGxsTrans;
 }
 
         /* General Internal Helper Functions 

libretroshare/src/rsserver/p3face.h

@@ -181,6 +181,7 @@ class RsServer: public RsControl, public RsTickingThread
 //		p3GxsForums *mGxsForums;
 //		p3GxsChannels *mGxsChannels;
 //		p3Wire *mWire;
+		p3GxsTrans* mGxsTrans;
 
 		/* Config */
 		p3ConfigMgr     *mConfigMgr;

libretroshare/src/rsserver/rsinit.cc

@@ -82,6 +82,10 @@
 #include "tcponudp/udpstunner.h"
 #endif // RS_USE_DHT_STUNNER
 
+#ifdef RS_GXS_TRANS
+#	include "gxstrans/p3gxstrans.h"
+#endif
+
 // #define GPG_DEBUG
 // #define AUTHSSL_DEBUG
 // #define FIM_DEBUG
@@ -945,14 +949,11 @@ RsGRouter *rsGRouter = NULL ;
 
 RsControl *RsControl::instance()
 {
-	static RsServer *rsicontrol = NULL ;
+	static RsServer rsicontrol;
-
+	return &rsicontrol;
-	if(rsicontrol == NULL) 
-		rsicontrol = new RsServer();
-	
-	return rsicontrol;
 }
 
+
 /*
  * The Real RetroShare Startup Function.
  */
@@ -1487,8 +1488,21 @@ int RsServer::StartupRetroShare()
         pqih->addService(gxschannels_ns, true);
         //pqih->addService(photo_ns, true);
 
-        // remove pword from memory
+#	ifdef RS_GXS_TRANS
-        rsInitConfig->gxs_passwd = "";
+	RsGeneralDataService* gxstrans_ds = new RsDataService(
+	            currGxsDir + "/", "gxstrans_db", RS_SERVICE_TYPE_GXS_TRANS,
+	            NULL, rsInitConfig->gxs_passwd );
+	mGxsTrans = new p3GxsTrans(gxstrans_ds, NULL, *mGxsIdService);
+	RsGxsNetService* gxstrans_ns = new RsGxsNetService(
+	            RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans,
+	            mGxsTrans->getServiceInfo(), mReputations, mGxsCircles,
+	            mGxsIdService, pgpAuxUtils);
+	mGxsTrans->setNetworkExchangeService(gxstrans_ns);
+	pqih->addService(gxstrans_ns, true);
+#	endif // RS_GXS_TRANS
+
+	// remove pword from memory
+	rsInitConfig->gxs_passwd = "";
 
 #endif // RS_ENABLE_GXS.
 
@@ -1496,8 +1510,9 @@ int RsServer::StartupRetroShare()
 	p3ServiceInfo *serviceInfo = new p3ServiceInfo(serviceCtrl);
 	mDisc = new p3discovery2(mPeerMgr, mLinkMgr, mNetMgr, serviceCtrl);
 	mHeart = new p3heartbeat(serviceCtrl, pqih);
-	msgSrv = new p3MsgService(serviceCtrl,mGxsIdService);
+	msgSrv = new p3MsgService( serviceCtrl, mGxsIdService, *mGxsTrans );
-	chatSrv = new p3ChatService(serviceCtrl,mGxsIdService, mLinkMgr, mHistoryMgr);
+	chatSrv = new p3ChatService( serviceCtrl,mGxsIdService, mLinkMgr,
+	                             mHistoryMgr, *mGxsTrans );
 	mStatusSrv = new p3StatusService(serviceCtrl);
 
 #ifdef ENABLE_GROUTER
@@ -1656,13 +1671,20 @@ int RsServer::StartupRetroShare()
 #ifdef ENABLE_GROUTER
 	mConfigMgr->addConfiguration("grouter.cfg", gr);
 #endif
-    mConfigMgr->addConfiguration("p3identity.cfg", mGxsIdService);
 
 #ifdef RS_USE_BITDHT
-    mConfigMgr->addConfiguration("bitdht.cfg", mBitDht);
+	mConfigMgr->addConfiguration("bitdht.cfg", mBitDht);
 #endif
 
 #ifdef RS_ENABLE_GXS
+
+#	ifdef RS_GXS_TRANS
+	mConfigMgr->addConfiguration("gxs_trans_ns.cfg", gxstrans_ns);
+	mConfigMgr->addConfiguration("gxs_trans.cfg", mGxsTrans);
+#	endif // RS_GXS_TRANS
+
+	mConfigMgr->addConfiguration("p3identity.cfg", mGxsIdService);
+
 	mConfigMgr->addConfiguration("identity.cfg", gxsid_ns);
 	mConfigMgr->addConfiguration("gxsforums.cfg", gxsforums_ns);
 	mConfigMgr->addConfiguration("gxschannels.cfg", gxschannels_ns);
@@ -1828,6 +1850,8 @@ int RsServer::StartupRetroShare()
     rsPosted = mPosted;
     rsGxsForums = mGxsForums;
     rsGxsChannels = mGxsChannels;
+    rsGxsTrans = mGxsTrans;
+
     //rsPhoto = mPhoto;
     //rsWire = mWire;
 
@@ -1857,6 +1881,11 @@ int RsServer::StartupRetroShare()
 	//createThread(*photo_ns);
 	//createThread(*wire_ns);
 
+#	ifdef RS_GXS_TRANS
+	startServiceThread(mGxsTrans, "gxs trans");
+	startServiceThread(gxstrans_ns, "gxs trans ns");
+#	endif // RS_GXS_TRANS
+
 #endif // RS_ENABLE_GXS
 
 	ftserver->StartupThreads();

libretroshare/src/serialiser/rstypeserializer.cc

@@ -27,6 +27,8 @@
 #include "serialiser/rsbaseserial.h"
 #include "serialiser/rstlvkeys.h"
 
+#include "rsitems/rsitem.h"
+
 #include "util/rsprint.h"
 
 #include <iomanip>
@@ -309,3 +311,57 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const RsTlvIt
 
     std::cerr << "  [" << typeid(s).name() << "] " << n << std::endl;
 }
+
+//============================================================================//
+//                          RsItem and derivated                              //
+//============================================================================//
+
+template<> uint32_t RsTypeSerializer::serial_size(const RsItem& s)
+{
+	RsGenericSerializer::SerializeContext ctx(
+	            NULL, 0, RsGenericSerializer::FORMAT_BINARY,
+	            RsGenericSerializer::SERIALIZATION_FLAG_NONE );
+
+	ctx.mOffset = 8;	// header size
+	const_cast<RsItem&>(s).serial_process(RsGenericSerializer::SIZE_ESTIMATE,
+	                                      ctx);
+
+	return ctx.mOffset;
+}
+
+template<> bool RsTypeSerializer::serialize( uint8_t data[], uint32_t size,
+                                             uint32_t &offset, const RsItem& s )
+{
+	RsGenericSerializer::SerializeContext ctx(
+	            data, size, RsGenericSerializer::FORMAT_BINARY,
+	            RsGenericSerializer::SERIALIZATION_FLAG_NONE );
+	ctx.mOffset = offset;
+	const_cast<RsItem&>(s).serial_process(RsGenericSerializer::SERIALIZE,
+	                                      ctx);
+	return true;
+}
+
+template<> bool RsTypeSerializer::deserialize( const uint8_t data[],
+                                               uint32_t size, uint32_t& offset,
+                                               RsItem& s )
+{
+	RsGenericSerializer::SerializeContext ctx(
+	            const_cast<uint8_t*>(data), size,
+	            RsGenericSerializer::FORMAT_BINARY,
+	            RsGenericSerializer::SERIALIZATION_FLAG_NONE );
+	ctx.mOffset = offset;
+	const_cast<RsItem&>(s).serial_process(RsGenericSerializer::DESERIALIZE,
+	                                      ctx);
+	return true;
+}
+
+template<> void RsTypeSerializer::print_data( const std::string& n,
+                                              const RsItem& s )
+{
+	RsGenericSerializer::SerializeContext ctx(
+	            NULL, 0,
+	            RsGenericSerializer::FORMAT_BINARY,
+	            RsGenericSerializer::SERIALIZATION_FLAG_NONE );
+	const_cast<RsItem&>(s).serial_process(RsGenericSerializer::PRINT,
+	                                      ctx);
+}

libretroshare/src/serialiser/rstypeserializer.h

@@ -33,409 +33,521 @@
 
 #include "serialiser/rsserializer.h"
 
+/** @def RS_REGISTER_SERIAL_MEMBER(I)
+ * Use this macro to register the members of `YourItem` for serial processing
+ * inside `YourItem::serial_process(j, ctx)`
+ *
+ * Inspired by http://stackoverflow.com/a/39345864
+ */
+#define RS_REGISTER_SERIAL_MEMBER(I) \
+	do { RsTypeSerializer::serial_process(j, ctx, I, #I); } while(0)
 
-class RsTypeSerializer
+/** @def RS_REGISTER_SERIAL_MEMBER_TYPED(I, T)
+ * This macro usage is similar to @see RS_REGISTER_SERIAL_MEMBER(I) but it
+ * permit to force serialization/deserialization type, it is expecially useful
+ * with enum class members ot RsTlvItem derivative members, be very careful with
+ * the type you pass, as reinterpret_cast on a reference is used that is
+ * expecially permissive so you can shot your feet if not carefull enough.
+ *
+ * If you are using this with an RsItem derivative (so passing RsItem as T)
+ * consider to register your item type with @see RS_REGISTER_ITEM_TYPE(T) in
+ * association with @see RS_REGISTER_SERIAL_MEMBER(I) that rely on template
+ * function generation, as in this particular case
+ * RS_REGISTER_SERIAL_MEMBER_TYPED(I, T) would cause the serial code rely on
+ * C++ dynamic dispatching that may have a noticeable impact on runtime
+ * performances.
+ */
+#define RS_REGISTER_SERIAL_MEMBER_TYPED(I, T) do {\
+	RsTypeSerializer::serial_process<T>(j, ctx, reinterpret_cast<T&>(I), #I);\
+	} while(0)
+
+/** @def RS_REGISTER_ITEM_TYPE(T)
+ * Use this macro into `youritem.cc` only if you need to process members of
+ * subtypes of RsItem.
+ *
+ * The usage of this macro is strictly needed only in some cases, for example if
+ * you are registering for serialization a member of a container that contains
+ * items of subclasses of RsItem like * `std::map<uint64_t, RsChatMsgItem>`
+ *
+ * @code{.cpp}
+struct PrivateOugoingMapItem : RsChatItem
 {
-	public:
+	PrivateOugoingMapItem() : RsChatItem(RS_PKT_SUBTYPE_OUTGOING_MAP) {}
-    	// This type should be used to pass a parameter to drive the serialisation if needed.
 
-		struct TlvMemBlock_proxy: public std::pair<void*&   ,uint32_t&>
+	void serial_process( RsGenericSerializer::SerializeJob j,
-        {
+						 RsGenericSerializer::SerializeContext& ctx );
-            TlvMemBlock_proxy(void   *& p,uint32_t& s) : std::pair<void*&,uint32_t&>(p,s) {}
-            TlvMemBlock_proxy(uint8_t*& p,uint32_t& s) : std::pair<void*&,uint32_t&>(*(void**)&p,s) {}
-        };
 
-		//=================================================================================================//
+	std::map<uint64_t, RsChatMsgItem> store;
-		//                                            Generic types                                        //
-		//=================================================================================================//
-
-		template<typename T>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,T& member,const std::string& member_name)
-		{
-			switch(j)
-			{
-				case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += serial_size(member) ;
-																break ;
-
-				case RsGenericSerializer::DESERIALIZE:   ctx.mOk = ctx.mOk && deserialize(ctx.mData,ctx.mSize,ctx.mOffset,member) ;
-																break ;
-
-				case RsGenericSerializer::SERIALIZE:     ctx.mOk = ctx.mOk && serialize(ctx.mData,ctx.mSize,ctx.mOffset,member) ;
-																break ;
-
-				case RsGenericSerializer::PRINT:
-                							print_data(member_name,member);
-                break;
-				default:
-																ctx.mOk = false ;
-																throw std::runtime_error("Unknown serial job") ;
-			}
-		}
-
-		//=================================================================================================//
-		//                                     Generic types + type_id                                     //
-		//=================================================================================================//
-
-		template<typename T>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,uint16_t type_id,T& member,const std::string& member_name)
-		{
-			switch(j)
-			{
-				case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += serial_size(type_id,member) ;
-																break ;
-
-				case RsGenericSerializer::DESERIALIZE:   ctx.mOk = ctx.mOk && deserialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member) ;
-																break ;
-
-				case RsGenericSerializer::SERIALIZE:     ctx.mOk = ctx.mOk && serialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member) ;
-																break ;
-
-				case RsGenericSerializer::PRINT:
-                							print_data(member_name,type_id,member);
-                break;
-				default:
-																ctx.mOk = false ;
-																throw std::runtime_error("Unknown serial job") ;
-			}
-		}
-		//=================================================================================================//
-		//                                            std::map<T,U>                                        //
-		//=================================================================================================//
-
-		template<typename T,typename U>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,std::map<T,U>& v,const std::string& member_name)
-		{
-			switch(j)
-			{
-			case RsGenericSerializer::SIZE_ESTIMATE:
-			{
-				ctx.mOffset += 4 ;
-				for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
-                {
-                    serial_process(j,ctx,const_cast<T&>(it->first),"map::*it->first") ;
-                    serial_process(j,ctx,const_cast<U&>(it->second),"map::*it->second") ;
-				}
-			}
-				break ;
-
-			case RsGenericSerializer::DESERIALIZE:
-			{
-                uint32_t n=0 ;
-				serial_process(j,ctx,n,"temporary size");
-
-				for(uint32_t i=0;i<n;++i)
-                {
-                    T t ;
-                    U u ;
-
-                    serial_process(j,ctx,t,"map::*it->first") ;
-                    serial_process(j,ctx,u,"map::*it->second") ;
-
-                    v[t] = u ;
-                }
-			}
-				break ;
-
-			case RsGenericSerializer::SERIALIZE:
-			{
-				uint32_t n=v.size();
-				serial_process(j,ctx,n,"temporary size");
-
-				for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
-                {
-                    serial_process(j,ctx,const_cast<T&>(it->first),"map::*it->first") ;
-                    serial_process(j,ctx,const_cast<U&>(it->second),"map::*it->second") ;
-                }
-			}
-				break ;
-
-			case RsGenericSerializer::PRINT:
-			{
-                if(v.empty())
-					std::cerr << "  Empty map \"" << member_name << "\"" << std::endl;
-				else
-					std::cerr << "  std::map of " << v.size() << " elements: \"" << member_name << "\"" << std::endl;
-
-				for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
-                {
-                    std::cerr << "  " ;
-
-                    serial_process(j,ctx,const_cast<T&>(it->first),"map::*it->first") ;
-                    serial_process(j,ctx,const_cast<U&>(it->second),"map::*it->second") ;
-                }
-			}
-				break;
-			default:
-                break;
-			}
-		}
-
-		//=================================================================================================//
-		//                                            std::vector<T>                                       //
-		//=================================================================================================//
-
-		template<typename T>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,std::vector<T>& v,const std::string& member_name)
-		{
-			switch(j)
-			{
-			case RsGenericSerializer::SIZE_ESTIMATE:
-			{
-				ctx.mOffset += 4 ;
-				for(uint32_t i=0;i<v.size();++i)
-					serial_process(j,ctx,v[i],member_name) ;
-			}
-				break ;
-
-			case RsGenericSerializer::DESERIALIZE:
-			{  uint32_t n=0 ;
-				serial_process(j,ctx,n,"temporary size") ;
-
-				v.resize(n) ;
-				for(uint32_t i=0;i<v.size();++i)
-					serial_process(j,ctx,v[i],member_name) ;
-			}
-				break ;
-
-			case RsGenericSerializer::SERIALIZE:
-			{
-				uint32_t n=v.size();
-				serial_process(j,ctx,n,"temporary size") ;
-				for(uint32_t i=0;i<v.size();++i)
-					serial_process(j,ctx,v[i],member_name) ;
-			}
-				break ;
-
-			case RsGenericSerializer::PRINT:
-			{
-                if(v.empty())
-					std::cerr << "  Empty array"<< std::endl;
-				else
-					std::cerr << "  Array of " << v.size() << " elements:" << std::endl;
-				for(uint32_t i=0;i<v.size();++i)
-                {
-                    std::cerr << "  " ;
-					serial_process(j,ctx,v[i],member_name) ;
-                }
-			}
-				break;
-			default:
-                break;
-			}
-		}
-		//=================================================================================================//
-		//                                             std::set<T>                                         //
-		//=================================================================================================//
-
-		template<typename T>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,std::set<T>& v,const std::string& member_name)
-		{
-			switch(j)
-			{
-			case RsGenericSerializer::SIZE_ESTIMATE:
-			{
-				ctx.mOffset += 4 ;
-				for(typename std::set<T>::iterator it(v.begin());it!=v.end();++it)
-					serial_process(j,ctx,const_cast<T&>(*it) ,member_name) ; // the const cast here is a hack to avoid serial_process to instantiate serialise(const T&)
-			}
-				break ;
-
-			case RsGenericSerializer::DESERIALIZE:
-			{  uint32_t n=0 ;
-				serial_process(j,ctx,n,"temporary size") ;
-
-				for(uint32_t i=0;i<n;++i)
-                {
-                    T tmp;
-					serial_process<T>(j,ctx,tmp,member_name) ;
-                    v.insert(tmp);
-                }
-			}
-				break ;
-
-			case RsGenericSerializer::SERIALIZE:
-			{
-				uint32_t n=v.size();
-				serial_process(j,ctx,n,"temporary size") ;
-				for(typename std::set<T>::iterator it(v.begin());it!=v.end();++it)
-					serial_process(j,ctx,const_cast<T&>(*it) ,member_name) ;	// the const cast here is a hack to avoid serial_process to instantiate serialise(const T&)
-			}
-				break ;
-
-			case RsGenericSerializer::PRINT:
-			{
-                if(v.empty())
-					std::cerr << "  Empty set"<< std::endl;
-				else
-					std::cerr << "  Set of " << v.size() << " elements:" << std::endl;
-			}
-				break;
-			default:
-                break;
-			}
-		}
-
-
-		//=================================================================================================//
-		//                                            std::list<T>                                         //
-		//=================================================================================================//
-
-		template<typename T>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,std::list<T>& v,const std::string& member_name)
-		{
-			switch(j)
-			{
-			case RsGenericSerializer::SIZE_ESTIMATE:
-			{
-				ctx.mOffset += 4 ;
-				for(typename std::list<T>::iterator it(v.begin());it!=v.end();++it)
-					serial_process(j,ctx,*it ,member_name) ;
-			}
-				break ;
-
-			case RsGenericSerializer::DESERIALIZE:
-			{  uint32_t n=0 ;
-				serial_process(j,ctx,n,"temporary size") ;
-
-				for(uint32_t i=0;i<n;++i)
-                {
-                    T tmp;
-					serial_process<T>(j,ctx,tmp,member_name) ;
-                    v.push_back(tmp);
-                }
-			}
-				break ;
-
-			case RsGenericSerializer::SERIALIZE:
-			{
-				uint32_t n=v.size();
-				serial_process(j,ctx,n,"temporary size") ;
-				for(typename std::list<T>::iterator it(v.begin());it!=v.end();++it)
-					serial_process(j,ctx,*it ,member_name) ;
-			}
-				break ;
-
-			case RsGenericSerializer::PRINT:
-			{
-                if(v.empty())
-					std::cerr << "  Empty list"<< std::endl;
-				else
-					std::cerr << "  List of " << v.size() << " elements:" << std::endl;
-			}
-				break;
-			default:
-                break;
-			}
-		}
-
-		//=================================================================================================//
-		//                                         t_RsFlags32<> types                                     //
-		//=================================================================================================//
-
-		template<int N>
-		static void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,t_RsFlags32<N>& v,const std::string& /*member_name*/)
-		{
-			switch(j)
-			{
-			case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += 4 ;
-				break ;
-
-			case RsGenericSerializer::DESERIALIZE:
-			{
-                uint32_t n=0 ;
-                deserialize<uint32_t>(ctx.mData,ctx.mSize,ctx.mOffset,n) ;
-                v = t_RsFlags32<N>(n) ;
-			}
-				break ;
-
-			case RsGenericSerializer::SERIALIZE:
-			{
-                uint32_t n=v.toUInt32() ;
-                serialize<uint32_t>(ctx.mData,ctx.mSize,ctx.mOffset,n) ;
-			}
-				break ;
-
-			case RsGenericSerializer::PRINT:
-				std::cerr << "  Flags of type " << std::hex << N << " : " << v.toUInt32() << std::endl;
-                break ;
-            }
-
-		}
-
-	protected:
-		template<typename T> static bool     serialize  (uint8_t data[], uint32_t size, uint32_t &offset, const T& member);
-		template<typename T> static bool     deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, T& member);
-		template<typename T> static uint32_t serial_size(const T& /* member */);
-		template<typename T> static void     print_data(const std::string& name,const T& /* member */);
-
-		template<typename T> static bool     serialize  (uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id,const T& member);
-		template<typename T> static bool     deserialize(const uint8_t data[], uint32_t size, uint32_t &offset,uint16_t type_id, T& member);
-		template<typename T> static uint32_t serial_size(uint16_t type_id,const T& /* member */);
-		template<typename T> static void     print_data(const std::string& name,uint16_t type_id,const T& /* member */);
-
-		template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER> static bool     serialize  (uint8_t data[], uint32_t size, uint32_t &offset, const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member);
-		template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER> static bool     deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member);
-		template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER> static uint32_t serial_size(const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& /* member */);
-		template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER> static void     print_data(const std::string& name,const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& /* member */);
-
-		template<class TLV_CLASS,uint32_t TLV_TYPE> static bool     serialize  (uint8_t data[], uint32_t size, uint32_t &offset, const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member);
-		template<class TLV_CLASS,uint32_t TLV_TYPE> static bool     deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, t_RsTlvList<TLV_CLASS,TLV_TYPE>& member);
-		template<class TLV_CLASS,uint32_t TLV_TYPE> static uint32_t serial_size(const t_RsTlvList<TLV_CLASS,TLV_TYPE>& /* member */);
-		template<class TLV_CLASS,uint32_t TLV_TYPE> static void     print_data(const std::string& name,const t_RsTlvList<TLV_CLASS,TLV_TYPE>& /* member */);
 };
 
-//=================================================================================================//
+RS_REGISTER_ITEM_TYPE(RsChatMsgItem)
-//                                         t_RsGenericId<>                                         //
+
-//=================================================================================================//
+void PrivateOugoingMapItem::serial_process(
+		RsGenericSerializer::SerializeJob j,
+		RsGenericSerializer::SerializeContext& ctx )
+{
+	// store is of type
+	RS_REGISTER_SERIAL_MEMBER(store);
+}
+ * @endcode
+ *
+ * If you use this macro with a lot of different item types this can cause the
+ * generated binary grow in size, consider the usage of
+ * @see RS_REGISTER_SERIAL_MEMBER_TYPED(I, T) passing RsItem as type in that
+ * case.
+ */
+#define RS_REGISTER_ITEM_TYPE(T) template<> \
+	void RsTypeSerializer::serial_process<T>( \
+	        RsGenericSerializer::SerializeJob j,\
+	        RsGenericSerializer::SerializeContext& ctx, T& item,\
+	const std::string& /*name*/) { item.serial_process(j, ctx); }
+
+struct RsTypeSerializer
+{
+	/** This type should be used to pass a parameter to drive the serialisation
+	 * if needed */
+	struct TlvMemBlock_proxy: public std::pair<void*&,uint32_t&>
+	{
+		TlvMemBlock_proxy(void*& p, uint32_t& s) :
+		    std::pair<void*&,uint32_t&>(p,s) {}
+		TlvMemBlock_proxy(uint8_t*& p,uint32_t& s) :
+		    std::pair<void*&,uint32_t&>(*(void**)&p,s) {}
+	};
+
+	/// Generic types
+	template<typename T>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            T& member,const std::string& member_name)
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+			ctx.mOffset += serial_size(member);
+			break;
+		case RsGenericSerializer::DESERIALIZE:
+			ctx.mOk = ctx.mOk &&
+			        deserialize(ctx.mData,ctx.mSize,ctx.mOffset,member);
+			break;
+		case RsGenericSerializer::SERIALIZE:
+			ctx.mOk = ctx.mOk &&
+			        serialize(ctx.mData,ctx.mSize,ctx.mOffset,member);
+			break;
+		case RsGenericSerializer::PRINT:
+			print_data(member_name,member);
+			break;
+		default:
+			ctx.mOk = false;
+			throw std::runtime_error("Unknown serial job");
+		}
+	}
+
+	/// Generic types + type_id
+	template<typename T>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            uint16_t type_id, T& member,
+	                            const std::string& member_name )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+			ctx.mOffset += serial_size(type_id,member);
+			break;
+		case RsGenericSerializer::DESERIALIZE:
+			ctx.mOk = ctx.mOk &&
+			        deserialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member);
+			break;
+		case RsGenericSerializer::SERIALIZE:
+			ctx.mOk = ctx.mOk &&
+			        serialize(ctx.mData,ctx.mSize,ctx.mOffset,type_id,member);
+			break;
+		case RsGenericSerializer::PRINT:
+			print_data(member_name,type_id,member);
+			break;
+		default:
+			ctx.mOk = false;
+			throw std::runtime_error("Unknown serial job");
+		}
+	}
+
+	/// std::map<T,U>
+	template<typename T,typename U>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            std::map<T,U>& v,
+	                            const std::string& member_name )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+		{
+			ctx.mOffset += 4;
+			for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
+			{
+				serial_process( j, ctx, const_cast<T&>(it->first),
+				                "map::*it->first" );
+				serial_process( j,ctx,const_cast<U&>(it->second),
+				                "map::*it->second" );
+			}
+			break;
+		}
+		case RsGenericSerializer::DESERIALIZE:
+		{
+			uint32_t n=0;
+			serial_process(j,ctx,n,"temporary size");
+
+			for(uint32_t i=0; i<n; ++i)
+			{
+				T t; U u;
+				serial_process(j, ctx, t, "map::*it->first");
+				serial_process(j, ctx, u, "map::*it->second");
+				v[t] = u;
+			}
+			break;
+		}
+		case RsGenericSerializer::SERIALIZE:
+		{
+			uint32_t n=v.size();
+			serial_process(j,ctx,n,"temporary size");
+
+			for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
+			{
+				serial_process( j, ctx, const_cast<T&>(it->first),
+				                "map::*it->first" );
+				serial_process( j, ctx, const_cast<U&>(it->second),
+				                "map::*it->second" );
+			}
+			break;
+		}
+		case RsGenericSerializer::PRINT:
+		{
+			if(v.empty())
+				std::cerr << "  Empty map \"" << member_name << "\""
+				          << std::endl;
+			else
+				std::cerr << "  std::map of " << v.size() << " elements: \""
+				          << member_name << "\"" << std::endl;
+
+			for(typename std::map<T,U>::iterator it(v.begin());it!=v.end();++it)
+			{
+				std::cerr << "  ";
+
+				serial_process( j, ctx,const_cast<T&>(it->first),
+				                "map::*it->first" );
+				serial_process( j, ctx, const_cast<U&>(it->second),
+				                "map::*it->second" );
+			}
+			break;
+		}
+		default: break;
+		}
+	}
+
+	/// std::vector<T>
+	template<typename T>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            std::vector<T>& v,
+	                            const std::string& member_name )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+		{
+			ctx.mOffset += 4;
+			for(uint32_t i=0;i<v.size();++i)
+				serial_process(j,ctx,v[i],member_name);
+			break;
+		}
+		case RsGenericSerializer::DESERIALIZE:
+		{
+			uint32_t n=0;
+			serial_process(j,ctx,n,"temporary size");
+			v.resize(n);
+			for(uint32_t i=0;i<v.size();++i)
+				serial_process(j,ctx,v[i],member_name);
+			break;
+		}
+		case RsGenericSerializer::SERIALIZE:
+		{
+			uint32_t n=v.size();
+			serial_process(j,ctx,n,"temporary size");
+			for(uint32_t i=0; i<v.size(); ++i)
+				serial_process(j,ctx,v[i],member_name);
+			break;
+		}
+		case RsGenericSerializer::PRINT:
+		{
+			if(v.empty())
+				std::cerr << "  Empty array"<< std::endl;
+			else
+				std::cerr << "  Array of " << v.size() << " elements:"
+				          << std::endl;
+			for(uint32_t i=0;i<v.size();++i)
+			{
+				std::cerr << "  " ;
+				serial_process(j,ctx,v[i],member_name);
+			}
+			break;
+		}
+		default: break;
+		}
+	}
+
+	/// std::set<T>
+	template<typename T>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            std::set<T>& v, const std::string& member_name )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+		{
+			ctx.mOffset += 4;
+			for(typename std::set<T>::iterator it(v.begin());it!=v.end();++it)
+				// the const cast here is a hack to avoid serial_process to
+				// instantiate serialise(const T&)
+				serial_process(j,ctx,const_cast<T&>(*it) ,member_name);
+			break;
+		}
+		case RsGenericSerializer::DESERIALIZE:
+		{
+			uint32_t n=0;
+			serial_process(j,ctx,n,"temporary size");
+			for(uint32_t i=0; i<n; ++i)
+			{
+				T tmp;
+				serial_process<T>(j,ctx,tmp,member_name);
+				v.insert(tmp);
+			}
+			break;
+		}
+		case RsGenericSerializer::SERIALIZE:
+		{
+			uint32_t n=v.size();
+			serial_process(j,ctx,n,"temporary size");
+			for(typename std::set<T>::iterator it(v.begin());it!=v.end();++it)
+				// the const cast here is a hack to avoid serial_process to
+				// instantiate serialise(const T&)
+				serial_process(j,ctx,const_cast<T&>(*it) ,member_name);
+			break;
+		}
+		case RsGenericSerializer::PRINT:
+		{
+			if(v.empty()) std::cerr << "  Empty set"<< std::endl;
+			else std::cerr << "  Set of " << v.size() << " elements:"
+			               << std::endl;
+			break;
+		}
+		default: break;
+		}
+	}
+
+	/// std::list<T>
+	template<typename T>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            std::list<T>& v,
+	                            const std::string& member_name )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE:
+		{
+			ctx.mOffset += 4;
+			for(typename std::list<T>::iterator it(v.begin());it!=v.end();++it)
+				serial_process(j,ctx,*it ,member_name);
+			break;
+		}
+		case RsGenericSerializer::DESERIALIZE:
+		{
+			uint32_t n=0;
+			serial_process(j,ctx,n,"temporary size");
+			for(uint32_t i=0;i<n;++i)
+			{
+				T tmp;
+				serial_process<T>(j,ctx,tmp,member_name);
+				v.push_back(tmp);
+			}
+			break;
+		}
+		case RsGenericSerializer::SERIALIZE:
+		{
+			uint32_t n=v.size();
+			serial_process(j,ctx,n,"temporary size");
+			for(typename std::list<T>::iterator it(v.begin());it!=v.end();++it)
+				serial_process(j,ctx,*it ,member_name);
+			break;
+		}
+		case RsGenericSerializer::PRINT:
+		{
+			if(v.empty()) std::cerr << "  Empty list"<< std::endl;
+			else std::cerr << "  List of " << v.size() << " elements:"
+			               << std::endl;
+			break;
+		}
+		default: break;
+		}
+	}
+
+	/// t_RsFlags32<> types
+	template<int N>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            t_RsFlags32<N>& v,
+	                            const std::string& /*member_name*/)
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE: ctx.mOffset += 4; break;
+		case RsGenericSerializer::DESERIALIZE:
+		{
+			uint32_t n=0;
+			deserialize<uint32_t>(ctx.mData,ctx.mSize,ctx.mOffset,n);
+			v = t_RsFlags32<N>(n);
+			break;
+		}
+		case RsGenericSerializer::SERIALIZE:
+		{
+			uint32_t n=v.toUInt32();
+			serialize<uint32_t>(ctx.mData,ctx.mSize,ctx.mOffset,n);
+			break;
+		}
+		case RsGenericSerializer::PRINT:
+			std::cerr << "  Flags of type " << std::hex << N << " : "
+			          << v.toUInt32() << std::endl;
+			break;
+		}
+	}
+
+/** TODO
+ * Serialization format is inside context, but context is not passed to
+ * following functions, that need to know the format to do the job, actually
+ * RsGenericSerializer::FORMAT_BINARY is assumed in all of them!!
+ */
+
+protected:
+	template<typename T> static bool serialize(
+	        uint8_t data[], uint32_t size, uint32_t &offset, const T& member );
+
+	template<typename T> static bool deserialize(
+	        const uint8_t data[], uint32_t size, uint32_t &offset, T& member);
+
+	template<typename T> static uint32_t serial_size(const T& member);
+
+	template<typename T> static void print_data(
+	        const std::string& name, const T& member);
+
+
+	template<typename T> static bool serialize(
+	        uint8_t data[], uint32_t size, uint32_t &offset, uint16_t type_id,
+	        const T& member );
+	template<typename T> static bool deserialize(
+	        const uint8_t data[], uint32_t size, uint32_t &offset,
+	        uint16_t type_id, T& member );
+	template<typename T> static uint32_t serial_size(
+	        uint16_t type_id,const T& member );
+	template<typename T> static void print_data(
+	        const std::string& name,uint16_t type_id,const T& member );
+
+	template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
+	static bool serialize(
+	        uint8_t data[], uint32_t size, uint32_t &offset,
+	        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member );
+
+	template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
+	static bool deserialize(
+	        const uint8_t data[], uint32_t size, uint32_t &offset,
+	        t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member );
+
+	template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
+	static uint32_t serial_size(
+	        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member );
+
+	template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
+	static void print_data(
+	        const std::string& name,
+	        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member );
+
+
+	template<class TLV_CLASS,uint32_t TLV_TYPE>
+	static bool serialize(
+	        uint8_t data[], uint32_t size, uint32_t &offset,
+	        const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member );
+
+	template<class TLV_CLASS,uint32_t TLV_TYPE>
+	static bool deserialize(
+	        const uint8_t data[], uint32_t size, uint32_t &offset,
+	        t_RsTlvList<TLV_CLASS,TLV_TYPE>& member );
+
+	template<class TLV_CLASS,uint32_t TLV_TYPE>
+	static uint32_t serial_size(const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member);
+
+	template<class TLV_CLASS,uint32_t TLV_TYPE>
+	static void print_data(
+	        const std::string& name,
+	        const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member);
+};
+
+
+
+// t_RsGenericId<>
+template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
+bool RsTypeSerializer::serialize (
+        uint8_t data[], uint32_t size, uint32_t &offset,
+        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member )
+{ return (*const_cast<const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER> *>(&member)).serialise(data,size,offset); }
 
 template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
-bool     RsTypeSerializer::serialize  (uint8_t data[], uint32_t size, uint32_t &offset, const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member)
+bool RsTypeSerializer::deserialize(
-{
+        const uint8_t data[], uint32_t size, uint32_t &offset,
-    return (*const_cast<const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER> *>(&member)).serialise(data,size,offset) ;
+        t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member )
-}
+{ return member.deserialise(data,size,offset); }
 
 template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
-bool     RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member)
+uint32_t RsTypeSerializer::serial_size(
-{
+        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member )
-    return member.deserialise(data,size,offset) ;
+{ return member.serial_size(); }
-}
 
 template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
-uint32_t RsTypeSerializer::serial_size(const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member)
+void RsTypeSerializer::print_data(
+        const std::string& /*name*/,
+        const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member )
 {
-    return member.serial_size();
+	std::cerr << "  [RsGenericId<" << std::hex << UNIQUE_IDENTIFIER << ">] : "
+	          << member << std::endl;
 }
 
-template<uint32_t ID_SIZE_IN_BYTES,bool UPPER_CASE,uint32_t UNIQUE_IDENTIFIER>
-void     RsTypeSerializer::print_data(const std::string& /* name */,const t_RsGenericIdType<ID_SIZE_IN_BYTES,UPPER_CASE,UNIQUE_IDENTIFIER>& member)
-{
-    std::cerr << "  [RsGenericId<" << std::hex << UNIQUE_IDENTIFIER << ">] : " << member << std::endl;
-}
-
-//=================================================================================================//
-//                                         t_RsTlvList<>                                           //
-//=================================================================================================//
 
+// t_RsTlvList<>
 template<class TLV_CLASS,uint32_t TLV_TYPE>
-bool     RsTypeSerializer::serialize  (uint8_t data[], uint32_t size, uint32_t &offset, const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
+bool RsTypeSerializer::serialize(
+        uint8_t data[], uint32_t size, uint32_t &offset,
+        const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member )
 {
-    return (*const_cast<const t_RsTlvList<TLV_CLASS,TLV_TYPE> *>(&member)).SetTlv(data,size,&offset) ;
+	return (*const_cast<const t_RsTlvList<TLV_CLASS,TLV_TYPE> *>(&member)).SetTlv(data,size,&offset);
 }
 
 template<class TLV_CLASS,uint32_t TLV_TYPE>
-bool     RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
+bool RsTypeSerializer::deserialize(
+        const uint8_t data[], uint32_t size, uint32_t &offset,
+        t_RsTlvList<TLV_CLASS,TLV_TYPE>& member )
 {
-    return member.GetTlv(const_cast<uint8_t*>(data),size,&offset) ;
+	return member.GetTlv(const_cast<uint8_t*>(data),size,&offset);
 }
 
 template<class TLV_CLASS,uint32_t TLV_TYPE>
-uint32_t RsTypeSerializer::serial_size(const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
+uint32_t RsTypeSerializer::serial_size(
-{
+        const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
-    return member.TlvSize();
+{ return member.TlvSize(); }
-}
 
 template<class TLV_CLASS,uint32_t TLV_TYPE>
-void     RsTypeSerializer::print_data(const std::string& /* name */,const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
+void RsTypeSerializer::print_data(
+        const std::string& /*name*/,
+        const t_RsTlvList<TLV_CLASS,TLV_TYPE>& member)
 {
-    std::cerr << "  [t_RsTlvString<" << std::hex << TLV_TYPE << ">] : size=" << member.mList.size() << std::endl;
+	std::cerr << "  [t_RsTlvString<" << std::hex << TLV_TYPE << ">] : size="
+	          << member.mList.size() << std::endl;
 }

libretroshare/src/services/p3idservice.cc

@@ -1130,13 +1130,20 @@ bool p3IdService::validateData(const uint8_t *data,uint32_t data_size,const RsTl
     timeStampKey(signature.keyId,info);
     return true ;
 }
-bool p3IdService::encryptData(const uint8_t *decrypted_data,uint32_t decrypted_data_size,uint8_t *& encrypted_data,uint32_t& encrypted_data_size,const RsGxsId& encryption_key_id,bool force_load,uint32_t& error_status)
+
+bool p3IdService::encryptData( const uint8_t *decrypted_data,
+                               uint32_t decrypted_data_size,
+                               uint8_t *& encrypted_data,
+                               uint32_t& encrypted_data_size,
+                               const RsGxsId& encryption_key_id,
+                               uint32_t& error_status,
+                               bool force_load )
 {
     RsTlvPublicRSAKey encryption_key ;
 
     // get the key, and let the cache find it.
-    for(int i=0;i<(force_load?6:1);++i)
+	for(int i=0; i<(force_load?6:1);++i)
-        if(getKey(encryption_key_id,encryption_key))
+		if(getKey(encryption_key_id,encryption_key))
             break ;
         else
             usleep(500*1000) ; // sleep half a sec.
@@ -1160,18 +1167,110 @@ bool p3IdService::encryptData(const uint8_t *decrypted_data,uint32_t decrypted_d
     return true ;
 }
 
-bool p3IdService::decryptData(const uint8_t *encrypted_data,uint32_t encrypted_data_size,uint8_t *& decrypted_data,uint32_t& decrypted_size,const RsGxsId& key_id,uint32_t& error_status)
+bool p3IdService::encryptData( const uint8_t* decrypted_data,
+                               uint32_t decrypted_data_size,
+                               uint8_t*& encrypted_data,
+                               uint32_t& encrypted_data_size,
+                               const std::set<RsGxsId>& encrypt_ids,
+                               uint32_t& error_status, bool force_load )
+{
+	std::set<const RsGxsId*> keyNotYetFoundIds;
+
+	for( std::set<RsGxsId>::const_iterator it = encrypt_ids.begin();
+	     it != encrypt_ids.end(); ++it )
+	{
+		const RsGxsId& gId(*it);
+		if(gId.isNull())
+		{
+			std::cerr << "p3IdService::encryptData(...) (EE) got null GXS id"
+			          << std::endl;
+			return false;
+		}
+		else keyNotYetFoundIds.insert(&gId);
+	}
+
+	if(keyNotYetFoundIds.empty())
+	{
+		std::cerr << "p3IdService::encryptData(...) (EE) got empty GXS ids set"
+		          << std::endl;
+		print_stacktrace();
+		return false;
+	}
+
+	std::vector<RsTlvPublicRSAKey> encryption_keys;
+	int maxRounds = force_load ? 6 : 1;
+	for( int i=0; i < maxRounds; ++i )
+	{
+		for( std::set<const RsGxsId*>::iterator it = keyNotYetFoundIds.begin();
+		     it !=keyNotYetFoundIds.end(); ++it )
+		{
+			RsTlvPublicRSAKey encryption_key;
+			if(getKey(**it, encryption_key) && !encryption_key.keyId.isNull())
+			{
+				encryption_keys.push_back(encryption_key);
+				keyNotYetFoundIds.erase(it);
+			}
+		}
+
+		if(keyNotYetFoundIds.empty()) break;
+		else usleep(500*1000);
+	}
+
+	if(!keyNotYetFoundIds.empty())
+	{
+		std::cerr << "p3IdService::encryptData(...) (EE) Cannot get "
+		          << "encryption key for: ";
+		for( std::set<const RsGxsId*>::iterator it = keyNotYetFoundIds.begin();
+		     it !=keyNotYetFoundIds.end(); ++it )
+			std::cerr << **it << " ";
+		std::cerr << std::endl;
+		print_stacktrace();
+
+		error_status = RS_GIXS_ERROR_KEY_NOT_AVAILABLE;
+		return false;
+	}
+
+	if(!GxsSecurity::encrypt( encrypted_data, encrypted_data_size,
+	                          decrypted_data, decrypted_data_size,
+	                          encryption_keys ))
+	{
+		std::cerr << "p3IdService::encryptData(...) (EE) Encryption failed."
+		          << std::endl;
+		print_stacktrace();
+
+		error_status = RS_GIXS_ERROR_UNKNOWN;
+		return false ;
+	}
+
+	for( std::set<RsGxsId>::const_iterator it = encrypt_ids.begin();
+	     it != encrypt_ids.end(); ++it )
+	{
+		timeStampKey( *it,
+		              RsIdentityUsage(
+		                  serviceType(),
+		                  RsIdentityUsage::IDENTITY_GENERIC_ENCRYPTION ) );
+	}
+
+	error_status = RS_GIXS_ERROR_NO_ERROR;
+	return true;
+}
+
+bool p3IdService::decryptData( const uint8_t *encrypted_data,
+                               uint32_t encrypted_data_size,
+                               uint8_t *& decrypted_data,
+                               uint32_t& decrypted_size,
+                               const RsGxsId& key_id, uint32_t& error_status,
+                               bool force_load )
 {
     RsTlvPrivateRSAKey encryption_key ;
 
     // Get the key, and let the cache find it. It's our own key, so we should be able to find it, even if it takes
     // some seconds.
 
-    for(int i=0;i<4;++i)
+	int maxRounds = force_load ? 6 : 1;
-        if(getPrivateKey(key_id,encryption_key))
+	for(int i=0; i<maxRounds ;++i)
-            break ;
+		if(getPrivateKey(key_id,encryption_key)) break;
-        else
+		else usleep(500*1000) ; // sleep half a sec.
-            usleep(500*1000) ; // sleep half a sec.
 
     if(encryption_key.keyId.isNull())
     {
@@ -1185,13 +1284,106 @@ bool p3IdService::decryptData(const uint8_t *encrypted_data,uint32_t encrypted_d
         std::cerr << "  (EE) Decryption failed." << std::endl;
         error_status = RS_GIXS_ERROR_UNKNOWN ;
         return false ;
-    }
+	}
-    error_status = RS_GIXS_ERROR_NO_ERROR ;
+	error_status = RS_GIXS_ERROR_NO_ERROR;
-    timeStampKey(key_id,RsIdentityUsage(serviceType(),RsIdentityUsage::IDENTITY_GENERIC_DECRYPTION)) ;
+	timeStampKey( key_id,
+	              RsIdentityUsage(
+	                  serviceType(),
+	                  RsIdentityUsage::IDENTITY_GENERIC_DECRYPTION) );
 
     return true ;
 }
 
+bool p3IdService::decryptData( const uint8_t* encrypted_data,
+                               uint32_t encrypted_data_size,
+                               uint8_t*& decrypted_data,
+                               uint32_t& decrypted_data_size,
+                               const std::set<RsGxsId>& decrypt_ids,
+                               uint32_t& error_status,
+                               bool force_load )
+{
+	std::set<const RsGxsId*> keyNotYetFoundIds;
+
+	for( std::set<RsGxsId>::const_iterator it = decrypt_ids.begin();
+	     it != decrypt_ids.end(); ++it )
+	{
+		const RsGxsId& gId(*it);
+		if(gId.isNull())
+		{
+			std::cerr << "p3IdService::decryptData(...) (EE) got null GXS id"
+			          << std::endl;
+			print_stacktrace();
+			return false;
+		}
+		else keyNotYetFoundIds.insert(&gId);
+	}
+
+	if(keyNotYetFoundIds.empty())
+	{
+		std::cerr << "p3IdService::decryptData(...) (EE) got empty GXS ids set"
+		          << std::endl;
+		print_stacktrace();
+		return false;
+	}
+
+	std::vector<RsTlvPrivateRSAKey> decryption_keys;
+	int maxRounds = force_load ? 6 : 1;
+	for( int i=0; i < maxRounds; ++i )
+	{
+		for( std::set<const RsGxsId*>::iterator it = keyNotYetFoundIds.begin();
+		     it !=keyNotYetFoundIds.end(); ++it )
+		{
+			RsTlvPrivateRSAKey decryption_key;
+			if( getPrivateKey(**it, decryption_key)
+			        && !decryption_key.keyId.isNull() )
+			{
+				decryption_keys.push_back(decryption_key);
+				keyNotYetFoundIds.erase(it);
+			}
+		}
+
+		if(keyNotYetFoundIds.empty()) break;
+		else usleep(500*1000);
+	}
+
+	if(!keyNotYetFoundIds.empty())
+	{
+		std::cerr << "p3IdService::decryptData(...) (EE) Cannot get private key"
+		          << " for: ";
+		for( std::set<const RsGxsId*>::iterator it = keyNotYetFoundIds.begin();
+		     it !=keyNotYetFoundIds.end(); ++it )
+			std::cerr << **it << " ";
+		std::cerr << std::endl;
+		print_stacktrace();
+
+		error_status = RS_GIXS_ERROR_KEY_NOT_AVAILABLE;
+		return false;
+	}
+
+	if(!GxsSecurity::decrypt( decrypted_data, decrypted_data_size,
+	                          encrypted_data, encrypted_data_size,
+	                          decryption_keys ))
+	{
+		std::cerr << "p3IdService::decryptData(...) (EE) Decryption failed."
+		          << std::endl;
+		print_stacktrace();
+
+		error_status = RS_GIXS_ERROR_UNKNOWN;
+		return false ;
+	}
+
+	for( std::set<RsGxsId>::const_iterator it = decrypt_ids.begin();
+	     it != decrypt_ids.end(); ++it )
+	{
+		timeStampKey( *it,
+		              RsIdentityUsage(
+		                  serviceType(),
+		                  RsIdentityUsage::IDENTITY_GENERIC_DECRYPTION ) );
+	}
+
+	error_status = RS_GIXS_ERROR_NO_ERROR;
+	return true;
+}
 
 #ifdef TO_BE_REMOVED
 /********************************************************************************/

libretroshare/src/services/p3idservice.h

@@ -296,11 +296,48 @@ public:
 
 	virtual bool isOwnId(const RsGxsId& key_id) ;
 
-	virtual bool signData(const uint8_t *data,uint32_t data_size,const RsGxsId& signer_id,RsTlvKeySignature& signature,uint32_t& signing_error) ;
+	virtual bool signData( const uint8_t* data,
-	virtual bool validateData(const uint8_t *data, uint32_t data_size, const RsTlvKeySignature& signature, bool force_load, const RsIdentityUsage &info, uint32_t& signing_error) ;
+	                       uint32_t data_size,
+	                       const RsGxsId& signer_id,
+	                       RsTlvKeySignature& signature,
+	                       uint32_t& signing_error);
+
+	virtual bool validateData( const uint8_t *data, uint32_t data_size,
+	                           const RsTlvKeySignature& signature,
+	                           bool force_load, const RsIdentityUsage &info,
+	                           uint32_t& signing_error );
+
+	virtual bool encryptData( const uint8_t* decrypted_data,
+	                          uint32_t decrypted_data_size,
+	                          uint8_t*& encrypted_data,
+	                          uint32_t& encrypted_data_size,
+	                          const RsGxsId& encryption_key_id,
+	                          uint32_t& error_status,
+	                          bool force_load = true );
+
+	bool encryptData( const uint8_t* decrypted_data,
+	                  uint32_t decrypted_data_size,
+	                  uint8_t*& encrypted_data,
+	                  uint32_t& encrypted_data_size,
+	                  const std::set<RsGxsId>& encrypt_ids,
+	                  uint32_t& error_status, bool force_loa = true );
+
+	virtual bool decryptData( const uint8_t* encrypted_data,
+	                          uint32_t encrypted_data_size,
+	                          uint8_t*& decrypted_data,
+	                          uint32_t& decrypted_data_size,
+	                          const RsGxsId& decryption_key_id,
+	                          uint32_t& error_status,
+	                          bool force_load = true );
+
+	virtual bool decryptData(const uint8_t* encrypted_data,
+	                          uint32_t encrypted_data_size,
+	                          uint8_t*& decrypted_data,
+	                          uint32_t& decrypted_data_size,
+	                          const std::set<RsGxsId>& decrypt_ids,
+	                          uint32_t& error_status,
+	                          bool force_load = true );
 
-	virtual bool encryptData(const uint8_t *decrypted_data,uint32_t decrypted_data_size,uint8_t *& encrypted_data,uint32_t& encrypted_data_size,const RsGxsId& encryption_key_id,bool force_load,uint32_t& encryption_error) ;
-	virtual bool decryptData(const uint8_t *encrypted_data,uint32_t encrypted_data_size,uint8_t *& decrypted_data,uint32_t& decrypted_data_size,const RsGxsId& encryption_key_id,uint32_t& encryption_error) ;
 
 	virtual bool haveKey(const RsGxsId &id);
 	virtual bool havePrivateKey(const RsGxsId &id);
@@ -308,7 +345,9 @@ public:
 	virtual bool getKey(const RsGxsId &id, RsTlvPublicRSAKey &key);
 	virtual bool getPrivateKey(const RsGxsId &id, RsTlvPrivateRSAKey &key);
 
-    virtual bool requestKey(const RsGxsId &id, const std::list<RsPeerId> &peers, const RsIdentityUsage &use_info);
+	virtual bool requestKey( const RsGxsId &id,
+	                         const std::list<RsPeerId> &peers,
+	                         const RsIdentityUsage &use_info );
 	virtual bool requestPrivateKey(const RsGxsId &id);
 
 	virtual bool serialiseIdentityToMemory(const RsGxsId& id,std::string& radix_string);

libretroshare/src/services/p3msgservice.cc

@@ -66,6 +66,8 @@
 //#define DISABLE_DISTANT_MESSAGES 
 //#define DEBUG_DISTANT_MSG
 
+typedef unsigned int uint;
+
 using namespace Rs::Msgs;
 
 static struct RsLog::logInfo msgservicezoneInfo = {RsLog::Default, "msgservice"};
@@ -84,23 +86,29 @@ static const uint32_t RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME = 2*30*86400 ; // ke
  * (3) from storage...
  */
 
-p3MsgService::p3MsgService(p3ServiceControl *sc, p3IdService *id_serv)
+p3MsgService::p3MsgService( p3ServiceControl *sc, p3IdService *id_serv,
-	:p3Service(), p3Config(), mIdService(id_serv), mServiceCtrl(sc), mMsgMtx("p3MsgService"), mMsgUniqueId(0) 
+                            p3GxsTrans& gxsMS )
+    : p3Service(), p3Config(),
+      gxsOngoingMutex("p3MsgService Gxs Outgoing Mutex"), mIdService(id_serv),
+      mServiceCtrl(sc), mMsgMtx("p3MsgService"), mMsgUniqueId(0),
+      recentlyReceivedMutex("p3MsgService recently received hash mutex"),
+      mGxsTransServ(gxsMS)
 {
 	_serialiser = new RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_NONE);	// this serialiser is used for services. It's not the same than the one returned by setupSerialiser(). We need both!!
 	addSerialType(_serialiser);
 
-    mMsgUniqueId = 1 ;	// MsgIds are not transmitted, but only used locally as a storage index. As such, thay do not need to be different
+	/* MsgIds are not transmitted, but only used locally as a storage index.
-    			// at friends nodes.
+	 * As such, thay do not need to be different at friends nodes. */
-    
+	mMsgUniqueId = 1;
-    mShouldEnableDistantMessaging = true ;
-    mDistantMessagingEnabled      = false ;
-    mDistantMessagePermissions      = RS_DISTANT_MESSAGING_CONTACT_PERMISSION_FLAG_FILTER_NONE ;
 
-    /* Initialize standard tag types */
+	mShouldEnableDistantMessaging = true;
-    if(sc)
+	mDistantMessagingEnabled = false;
-	    initStandardTagTypes();
+	mDistantMessagePermissions = RS_DISTANT_MESSAGING_CONTACT_PERMISSION_FLAG_FILTER_NONE;
 
+	if(sc) initStandardTagTypes(); // Initialize standard tag types
+
+	mGxsTransServ.registerGxsTransClient( GxsTransSubServices::P3_MSG_SERVICE,
+	                                      this );
 }
 
 const std::string MSG_APP_NAME = "msg";
@@ -142,11 +150,11 @@ int	p3MsgService::tick()
 
 	if(now > last_management_time + 5)
 	{
-		manageDistantPeers() ;
+		manageDistantPeers();
-		checkOutgoingMessages(); 
+		checkOutgoingMessages();
-        	cleanListOfReceivedMessageHashes();
+		cleanListOfReceivedMessageHashes();
 
-		last_management_time = now ;
+		last_management_time = now;
 	}
 
 	return 0;
@@ -154,21 +162,21 @@ int	p3MsgService::tick()
 
 void p3MsgService::cleanListOfReceivedMessageHashes()
 {
-    RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
+	RS_STACK_MUTEX(recentlyReceivedMutex);
 
-    time_t now = time(NULL) ;
+	time_t now = time(NULL);
-    
+
-    for(std::map<Sha1CheckSum,uint32_t>::iterator it(mRecentlyReceivedDistantMessageHashes.begin());it!=mRecentlyReceivedDistantMessageHashes.end();)
+	for( auto it = mRecentlyReceivedMessageHashes.begin();
-	    if(now > RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME + it->second)
+	     it != mRecentlyReceivedMessageHashes.end(); )
-	    {
+		if( now > RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME + it->second )
-		    std::cerr << "p3MsgService(): cleanListOfReceivedMessageHashes(). Removing old hash " << it->first << ", aged " << now - it->second << " secs ago" << std::endl;
+		{
-		    std::map<Sha1CheckSum,uint32_t>::iterator  tmp(it) ;
+			std::cerr << "p3MsgService(): cleanListOfReceivedMessageHashes(). "
-		    ++tmp ;
+			          << "Removing old hash " << it->first << ", aged "
-		    mRecentlyReceivedDistantMessageHashes.erase(it) ;
+			          << now - it->second << " secs ago" << std::endl;
-		    it=tmp ;
+
-	    }
+			it = mRecentlyReceivedMessageHashes.erase(it);
-	    else
+		}
-		    ++it ;
+		else ++it;
 }
 
 int	p3MsgService::status()
@@ -349,111 +357,112 @@ void p3MsgService::checkSizeAndSendMessage(RsMsgItem *msg)
     sendItem(msg) ;
 }
 
-int     p3MsgService::checkOutgoingMessages()
+int p3MsgService::checkOutgoingMessages()
 {
-    /* iterate through the outgoing queue 
+	bool changed = false;
-	 *
+	std::list<RsMsgItem*> output_queue;
-	 * if online, send
-	 */
 
-    bool changed = false ;
+	{
-    std::list<RsMsgItem*> output_queue ;
+		RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
 
-    {
+		const RsPeerId& ownId = mServiceCtrl->getOwnId();
-	    RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
-        
-	    const RsPeerId& ownId = mServiceCtrl->getOwnId();
 
-	    std::list<uint32_t>::iterator it;
+		std::list<uint32_t>::iterator it;
-	    std::list<uint32_t> toErase;
+		std::list<uint32_t> toErase;
 
-	    std::map<uint32_t, RsMsgItem *>::iterator mit;
+		std::map<uint32_t, RsMsgItem *>::iterator mit;
-	    for(mit = msgOutgoing.begin(); mit != msgOutgoing.end(); ++mit)
+		for( mit = msgOutgoing.begin(); mit != msgOutgoing.end(); ++mit )
-	    {
+		{
-		    if (mit->second->msgFlags & RS_MSG_FLAGS_TRASH)
+			if (mit->second->msgFlags & RS_MSG_FLAGS_TRASH) continue;
-			    continue;
 
-		    /* find the certificate */
+			/* find the certificate */
-		    RsPeerId pid = mit->second->PeerId();
+			RsPeerId pid = mit->second->PeerId();
-		    bool should_send = false ;
+			bool should_send = false;
 
-		    if( pid == ownId) 
+			if( pid == ownId) should_send = true;
-			    should_send = true ;
 
-		    if( mServiceCtrl->isPeerConnected(getServiceInfo().mServiceType, pid) ) /* FEEDBACK Msg to Ourselves */
+			// FEEDBACK Msg to Ourselves
-			    should_send = true ;
+			if( mServiceCtrl->isPeerConnected(getServiceInfo().mServiceType,
+			                                  pid) )
+				should_send = true;
 
-		    if((mit->second->msgFlags & RS_MSG_FLAGS_DISTANT) && !(mit->second->msgFlags & RS_MSG_FLAGS_ROUTED))
+			if( (mit->second->msgFlags & RS_MSG_FLAGS_DISTANT) &&
-				    should_send = true ;
+			        !(mit->second->msgFlags & RS_MSG_FLAGS_ROUTED))
+				should_send = true;
 
-		    if(should_send)
+			if(should_send)
-		    {
+			{
-			    /* send msg */
+				/* send msg */
-			    pqioutput(PQL_DEBUG_BASIC, msgservicezone, 
+				pqioutput( PQL_DEBUG_BASIC, msgservicezone,
-			              "p3MsgService::checkOutGoingMessages() Sending out message");
+				           "p3MsgService::checkOutGoingMessages() Sending out message");
-			    /* remove the pending flag */
+				/* remove the pending flag */
 
-			    output_queue.push_back(mit->second) ;
+				output_queue.push_back(mit->second) ;
 
-			    // When the message is a distant msg, dont remove it yet from the list. Only mark it as being sent, so that we don't send it again.
+				/* When the message is a distant msg, dont remove it yet from
-			    //
+				 * the list. Only mark it as being sent, so that we don't send
-			    if(!(mit->second->msgFlags & RS_MSG_FLAGS_DISTANT))
+				 * it again. */
-			    {
+				if(!(mit->second->msgFlags & RS_MSG_FLAGS_DISTANT))
-				    (mit->second)->msgFlags &= ~RS_MSG_FLAGS_PENDING;
+				{
-				    toErase.push_back(mit->first);
+					(mit->second)->msgFlags &= ~RS_MSG_FLAGS_PENDING;
-				    changed = true ;
+					toErase.push_back(mit->first);
-			    }
+					changed = true;
-			    else
+				}
-			    {
+				else
+				{
 #ifdef DEBUG_DISTANT_MSG
-				    std::cerr << "Message id " << mit->first << " is distant: kept in outgoing, and marked as ROUTED" << std::endl;
+					std::cerr << "Message id " << mit->first << " is distant: "
+					          << "kept in outgoing, and marked as ROUTED"
+					          << std::endl;
 #endif
-				    mit->second->msgFlags |= RS_MSG_FLAGS_ROUTED ;
+					mit->second->msgFlags |= RS_MSG_FLAGS_ROUTED;
-			    }
+				}
-		    }
+			}
-		    else
+			else
-		    {
+			{
-			    pqioutput(PQL_DEBUG_BASIC, msgservicezone, 
+				pqioutput( PQL_DEBUG_BASIC, msgservicezone,
-			              "p3MsgService::checkOutGoingMessages() Delaying until available...");
+				           "p3MsgService::checkOutGoingMessages() Delaying until available...");
-		    }
+			}
-	    }
+		}
 
-	    /* clean up */
+		/* clean up */
-	    for(it = toErase.begin(); it != toErase.end(); ++it)
+		for(it = toErase.begin(); it != toErase.end(); ++it)
-	    {
+		{
-		    mit = msgOutgoing.find(*it);
+			mit = msgOutgoing.find(*it);
-		    if (mit != msgOutgoing.end())
+			if ( mit != msgOutgoing.end() ) msgOutgoing.erase(mit);
-		    {
-			    msgOutgoing.erase(mit);
-		    }
 
-		    std::map<uint32_t, RsMsgSrcId*>::iterator srcIt = mSrcIds.find(*it);
+			std::map<uint32_t, RsMsgSrcId*>::iterator srcIt = mSrcIds.find(*it);
-		    if (srcIt != mSrcIds.end()) {
+			if (srcIt != mSrcIds.end())
-			    delete (srcIt->second);
+			{
-			    mSrcIds.erase(srcIt);
+				delete (srcIt->second);
-		    }
+				mSrcIds.erase(srcIt);
-	    }
+			}
+		}
 
-	    if (toErase.size() > 0)
+		if (toErase.size() > 0) IndicateConfigChanged();
-	    {
+	}
-		    IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
-	    }
-    }
 
-    for(std::list<RsMsgItem*>::const_iterator it(output_queue.begin());it!=output_queue.end();++it)
+	for( std::list<RsMsgItem*>::const_iterator it(output_queue.begin());
-	    if((*it)->msgFlags & RS_MSG_FLAGS_DISTANT)		// don't split distant messages. The global router takes care of it.
+	     it != output_queue.end(); ++it )
-		    sendDistantMsgItem(*it) ;
+		if( (*it)->msgFlags & RS_MSG_FLAGS_DISTANT ) // don't split distant messages. The global router takes care of it.
-	    else
+			sendDistantMsgItem(*it);
-		    checkSizeAndSendMessage(*it) ;
+		else
+			checkSizeAndSendMessage(*it);
 
-    if(changed)
+	if(changed)
-	    RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD);
+		RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD);
 
-    return 0;
+	return 0;
 }
 
-bool    p3MsgService::saveList(bool& cleanup, std::list<RsItem*>& itemList)
+bool p3MsgService::saveList(bool& cleanup, std::list<RsItem*>& itemList)
 {
+	RsMsgGRouterMap* gxsmailmap = new RsMsgGRouterMap;
+	{
+		RS_STACK_MUTEX(gxsOngoingMutex);
+		gxsmailmap->ongoing_msgs = gxsOngoingMessages;
+	}
+	itemList.push_front(gxsmailmap);
 
 	std::map<uint32_t, RsMsgItem *>::iterator mit;
 	std::map<uint32_t, RsMsgTagType* >::iterator mit2;
@@ -461,9 +470,7 @@ bool    p3MsgService::saveList(bool& cleanup, std::list<RsItem*>& itemList)
 	std::map<uint32_t, RsMsgSrcId* >::iterator lit;
 	std::map<uint32_t, RsMsgParentId* >::iterator mit4;
 
-    MsgTagType stdTags;
+	cleanup = true;
-
-    cleanup = true;
 
 	mMsgMtx.lock();
 
@@ -490,10 +497,13 @@ bool    p3MsgService::saveList(bool& cleanup, std::list<RsItem*>& itemList)
     grmap->ongoing_msgs = _ongoing_messages ;
 
     itemList.push_back(grmap) ;
-    
+
-    RsMsgDistantMessagesHashMap *ghm = new RsMsgDistantMessagesHashMap ;
+	RsMsgDistantMessagesHashMap *ghm = new RsMsgDistantMessagesHashMap;
-    ghm->hash_map = mRecentlyReceivedDistantMessageHashes ;
+	{
-    itemList.push_back(ghm) ;
+		RS_STACK_MUTEX(recentlyReceivedMutex);
+		ghm->hash_map = mRecentlyReceivedMessageHashes;
+	}
+	itemList.push_back(ghm);
 
 	RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet ;
 	RsTlvKeyValue kv;
@@ -505,7 +515,7 @@ bool    p3MsgService::saveList(bool& cleanup, std::list<RsItem*>& itemList)
         kv.value = RsUtil::NumberToString(mDistantMessagePermissions) ;
 	vitem->tlvkvs.pairs.push_back(kv) ;
 
-	itemList.push_back(vitem) ;
+	itemList.push_back(vitem);
 
 	return true;
 }
@@ -567,8 +577,19 @@ void p3MsgService::initStandardTagTypes()
 	}
 }
 
-bool    p3MsgService::loadList(std::list<RsItem*>& load)
+bool p3MsgService::loadList(std::list<RsItem*>& load)
 {
+	auto gxsmIt = load.begin();
+	RsMsgGRouterMap* gxsmailmap = dynamic_cast<RsMsgGRouterMap*>(*gxsmIt);
+	if(gxsmailmap)
+	{
+		{
+			RS_STACK_MUTEX(gxsOngoingMutex);
+			gxsOngoingMessages = gxsmailmap->ongoing_msgs;
+		}
+		delete *gxsmIt; load.erase(gxsmIt);
+	}
+
     RsMsgItem *mitem;
     RsMsgTagType* mtt;
     RsMsgTags* mti;
@@ -578,7 +599,7 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
     RsMsgDistantMessagesHashMap *ghm;
 
     std::list<RsMsgItem*> items;
-    std::list<RsItem*>::iterator it;
+	std::list<RsItem*>::iterator it;
     std::map<uint32_t, RsMsgTagType*>::iterator tagIt;
     std::map<uint32_t, RsPeerId> srcIdMsgMap;
     std::map<uint32_t, RsPeerId>::iterator srcIt;
@@ -586,9 +607,9 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
     uint32_t max_msg_id = 0 ;
     
     // load items and calculate next unique msgId
-    for(it = load.begin(); it != load.end(); ++it)
+	for(it = load.begin(); it != load.end(); ++it)
     {
-	    if (NULL != (mitem = dynamic_cast<RsMsgItem *>(*it)))
+		if (NULL != (mitem = dynamic_cast<RsMsgItem *>(*it)))
 	    {
 		    /* STORE MsgID */
 		    if (mitem->msgId > max_msg_id) 
@@ -596,18 +617,21 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
 		    
 		    items.push_back(mitem);
 	    }
-	    else if (NULL != (grm = dynamic_cast<RsMsgGRouterMap *>(*it)))
+		else if (NULL != (grm = dynamic_cast<RsMsgGRouterMap *>(*it)))
 	    {
-		    // merge.
+			typedef std::map<GRouterMsgPropagationId,uint32_t> tT;
-		    for(std::map<GRouterMsgPropagationId,uint32_t>::const_iterator bit(grm->ongoing_msgs.begin());bit!=grm->ongoing_msgs.end();++bit)
+			for( tT::const_iterator bit = grm->ongoing_msgs.begin();
-			    _ongoing_messages.insert(*bit) ;
+			     bit != grm->ongoing_msgs.end(); ++bit )
-
+				_ongoing_messages.insert(*bit);
-            delete *it ;
+			delete *it;
-            continue ;
+			continue;
-	    }
+		}
-        else if(NULL != (ghm = dynamic_cast<RsMsgDistantMessagesHashMap*>(*it)))
+		else if(NULL != (ghm = dynamic_cast<RsMsgDistantMessagesHashMap*>(*it)))
-        {
+		{
-            mRecentlyReceivedDistantMessageHashes = ghm->hash_map ;
+			{
+				RS_STACK_MUTEX(recentlyReceivedMutex);
+				mRecentlyReceivedMessageHashes = ghm->hash_map;
+			}
 #ifdef DEBUG_DISTANT_MSG
             std::cerr << "  loaded recently received message map: " << std::endl;
             
@@ -632,23 +656,23 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
 		    }
 
 	    }
-	    else if(NULL != (mti = dynamic_cast<RsMsgTags *>(*it)))
+		else if(NULL != (mti = dynamic_cast<RsMsgTags *>(*it)))
 	    {
 		    mMsgTags.insert(std::pair<uint32_t, RsMsgTags* >(mti->msgId, mti));
 	    }
-	    else if(NULL != (msi = dynamic_cast<RsMsgSrcId *>(*it)))
+		else if(NULL != (msi = dynamic_cast<RsMsgSrcId *>(*it)))
 	    {
 		    srcIdMsgMap.insert(std::pair<uint32_t, RsPeerId>(msi->msgId, msi->srcId));
 		    mSrcIds.insert(std::pair<uint32_t, RsMsgSrcId*>(msi->msgId, msi)); // does not need to be kept
 	    }
-	    else if(NULL != (msp = dynamic_cast<RsMsgParentId *>(*it)))
+		else if(NULL != (msp = dynamic_cast<RsMsgParentId *>(*it)))
 	    {
 		    mParentId.insert(std::pair<uint32_t, RsMsgParentId*>(msp->msgId, msp));
 	    }
 
 	    RsConfigKeyValueSet *vitem = NULL ;
 
-	    if(NULL != (vitem = dynamic_cast<RsConfigKeyValueSet*>(*it)))
+		if(NULL != (vitem = dynamic_cast<RsConfigKeyValueSet*>(*it)))
 	    {
 		    for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) 
 		    {
@@ -679,7 +703,7 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
 			    }
 		    }
 
-		    delete *it ;
+			delete *it ;
 		    continue ;
 	    }
     }
@@ -697,7 +721,7 @@ bool    p3MsgService::loadList(std::list<RsItem*>& load)
 		    mitem->msgId = getNewUniqueMsgId();
 	    }
 
-	    RsStackMutex stack(mMsgMtx); /********** STACK LOCKED MTX ******/
+		RS_STACK_MUTEX(mMsgMtx);
 
 	    srcIt = srcIdMsgMap.find(mitem->msgId);
 	    if(srcIt != srcIdMsgMap.end()) {
@@ -1111,16 +1135,17 @@ uint32_t     p3MsgService::sendMessage(RsMsgItem *item)	// no from field because
 
     return item->msgId;
 }
-uint32_t     p3MsgService::sendDistantMessage(RsMsgItem *item,const RsGxsId& from)
-{
-	if(!item)
-		return 0 ;
 
-	item->msgId     = getNewUniqueMsgId(); /* grabs Mtx as well */
+uint32_t p3MsgService::sendDistantMessage(RsMsgItem *item, const RsGxsId& from)
-	item->msgFlags |= (RS_MSG_FLAGS_DISTANT | RS_MSG_FLAGS_OUTGOING | RS_MSG_FLAGS_PENDING); /* add pending flag */
+{
+	if(!item) return 0;
+
+	item->msgId  = getNewUniqueMsgId(); /* grabs Mtx as well */
+	item->msgFlags |= ( RS_MSG_FLAGS_DISTANT | RS_MSG_FLAGS_OUTGOING |
+	                    RS_MSG_FLAGS_PENDING ); /* add pending flag */
 
 	{
-		RS_STACK_MUTEX(mMsgMtx) ;
+		RS_STACK_MUTEX(mMsgMtx);
 
 		/* STORE MsgID */
 		msgOutgoing[item->msgId] = item;
@@ -1132,17 +1157,16 @@ uint32_t     p3MsgService::sendDistantMessage(RsMsgItem *item,const RsGxsId& fro
 
 			RsMsgSrcId* msi = new RsMsgSrcId();
 			msi->msgId = item->msgId;
-			msi->srcId = RsPeerId(from) ;
+			msi->srcId = RsPeerId(from);
 			mSrcIds.insert(std::pair<uint32_t, RsMsgSrcId*>(msi->msgId, msi));
 		}
 	}
 
 	IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
 
-	RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST, NOTIFY_TYPE_ADD);
+	RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST,
-
+	                                      NOTIFY_TYPE_ADD );
 	return item->msgId;
-
 }
 
 bool 	p3MsgService::MessageSend(MessageInfo &info)
@@ -1857,73 +1881,91 @@ void p3MsgService::manageDistantPeers()
     }
 }
 
-void p3MsgService::notifyDataStatus(const GRouterMsgPropagationId& id, const RsGxsId &signer_id, uint32_t data_status)
+void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id,
+                                     const RsGxsId &signer_id,
+                                     uint32_t data_status )
 {
-    if(data_status == GROUTER_CLIENT_SERVICE_DATA_STATUS_FAILED)
+	if(data_status == GROUTER_CLIENT_SERVICE_DATA_STATUS_FAILED)
-    {
+	{
-	    RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
+		RS_STACK_MUTEX(mMsgMtx);
 
-	    std::cerr << "(WW) p3MsgService::notifyDataStatus: Global router tells us that item ID " << id << " could not be delivered on time." ;
+		std::cerr << "(WW) p3MsgService::notifyDataStatus: Global router tells "
-	    std::map<GRouterMsgPropagationId,uint32_t>::iterator it = _ongoing_messages.find(id) ;
+		          << "us that item ID " << id
-            
+		          << " could not be delivered on time.";
-	    if(it == _ongoing_messages.end())
-	    {
-		    std::cerr << "  (EE) cannot find pending message to acknowledge. Weird. grouter id = " << id << std::endl;
-		    return ;
-	    }
-	    uint32_t msg_id = it->second ;
-	    std::cerr << "  message id = " << msg_id << std::endl;
-	    mDistantOutgoingMsgSigners[msg_id] = signer_id ;	// this is needed because it's not saved in config, but we should probably include it in _ongoing_messages
 
-	    std::map<uint32_t,RsMsgItem*>::iterator mit = msgOutgoing.find(msg_id) ;
+		auto it = _ongoing_messages.find(id);
+		if(it == _ongoing_messages.end())
+		{
+			std::cerr << " (EE) cannot find pending message to acknowledge. "
+			          << "Weird. grouter id = " << id << std::endl;
+			return;
+		}
 
-	    if(mit == msgOutgoing.end())
+		uint32_t msg_id = it->second;
-	    {
+		std::cerr << " message id = " << msg_id << std::endl;
-		    std::cerr << "  (EE) message has been notified as not delivered, but it not on outgoing list. Something's wrong!!" << std::endl;
-		    return ;
-	    }
-        std::cerr << "  reseting the ROUTED flag so that the message is requested again" << std::endl;
 
-	    mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED ;	// clear the routed flag so that the message is requested again
+		/* this is needed because it's not saved in config, but we should
-	    return ;
+		 * probably include it in _ongoing_messages */
-    }
+		mDistantOutgoingMsgSigners[msg_id] = signer_id;
-    
+
-    if(data_status == GROUTER_CLIENT_SERVICE_DATA_STATUS_RECEIVED)
+		std::map<uint32_t,RsMsgItem*>::iterator mit = msgOutgoing.find(msg_id);
-    {
+		if(mit == msgOutgoing.end())
-	    RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
+		{
+			std::cerr << "  (II) message has been notified as not delivered, "
+			          << "but it's not in outgoing list. Probably it has been "
+			          << "delivered successfully by other means."
+			          << std::endl;
+		}
+		else
+		{
+			std::cerr << "  reseting the ROUTED flag so that the message is "
+			          << "requested again" << std::endl;
+
+			// clear the routed flag so that the message is requested again
+			mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED;
+		}
+		return;
+	}
+
+	if(data_status == GROUTER_CLIENT_SERVICE_DATA_STATUS_RECEIVED)
+	{
+		RS_STACK_MUTEX(mMsgMtx);
 #ifdef DEBUG_DISTANT_MSG
-	    std::cerr << "p3MsgService::acknowledgeDataReceived(): acknowledging data received for msg propagation id  " << id << std::endl;
+		std::cerr << "p3MsgService::acknowledgeDataReceived(): acknowledging data received for msg propagation id  " << id << std::endl;
 #endif
-	    std::map<GRouterMsgPropagationId,uint32_t>::iterator it = _ongoing_messages.find(id) ;
+		auto it = _ongoing_messages.find(id);
+		if(it == _ongoing_messages.end())
+		{
+			std::cerr << "  (EE) cannot find pending message to acknowledge. "
+			          << "Weird. grouter id = " << id << std::endl;
+			return;
+		}
 
-	    if(it == _ongoing_messages.end())
+		uint32_t msg_id = it->second;
-	    {
-		    std::cerr << "  (EE) cannot find pending message to acknowledge. Weird. grouter id = " << id << std::endl;
-		    return ;
-	    }
 
-	    uint32_t msg_id = it->second ;
+		// we should now remove the item from the msgOutgoing list.
+		std::map<uint32_t,RsMsgItem*>::iterator it2 = msgOutgoing.find(msg_id);
+		if(it2 == msgOutgoing.end())
+		{
+			std::cerr << "(II) message has been notified as delivered, but it's"
+			          << " not in outgoing list. Probably it has been delivered"
+			          << " successfully by other means." << std::endl;
+			return;
+		}
 
-	    // we should now remove the item from the msgOutgoing list.
+		delete it2->second;
+		msgOutgoing.erase(it2);
 
-	    std::map<uint32_t,RsMsgItem*>::iterator it2 = msgOutgoing.find(msg_id) ;
+		RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST,
+		                                      NOTIFY_TYPE_ADD );
+		IndicateConfigChanged();
 
-	    if(it2 == msgOutgoing.end())
+		return;
-	    {
+	}
-		    std::cerr << "(EE) message has been ACKed, but is not in outgoing list. Something's wrong!!" << std::endl;
+	std::cerr << "p3MsgService: unhandled data status info from global router"
-		    return ;
+	          << " for msg ID " << id << ": this is a bug." << std::endl;
-	    }
-
-	    delete it2->second ;
-	    msgOutgoing.erase(it2) ;
-
-	    RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD);
-	    IndicateConfigChanged() ;
-        
-        	return ;
-    }
-    std::cerr << "p3MsgService: unhandled data status info from global router for msg ID " << id << ": this is a bug." << std::endl;
 }
+
 bool p3MsgService::acceptDataFromPeer(const RsGxsId& to_gxs_id)
 {
     if(mDistantMessagePermissions & RS_DISTANT_MESSAGING_CONTACT_PERMISSION_FLAG_FILTER_NON_CONTACTS)
@@ -1949,81 +1991,256 @@ uint32_t p3MsgService::getDistantMessagingPermissionFlags()
 {
     return mDistantMessagePermissions ;
 }
-            
+
-void p3MsgService::receiveGRouterData(const RsGxsId &destination_key, const RsGxsId &signing_key, GRouterServiceId &/*client_id*/, uint8_t *data, uint32_t data_size)
+bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId,
+                                const RsGxsId& recipientId,
+                                const uint8_t* data, uint32_t dataSize )
 {
-    std::cerr << "p3MsgService::receiveGRouterData(): received message item of size " << data_size << ", for key " << destination_key << std::endl;
+	std::cout << __PRETTY_FUNCTION__ << " " << authorId << ", " << recipientId
+	          << ",, " << dataSize << std::endl;
 
-    // first make sure that we havn't already received the data. Since we allow to re-send messages, it's necessary to check.
+	Sha1CheckSum hash = RsDirUtil::sha1sum(data, dataSize);
-    
+
-    Sha1CheckSum hash = RsDirUtil::sha1sum(data,data_size) ;
+	{
-    
+		RS_STACK_MUTEX(recentlyReceivedMutex);
-    if(mRecentlyReceivedDistantMessageHashes.find(hash) != mRecentlyReceivedDistantMessageHashes.end())
+		if( mRecentlyReceivedMessageHashes.find(hash) !=
-    {
+		        mRecentlyReceivedMessageHashes.end() )
-        std::cerr << "(WW) receiving distant message of hash " << hash << " more than once. This is not a bug, unless it happens very often." << std::endl;
+		{
+			std::cerr << __PRETTY_FUNCTION__ << " (II) receiving "
+			          << "message of hash " << hash << " more than once. "
+			          << "Probably it has arrived  before by other means."
+			          << std::endl;
+			return true;
+		}
+		mRecentlyReceivedMessageHashes[hash] = time(NULL);
+	}
+
+	IndicateConfigChanged();
+
+	RsItem *item = _serialiser->deserialise(const_cast<uint8_t*>(data), &dataSize);
+	RsMsgItem *msg_item = dynamic_cast<RsMsgItem*>(item);
+
+	if(msg_item)
+	{
+		std::cerr << __PRETTY_FUNCTION__ << " Encrypted item correctly "
+		          << "deserialised. Passing on to incoming list."
+		          << std::endl;
+
+		msg_item->msgFlags |= RS_MSG_FLAGS_DISTANT;
+		/* we expect complete msgs - remove partial flag just in case
+			 * someone has funny ideas */
+		msg_item->msgFlags &= ~RS_MSG_FLAGS_PARTIAL;
+
+		// hack to pass on GXS id.
+		msg_item->PeerId(RsPeerId(authorId));
+		handleIncomingItem(msg_item);
+	}
+	else
+	{
+		std::cerr << __PRETTY_FUNCTION__ << " Item could not be "
+		          << "deserialised. Format error??" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId,
+                                             GxsTransSendStatus status )
+{
+	std::cout << __PRETTY_FUNCTION__ << " " << mailId << ", "
+	          << static_cast<uint>(status) << std::endl;
+
+	if( status == GxsTransSendStatus::RECEIPT_RECEIVED )
+	{
+		uint32_t msg_id;
+
+		{
+			RS_STACK_MUTEX(gxsOngoingMutex);
+
+			auto it = gxsOngoingMessages.find(mailId);
+			if(it == gxsOngoingMessages.end())
+			{
+				std::cerr << __PRETTY_FUNCTION__<< " "
+				          << mailId
+				          << ", " << static_cast<uint>(status)
+				          << " (EE) cannot find pending message to acknowledge!"
+				          << std::endl;
+				return false;
+			}
+
+			msg_id = it->second;
+		}
+
+		// we should now remove the item from the msgOutgoing list.
+
+		{
+			RS_STACK_MUTEX(mMsgMtx);
+
+			auto it2 = msgOutgoing.find(msg_id);
+			if(it2 == msgOutgoing.end())
+			{
+				std::cerr << __PRETTY_FUNCTION__ << " " << mailId
+				          << ", " << static_cast<uint>(status) << " (II) "
+				          << "received receipt for message that is not in "
+				          << "outgoing list, probably it has been acknoweldged "
+				          << "before by other means." << std::endl;
+				return true;
+			}
+
+			delete it2->second;
+			msgOutgoing.erase(it2);
+		}
+
+		RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST,
+		                                      NOTIFY_TYPE_ADD );
+		IndicateConfigChanged();
+
+		return true;
+	}
+
+	if( status >= GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE )
+	{
+		uint32_t msg_id;
+
+		{
+			RS_STACK_MUTEX(gxsOngoingMutex);
+
+			std::cerr << __PRETTY_FUNCTION__ << " mail delivery "
+			          << "mailId: " << mailId
+			          << " failed with " << static_cast<uint32_t>(status);
+
+			auto it = gxsOngoingMessages.find(mailId);
+			if(it == gxsOngoingMessages.end())
+			{
+				std::cerr << " cannot find pending message to notify"
+				          << std::endl;
+				return false;
+			}
+
+			msg_id = it->second;
+		}
+
+		std::cerr << " message id = " << msg_id << std::endl;
+
+		{
+			RS_STACK_MUTEX(mMsgMtx);
+			auto mit = msgOutgoing.find(msg_id);
+			if( mit == msgOutgoing.end() )
+			{
+				std::cerr << " message has been notified as not delivered, "
+				          << "but it not on outgoing list."
+				          << std::endl;
+				return true;
+			}
+			std::cerr << "  reseting the ROUTED flag so that the message is "
+			          << "requested again" << std::endl;
+
+			// clear the routed flag so that the message is requested again
+			mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED;
+			return true;
+		}
+	}
+
+	return true;
+}
+
+void p3MsgService::receiveGRouterData( const RsGxsId &destination_key,
+                                       const RsGxsId &signing_key,
+                                       GRouterServiceId &/*client_id*/,
+                                       uint8_t *data, uint32_t data_size )
+{
+	std::cerr << "p3MsgService::receiveGRouterData(): received message item of"
+	          << " size " << data_size << ", for key " << destination_key
+	          << std::endl;
+
+	/* first make sure that we havn't already received the data. Since we allow
+	 * to re-send messages, it's necessary to check. */
+
+	Sha1CheckSum hash = RsDirUtil::sha1sum(data, data_size);
+
+	{
+		RS_STACK_MUTEX(recentlyReceivedMutex);
+		if( mRecentlyReceivedMessageHashes.find(hash) !=
+		        mRecentlyReceivedMessageHashes.end() )
+		{
+			std::cerr << "p3MsgService::receiveGRouterData(...) (II) receiving"
+			          << "distant message of hash " << hash << " more than once"
+			          << ". Probably it has arrived  before by other means."
+			          << std::endl;
+			free(data);
+			return;
+		}
+		mRecentlyReceivedMessageHashes[hash] = time(NULL);
+	}
+
+	IndicateConfigChanged() ;
+
+	RsItem *item = _serialiser->deserialise(data,&data_size) ;
 	free(data) ;
-        return ;
-    }
-    mRecentlyReceivedDistantMessageHashes[hash] = time(NULL) ;
-    IndicateConfigChanged() ;
-    
-    RsItem *item = _serialiser->deserialise(data,&data_size) ;
-    free(data) ;
 
-    RsMsgItem *msg_item = dynamic_cast<RsMsgItem*>(item) ;
+	RsMsgItem *msg_item = dynamic_cast<RsMsgItem*>(item) ;
 
-    if(msg_item != NULL)
+	if(msg_item != NULL)
-    {
+	{
-        std::cerr << "  Encrypted item correctly deserialised. Passing on to incoming list." << std::endl;
+		std::cerr << "  Encrypted item correctly deserialised. Passing on to incoming list." << std::endl;
 
-	msg_item->msgFlags |= RS_MSG_FLAGS_DISTANT ;
+		msg_item->msgFlags |= RS_MSG_FLAGS_DISTANT ;
-	/* we expect complete msgs - remove partial flag just in case someone has funny ideas */
+		/* we expect complete msgs - remove partial flag just in case someone has funny ideas */
-	msg_item->msgFlags &= ~RS_MSG_FLAGS_PARTIAL;
+		msg_item->msgFlags &= ~RS_MSG_FLAGS_PARTIAL;
 
-        msg_item->PeerId(RsPeerId(signing_key)) ;	// hack to pass on GXS id.
+		msg_item->PeerId(RsPeerId(signing_key)) ;	// hack to pass on GXS id.
-        handleIncomingItem(msg_item) ;
+		handleIncomingItem(msg_item) ;
-    }
+	}
-    else
+	else
-        std::cerr << "  Item could not be deserialised. Format error??" << std::endl;
+		std::cerr << "  Item could not be deserialised. Format error??" << std::endl;
 }
 
 void p3MsgService::sendDistantMsgItem(RsMsgItem *msgitem)
 {
-    RsGxsId destination_key_id(msgitem->PeerId()) ;
+	RsGxsId destination_key_id(msgitem->PeerId());
-    RsGxsId signing_key_id ;
+	RsGxsId signing_key_id;
 
-    msgitem->msgFlags |= RS_MSG_FLAGS_DISTANT ;// just in case, but normally we should always have this flag set, when ending up here.
+	/* just in case, but normally we should always have this flag set, when
-    
+	 * ending up here. */
-    {
+	msgitem->msgFlags |= RS_MSG_FLAGS_DISTANT;
-        RS_STACK_MUTEX(mMsgMtx) ;
 
-        std::map<uint32_t,RsGxsId>::const_iterator it = mDistantOutgoingMsgSigners.find(msgitem->msgId) ;
+	{
-        
+		RS_STACK_MUTEX(mMsgMtx);
-        if(it == mDistantOutgoingMsgSigners.end())
-        {
-            std::cerr << "(EE) no signer registered for distant message " << msgitem->msgId << ". Cannot send!" << std::endl;
-            return ;
-        }
-            
-        signing_key_id = it->second ;
 
-        if(signing_key_id.isNull())
+		std::map<uint32_t,RsGxsId>::const_iterator it =
-        {
+		        mDistantOutgoingMsgSigners.find(msgitem->msgId);
-            std::cerr << "ERROR: cannot find signing key id for msg id " << msgitem->msgId << std::endl;
+
-            std::cerr << "  available keys are:" << std::endl;
+		if(it == mDistantOutgoingMsgSigners.end())
-            for(std::map<uint32_t,RsGxsId>::const_iterator it(mDistantOutgoingMsgSigners.begin());it!=mDistantOutgoingMsgSigners.end();++it)
+		{
-                std::cerr << "    " << it->first << " " << it->second << std::endl;
+			std::cerr << "(EE) no signer registered for distant message "
-            return ;
+			          << msgitem->msgId << ". Cannot send!" << std::endl;
-        }
+			return;
-    }
+		}
+
+		signing_key_id = it->second;
+
+		if(signing_key_id.isNull())
+		{
+			std::cerr << "ERROR: cannot find signing key id for msg id "
+			          << msgitem->msgId << " available keys are:" << std::endl;
+			typedef std::map<uint32_t,RsGxsId>::const_iterator itT;
+			for( itT it = mDistantOutgoingMsgSigners.begin();
+			     it != mDistantOutgoingMsgSigners.end(); ++it )
+				std::cerr << "\t" << it->first << " " << it->second
+				          << std::endl;
+			return;
+		}
+	}
 #ifdef DEBUG_DISTANT_MSG
-    std::cerr << "p3MsgService::sendDistanteMsgItem(): sending distant msg item" << std::endl;
+	std::cerr << "p3MsgService::sendDistanteMsgItem(): sending distant msg item"
-    std::cerr << "  msg ID  : " << msgitem->msgId << std::endl;
+	          << " msg ID: " << msgitem->msgId << " to peer:"
-    std::cerr << "  to peer : " << destination_key_id << std::endl;
+	          << destination_key_id << " signing: " << signing_key_id
-    std::cerr << "  signing : " << signing_key_id << std::endl;
+	          << std::endl;
 #endif
 
-    // The item is serialized and turned into a generic turtle item. Use use the explicit serialiser to make sure that the msgId is not included
+	/* The item is serialized and turned into a generic turtle item. Use use the
+	 * explicit serialiser to make sure that the msgId is not included */
 
     uint32_t msg_serialized_rssize = RsMsgSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_NONE).size(msgitem) ;
     RsTemporaryMemory msg_serialized_data(msg_serialized_rssize) ;
@@ -2034,19 +2251,32 @@ void p3MsgService::sendDistantMsgItem(RsMsgItem *msgitem)
         return ;
     }
 #ifdef DEBUG_DISTANT_MSG
-    std::cerr << "  serialised size : " << msg_serialized_rssize << std::endl;
+	std::cerr << "  serialised size : " << msg_serialized_rssize << std::endl;
 #endif
 
-    GRouterMsgPropagationId grouter_message_id ;
+	GRouterMsgPropagationId grouter_message_id;
-    mGRouter->sendData(destination_key_id,GROUTER_CLIENT_ID_MESSAGES,msg_serialized_data,msg_serialized_rssize,signing_key_id,grouter_message_id) ;
+	mGRouter->sendData( destination_key_id, GROUTER_CLIENT_ID_MESSAGES,
+	                    msg_serialized_data, msg_serialized_rssize,
+	                    signing_key_id, grouter_message_id );
+	RsGxsTransId gxsMailId;
+	mGxsTransServ.sendData( gxsMailId, GxsTransSubServices::P3_MSG_SERVICE,
+	                         signing_key_id, destination_key_id,
+	                         msg_serialized_data, msg_serialized_rssize );
 
-    // now store the grouter id along with the message id, so that we can keep track of received messages
+	/* now store the grouter id along with the message id, so that we can keep
+	 * track of received messages */
 
-    {
+	{
-        RS_STACK_MUTEX(mMsgMtx) ;
+		RS_STACK_MUTEX(mMsgMtx);
-        _ongoing_messages[grouter_message_id] = msgitem->msgId ;
+		_ongoing_messages[grouter_message_id] = msgitem->msgId;
-    }
+	}
-    IndicateConfigChanged();	// save _ongoing_messages
+
+	{
+		RS_STACK_MUTEX(gxsOngoingMutex);
+		gxsOngoingMessages[gxsMailId] = msgitem->msgId;
+	}
+
+	IndicateConfigChanged(); // save _ongoing_messages
 }
 
 

libretroshare/src/services/p3msgservice.h

@@ -49,16 +49,19 @@
 #include "grouter/grouterclientservice.h"
 #include "turtle/p3turtle.h"
 #include "turtle/turtleclientservice.h"
+#include "gxstrans/p3gxstrans.h"
 
 class p3LinkMgr;
 class p3IdService;
 
 // Temp tweak to test grouter
-class p3MsgService: public p3Service, public p3Config, public pqiServiceMonitor, public GRouterClientService
+struct p3MsgService :
+        p3Service, p3Config, pqiServiceMonitor, GRouterClientService,
+        GxsTransClient
 {
-public:
+	p3MsgService(p3ServiceControl *sc, p3IdService *id_service, p3GxsTrans& gxsMS);
-    p3MsgService(p3ServiceControl *sc, p3IdService *id_service);
+
-    virtual RsServiceInfo getServiceInfo();
+	virtual RsServiceInfo getServiceInfo();
 
     /* External Interface */
     bool 	getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList);
@@ -106,7 +109,9 @@ public:
 
     /*** Overloaded from pqiMonitor ***/
     virtual void    statusChange(const std::list<pqiServicePeer> &plist);
-    int     checkOutgoingMessages();
+
+	/// iterate through the outgoing queue if online, send
+	int checkOutgoingMessages();
     /*** Overloaded from pqiMonitor ***/
 
     /*** overloaded from p3turtle   ***/
@@ -130,16 +135,27 @@ public:
     void setDistantMessagingPermissionFlags(uint32_t flags) ;
     uint32_t getDistantMessagingPermissionFlags() ;
 
-private:
+	/// @see GxsTransClient::receiveGxsTransMail(...)
-    void sendDistantMsgItem(RsMsgItem *msgitem) ;
+	virtual bool receiveGxsTransMail( const RsGxsId& authorId,
+	                                  const RsGxsId& recipientId,
+	                                  const uint8_t* data, uint32_t dataSize );
 
-    // This contains the ongoing tunnel handling contacts.
+	/// @see GxsTransClient::notifyGxsTransSendStatus(...)
-    // The map is indexed by the hash
+	virtual bool notifyGxsTransSendStatus( RsGxsTransId mailId,
-    //
+	                                       GxsTransSendStatus status );
-    std::map<GRouterMsgPropagationId,uint32_t> _ongoing_messages ;
+
+private:
+	void sendDistantMsgItem(RsMsgItem *msgitem);
+
+	/** This contains the ongoing tunnel handling contacts.
+	 * The map is indexed by the hash */
+	std::map<GRouterMsgPropagationId, uint32_t> _ongoing_messages;
+
+	/// Contains ongoing messages handed to gxs mail
+	std::map<RsGxsTransId, uint32_t> gxsOngoingMessages;
+	RsMutex gxsOngoingMutex;
 
     // Overloaded from GRouterClientService
-
     virtual bool acceptDataFromPeer(const RsGxsId& gxs_id) ;
     virtual void receiveGRouterData(const RsGxsId& destination_key,const RsGxsId& signing_key, GRouterServiceId &client_id, uint8_t *data, uint32_t data_size) ;
     virtual void notifyDataStatus(const GRouterMsgPropagationId& msg_id,const RsGxsId& signer_id,uint32_t data_status) ;
@@ -194,8 +210,9 @@ private:
     std::map<uint32_t, RsMsgTagType*> mTags;
     std::map<uint32_t, RsMsgTags*> mMsgTags;
 
-    uint32_t mMsgUniqueId;
+	uint32_t mMsgUniqueId;
-    std::map<Sha1CheckSum,uint32_t> mRecentlyReceivedDistantMessageHashes;
+	std::map<Sha1CheckSum, uint32_t> mRecentlyReceivedMessageHashes;
+	RsMutex recentlyReceivedMutex;
 
     // used delete msgSrcIds after config save
     std::map<uint32_t, RsMsgSrcId*> mSrcIds;
@@ -211,6 +228,8 @@ private:
     bool mDistantMessagingEnabled ;
     uint32_t mDistantMessagePermissions ;
     bool mShouldEnableDistantMessaging ;
+
+	p3GxsTrans& mGxsTransServ;
 };
 
 #endif // MESSAGE_SERVICE_HEADER

libretroshare/src/util/rsthreads.cc

@@ -61,6 +61,16 @@ int RS_pthread_setname_np(pthread_t __target_thread, const char *__buf) {
 	#include <iostream>
 #endif
 
+void RsThread::go()
+{
+    mShouldStopSemaphore.set(0) ;
+    mHasStoppedSemaphore.set(0) ;
+
+    runloop();
+
+    mHasStoppedSemaphore.set(1);
+    mShouldStopSemaphore.set(0);
+}
 void *RsThread::rsthread_init(void* p)
 {
   RsThread *thread = (RsThread *) p;
@@ -76,7 +86,7 @@ void *RsThread::rsthread_init(void* p)
     std::cerr << "[Thread ID:" << std::hex << pthread_self() << std::dec << "] thread is started. Calling runloop()..." << std::endl;
 #endif
     
-  thread -> runloop();
+  thread->go();
   return NULL;
 }
 RsThread::RsThread() 
@@ -216,14 +226,11 @@ RsTickingThread::RsTickingThread()
 
 void RsSingleJobThread::runloop()
 {
-    mHasStoppedSemaphore.set(0) ;
     run() ;
 }
 
 void RsTickingThread::runloop()
 {
-    mHasStoppedSemaphore.set(0) ;	// first time we are 100% the thread is actually running.
-
 #ifdef DEBUG_THREADS
     THREAD_DEBUG << "RsTickingThread::runloop(). Setting stopped=0" << std::endl;
 #endif
@@ -235,7 +242,6 @@ void RsTickingThread::runloop()
 #ifdef DEBUG_THREADS
             THREAD_DEBUG << "pqithreadstreamer::runloop(): asked to stop. setting hasStopped=1, and returning. Thread ends." << std::endl;
 #endif
-            mHasStoppedSemaphore.set(1);
             return ;
         }
 

libretroshare/src/util/rsthreads.h

@@ -263,6 +263,7 @@ class RsThread
 
 protected:
     virtual void runloop() =0; /* called once the thread is started. Should be overloaded by subclasses. */
+    void go() ;	// this one calls runloop and also sets the flags correctly when the thread is finished running.
 
     RsSemaphore mHasStoppedSemaphore;
     RsSemaphore mShouldStopSemaphore;

retroshare-gui/src/gui/chat/ChatWidget.cpp

@@ -418,9 +418,11 @@ ChatWidget::ChatType ChatWidget::chatType()
 
 void ChatWidget::blockSending(QString msg)
 {
-    sendingBlocked = true;
+#ifndef RS_ASYNC_CHAT
-    ui->sendButton->setEnabled(false);
+	sendingBlocked = true;
-    ui->sendButton->setToolTip(msg);
+	ui->sendButton->setEnabled(false);
+#endif
+	ui->sendButton->setToolTip(msg);
 }
 
 void ChatWidget::unblockSending()

retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp

@@ -107,41 +107,52 @@ void PopupDistantChatDialog::updateDisplay()
 
     QString msg;
 
-    switch(tinfo.status)
+	switch(tinfo.status)
-    {
+	{
-    case RS_DISTANT_CHAT_STATUS_UNKNOWN: //std::cerr << "Unknown hash. Error!" << std::endl;
+	case RS_DISTANT_CHAT_STATUS_UNKNOWN:
-	    _status_label->setIcon(QIcon(IMAGE_GRY_LED)) ;
+		//std::cerr << "Unknown hash. Error!" << std::endl;
-	    msg = tr("Chat remotely closed. Please close this window.");
+		_status_label->setIcon(QIcon(IMAGE_GRY_LED));
-	    _status_label->setToolTip(msg) ;
+		msg = tr("Remote status unknown.");
-	    getChatWidget()->updateStatusString("%1", msg, true);
+		_status_label->setToolTip(msg);
-	    getChatWidget()->blockSending(tr("Can't send message, because there is no tunnel."));
+		getChatWidget()->updateStatusString("%1", msg, true);
-	    setPeerStatus(RS_STATUS_OFFLINE) ;
+		getChatWidget()->blockSending(tr( "Can't send message immediately, "
-	    break ;
+		                                  "because there is no tunnel "
-    case RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED: std::cerr << "Chat remotely closed. " << std::endl;
+		                                  "available." ));
-	    _status_label->setIcon(QIcon(IMAGE_RED_LED)) ;
+		setPeerStatus(RS_STATUS_OFFLINE);
-	    _status_label->setToolTip(QObject::tr("Distant peer has closed the chat")) ;
+		break ;
-
+	case RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED:
-	    getChatWidget()->updateStatusString("%1", tr("The person you're talking to has deleted the secured chat tunnel. You may remove the chat window now."), true);
+		std::cerr << "Chat remotely closed. " << std::endl;
-	    getChatWidget()->blockSending(tr("Can't send message, because the chat partner deleted the secure tunnel."));
+		_status_label->setIcon(QIcon(IMAGE_RED_LED));
-	    setPeerStatus(RS_STATUS_OFFLINE) ;
+		_status_label->setToolTip(
-
+		            QObject::tr("Distant peer has closed the chat") );
-	    break ;
+		getChatWidget()->updateStatusString(
-    case RS_DISTANT_CHAT_STATUS_TUNNEL_DN: //std::cerr << "Tunnel asked. Waiting for reponse. " << std::endl;
+		            "%1", tr( "The person you are talking to has deleted the"
-	    _status_label->setIcon(QIcon(IMAGE_RED_LED)) ;
+		                      " secured chat tunnel." ), true );
-	    msg = QObject::tr("Tunnel is pending...");
+		getChatWidget()->blockSending(tr( "The chat partner deleted the secure"
-	    _status_label->setToolTip(msg) ;
+		                                  " tunnel, messages will be delivered"
-	    getChatWidget()->updateStatusString("%1", msg, true);
+		                                  " as soon as possible"));
-	    getChatWidget()->blockSending(msg);
+		setPeerStatus(RS_STATUS_OFFLINE) ;
-	    setPeerStatus(RS_STATUS_OFFLINE) ;
+		break ;
-	    break ;
+	case RS_DISTANT_CHAT_STATUS_TUNNEL_DN:
-    case RS_DISTANT_CHAT_STATUS_CAN_TALK: //std::cerr << "Tunnel is ok and data is transmitted." << std::endl;
+		//std::cerr << "Tunnel asked. Waiting for reponse. " << std::endl;
-	    _status_label->setIcon(QIcon(IMAGE_GRN_LED)) ;
+		_status_label->setIcon(QIcon(IMAGE_RED_LED));
-	    msg = QObject::tr("Secured tunnel is working. You can talk!");
+		msg = QObject::tr( "Tunnel is pending... Messages will be delivered as"
-	    _status_label->setToolTip(msg) ;
+		                   " soon as possible" );
-	    getChatWidget()->unblockSending();
+		_status_label->setToolTip(msg);
-	    setPeerStatus(RS_STATUS_ONLINE) ;
+		getChatWidget()->updateStatusString("%1", msg, true);
-	    break ;
+		getChatWidget()->blockSending(msg);
-    }
+		setPeerStatus(RS_STATUS_OFFLINE);
+		break;
+	case RS_DISTANT_CHAT_STATUS_CAN_TALK:
+		//std::cerr << "Tunnel is ok and data is transmitted." << std::endl;
+		_status_label->setIcon(QIcon(IMAGE_GRN_LED));
+		msg = QObject::tr( "Secured tunnel is working. "
+		                   "Messages are delivered immediately!" );
+		_status_label->setToolTip(msg);
+		getChatWidget()->unblockSending();
+		setPeerStatus(RS_STATUS_ONLINE);
+		break;
+	}
 }
 
 void PopupDistantChatDialog::closeEvent(QCloseEvent *e)

retroshare-gui/src/gui/icons.qrc

@@ -13,6 +13,12 @@
     <file>icons/anonymous_blue_128.png</file>
     <file>icons/anonymous_green_128.png</file>
     <file>icons/aol.png</file>
+    <file>icons/transport128.png</file>
+    <file>icons/bandwidth128.png</file>
+    <file>icons/RTT128.png</file>
+    <file>icons/DHT128.png</file>
+    <file>icons/turtle128.png</file>
+    <file>icons/GRouter128.png</file>
     <file>icons/avatar_128.png</file>
     <file>icons/avatar_grey_128.png</file>
     <file>icons/biohazard_red.png</file>

retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp

@@ -0,0 +1,443 @@
+/****************************************************************
+ *  RetroShare is distributed under the following license:
+ *
+ *  Copyright (C) 20011, RetroShare Team
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#include <iostream>
+#include <QTimer>
+#include <QObject>
+#include <QFontMetrics>
+#include <QWheelEvent>
+#include <QDateTime>
+#include <time.h>
+
+#include <QMenu>
+#include <QPainter>
+#include <QStylePainter>
+#include <QLayout>
+#include <QHeaderView>
+
+#include <retroshare/rsgxstrans.h>
+#include <retroshare/rspeers.h>
+#include <retroshare/rsidentity.h>
+#include <retroshare/rsgxstrans.h>
+
+#include "GxsTransportStatistics.h"
+
+#include "gui/Identity/IdDetailsDialog.h"
+#include "gui/settings/rsharesettings.h"
+#include "util/QtVersion.h"
+#include "gui/common/UIStateHelper.h"
+#include "util/misc.h"
+#include "gui/gxs/GxsIdLabel.h"
+
+#define COL_PENDING_ID                  0
+#define COL_PENDING_DESTINATION         1
+#define COL_PENDING_DATASTATUS          2
+#define COL_PENDING_DATASIZE            3
+#define COL_PENDING_DATAHASH            4
+#define COL_PENDING_SEND                5
+#define COL_PENDING_GROUP_ID            6
+
+#define COL_GROUP_GRP_ID                  0
+#define COL_GROUP_NUM_MSGS                1
+#define COL_GROUP_SIZE_MSGS               2
+#define COL_GROUP_SUBSCRIBED              3
+#define COL_GROUP_POPULARITY              4
+
+static const int PARTIAL_VIEW_SIZE                           =  9 ;
+static const int MAX_TUNNEL_REQUESTS_DISPLAY                 = 10 ;
+static const int GXSTRANS_STATISTICS_DELAY_BETWEEN_GROUP_REQ = 30 ;	// never request more than every 30 secs.
+
+#define GXSTRANS_GROUP_META  0x01
+#define GXSTRANS_GROUP_DATA  0x02
+#define GXSTRANS_GROUP_STAT  0x03
+#define GXSTRANS_MSG_META    0x04
+
+#define DEBUG_GXSTRANS_STATS 1
+
+GxsTransportStatistics::GxsTransportStatistics(QWidget *parent)
+    : RsAutoUpdatePage(2000,parent)
+{
+	setupUi(this) ;
+	
+	mStateHelper = new UIStateHelper(this);
+	mStateHelper->addWidget(GXSTRANS_GROUP_META, treeWidget);
+
+	mTransQueue = new TokenQueue(rsGxsTrans->getTokenService(), this);
+
+	m_bProcessSettings = false;
+    mLastGroupReqTS = 0 ;
+
+	/* Set header resize modes and initial section sizes Uploads TreeView*/
+	QHeaderView_setSectionResizeMode(treeWidget->header(), QHeaderView::ResizeToContents);
+
+	connect(treeWidget, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(CustomPopupMenu(QPoint)));
+
+
+	// load settings
+	processSettings(true);
+}
+
+GxsTransportStatistics::~GxsTransportStatistics()
+{
+
+    // save settings
+    processSettings(false);
+}
+
+void GxsTransportStatistics::processSettings(bool bLoad)
+{
+    m_bProcessSettings = true;
+
+    Settings->beginGroup(QString("GxsTransportStatistics"));
+
+    if (bLoad) {
+        // load settings
+
+        // state of splitter
+        //splitter->restoreState(Settings->value("Splitter").toByteArray());
+    } else {
+        // save settings
+
+        // state of splitter
+        //Settings->setValue("Splitter", splitter->saveState());
+
+    }
+
+    Settings->endGroup();
+    
+    m_bProcessSettings = false;
+}
+
+void GxsTransportStatistics::CustomPopupMenu( QPoint )
+{
+	QMenu contextMnu( this );
+	
+	QTreeWidgetItem *item = treeWidget->currentItem();
+	if (item) {
+	contextMnu.addAction(QIcon(":/images/info16.png"), tr("Details"), this, SLOT(personDetails()));
+
+  }
+
+	contextMnu.exec(QCursor::pos());
+}
+
+void GxsTransportStatistics::updateDisplay()
+{
+    time_t now = time(NULL) ;
+
+    if(mLastGroupReqTS + GXSTRANS_STATISTICS_DELAY_BETWEEN_GROUP_REQ < now)
+    {
+        requestGroupMeta();
+        mLastGroupReqTS = now ;
+    }
+
+	//_tst_CW->updateContent() ;
+
+	updateContent();
+}
+
+QString GxsTransportStatistics::getPeerName(const RsPeerId &peer_id)
+{
+	static std::map<RsPeerId, QString> names ;
+
+	std::map<RsPeerId,QString>::const_iterator it = names.find(peer_id) ;
+
+	if( it != names.end())
+		return it->second ;
+	else
+	{
+		RsPeerDetails detail ;
+		if(!rsPeers->getPeerDetails(peer_id,detail))
+			return tr("Unknown Peer");
+
+		return (names[peer_id] = QString::fromUtf8(detail.name.c_str())) ;
+	}
+}
+
+static QString getStatusString(GxsTransSendStatus status)
+{
+    switch(status)
+	{
+    case GxsTransSendStatus::PENDING_PROCESSING         :  return QObject::tr("Processing") ;
+	case GxsTransSendStatus::PENDING_PREFERRED_GROUP    :  return QObject::tr("Choosing group") ;
+	case GxsTransSendStatus::PENDING_RECEIPT_CREATE     :  return QObject::tr("Creating receipt") ;
+	case GxsTransSendStatus::PENDING_RECEIPT_SIGNATURE  :  return QObject::tr("Signing receipt") ;
+	case GxsTransSendStatus::PENDING_SERIALIZATION      :  return QObject::tr("Serializing") ;
+	case GxsTransSendStatus::PENDING_PAYLOAD_CREATE     :  return QObject::tr("Creating payload") ;
+	case GxsTransSendStatus::PENDING_PAYLOAD_ENCRYPT    :  return QObject::tr("Encrypting payload") ;
+	case GxsTransSendStatus::PENDING_PUBLISH            :  return QObject::tr("Publishing") ;
+	case GxsTransSendStatus::PENDING_RECEIPT_RECEIVE    :  return QObject::tr("Waiting for receipt") ;
+	case GxsTransSendStatus::RECEIPT_RECEIVED           :  return QObject::tr("Receipt received") ;
+	case GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE   :  return QObject::tr("Receipt signature failed") ;
+	case GxsTransSendStatus::FAILED_ENCRYPTION          :  return QObject::tr("Encryption failed") ;
+	case GxsTransSendStatus::UNKNOWN                    :
+	default                                             :  return QObject::tr("Unknown") ;
+	}
+}
+
+void GxsTransportStatistics::updateContent()
+{
+    RsGxsTrans::GxsTransStatistics transinfo ;
+
+    rsGxsTrans->getStatistics(transinfo) ;
+
+
+    // clear
+
+    treeWidget->clear();
+    time_t now = time(NULL) ;
+    
+    // 1 - fill the table for pending packets
+
+    groupBox->setTitle(tr("Pending data items")+": " + QString::number(transinfo.outgoing_records.size()) );
+
+    for(uint32_t i=0;i<transinfo.outgoing_records.size();++i)
+    {
+        const RsGxsTransOutgoingRecord& rec(transinfo.outgoing_records[i]) ;
+
+        QTreeWidgetItem *item = new QTreeWidgetItem();
+        treeWidget->addTopLevelItem(item);
+        
+        RsIdentityDetails details ;
+        rsIdentity->getIdDetails(rec.recipient,details);
+        QString nickname = QString::fromUtf8(details.mNickname.c_str());
+        
+        if(nickname.isEmpty())
+          nickname = tr("Unknown");
+
+        item -> setData(COL_PENDING_ID,           Qt::DisplayRole, QString::number(rec.trans_id,16).rightJustified(8,'0'));
+        item -> setData(COL_PENDING_DATASTATUS,   Qt::DisplayRole, getStatusString(rec.status));
+        item -> setData(COL_PENDING_DATASIZE,     Qt::DisplayRole, misc::friendlyUnit(rec.data_size));
+        item -> setData(COL_PENDING_DATAHASH,     Qt::DisplayRole, QString::fromStdString(rec.data_hash.toStdString()));
+		item -> setData(COL_PENDING_SEND,         Qt::DisplayRole, QDateTime::fromTime_t(rec.send_TS).toString());
+        item -> setData(COL_PENDING_GROUP_ID,     Qt::DisplayRole, QString::fromStdString(rec.group_id.toStdString()));
+
+        GxsIdLabel *label = new GxsIdLabel() ;
+        label->setId(rec.recipient) ;
+
+        treeWidget -> setItemWidget(item,COL_PENDING_DESTINATION, label) ;
+    }
+
+    // 2 - fill the table for pending group data
+
+    // record openned groups
+
+    std::set<RsGxsGroupId> openned_groups ;
+
+    for(uint32_t i=0;i<groupTreeWidget->topLevelItemCount();++i)
+        if(groupTreeWidget->isItemExpanded(groupTreeWidget->topLevelItem(i)))
+            openned_groups.insert(RsGxsGroupId(groupTreeWidget->topLevelItem(i)->data(COL_GROUP_GRP_ID,Qt::DisplayRole).toString().toStdString())) ;
+
+    groupTreeWidget->clear();
+
+    for(std::map<RsGxsGroupId,RsGxsTransGroupStatistics>::const_iterator it(mGroupStats.begin());it!=mGroupStats.end();++it)
+    {
+        const RsGxsTransGroupStatistics& stat(it->second) ;
+
+		QTreeWidgetItem *item = new QTreeWidgetItem();
+        groupTreeWidget->addTopLevelItem(item);
+		groupTreeWidget->setItemExpanded(item,openned_groups.find(it->first) != openned_groups.end());
+
+        item->setData(COL_GROUP_GRP_ID,    Qt::DisplayRole,  QString::fromStdString(stat.mGrpId.toStdString())) ;
+        item->setData(COL_GROUP_NUM_MSGS,  Qt::DisplayRole,  QString::number(stat.mNumMsgs)) ;
+        item->setData(COL_GROUP_SIZE_MSGS, Qt::DisplayRole,  QString::number(stat.mTotalSizeOfMsgs)) ;
+        item->setData(COL_GROUP_SUBSCRIBED,Qt::DisplayRole,  stat.subscribed?tr("Yes"):tr("No")) ;
+        item->setData(COL_GROUP_POPULARITY,Qt::DisplayRole,  QString::number(stat.popularity)) ;
+
+        for(uint32_t i=0;i<it->second.messages_metas.size();++i)
+        {
+            QTreeWidgetItem *sitem = new QTreeWidgetItem(item) ;
+
+            const RsMsgMetaData& meta(it->second.messages_metas[i]) ;
+
+            GxsIdLabel *label = new GxsIdLabel();
+            label->setId(meta.mAuthorId) ;
+            groupTreeWidget->setItemWidget(sitem,COL_GROUP_GRP_ID,label) ;
+
+            sitem->setData(COL_GROUP_NUM_MSGS,Qt::DisplayRole, QDateTime::fromTime_t(meta.mPublishTs).toString());
+        }
+    }
+}
+
+void GxsTransportStatistics::personDetails()
+{
+    QTreeWidgetItem *item = treeWidget->currentItem();
+    std::string id = item->text(COL_PENDING_DESTINATION).toStdString();
+
+    if (id.empty()) {
+        return;
+    }
+    
+    IdDetailsDialog *dialog = new IdDetailsDialog(RsGxsGroupId(id));
+    dialog->show();
+}
+
+void GxsTransportStatistics::loadRequest(const TokenQueue *queue, const TokenRequest &req)
+{
+#ifdef DEBUG_GXSTRANS_STATS
+	std::cerr << "GxsTransportStatistics::loadRequest() UserType: " << req.mUserType << std::endl;
+#endif
+
+	if (queue != mTransQueue)
+	{
+		std::cerr << "Wrong queue!" << std::endl;
+		return ;
+	}
+
+	/* now switch on req */
+	switch(req.mUserType)
+	{
+	case GXSTRANS_GROUP_META: loadGroupMeta(req.mToken);
+		break;
+
+	case GXSTRANS_GROUP_STAT: loadGroupStat(req.mToken);
+		break;
+
+	case GXSTRANS_MSG_META: loadMsgMeta(req.mToken);
+		break;
+
+	default:
+		std::cerr << "GxsTransportStatistics::loadRequest() ERROR: INVALID TYPE";
+		std::cerr << std::endl;
+		break;
+	}
+}
+
+void GxsTransportStatistics::requestGroupMeta()
+{
+	mStateHelper->setLoading(GXSTRANS_GROUP_META, true);
+
+#ifdef DEBUG_GXSTRANS_STATS
+	std::cerr << "GxsTransportStatisticsWidget::requestGroupMeta()";
+	std::cerr << std::endl;
+#endif
+
+	mTransQueue->cancelActiveRequestTokens(GXSTRANS_GROUP_META);
+
+	RsTokReqOptions opts;
+	opts.mReqType = GXS_REQUEST_TYPE_GROUP_META;
+
+	uint32_t token;
+	mTransQueue->requestGroupInfo(token,  RS_TOKREQ_ANSTYPE_SUMMARY, opts, GXSTRANS_GROUP_META);
+}
+void GxsTransportStatistics::requestGroupStat(const RsGxsGroupId &groupId)
+{
+	mTransQueue->cancelActiveRequestTokens(GXSTRANS_GROUP_STAT);
+	uint32_t token;
+	rsGxsTrans->getTokenService()->requestGroupStatistic(token, groupId);
+	mTransQueue->queueRequest(token, 0, RS_TOKREQ_ANSTYPE_ACK, GXSTRANS_GROUP_STAT);
+}
+void GxsTransportStatistics::requestMsgMeta(const RsGxsGroupId& grpId)
+{
+	mStateHelper->setLoading(GXSTRANS_MSG_META, true);
+
+#ifdef DEBUG_GXSTRANS_STATS
+	std::cerr << "GxsTransportStatisticsWidget::requestGroupMeta()";
+	std::cerr << std::endl;
+#endif
+
+	mTransQueue->cancelActiveRequestTokens(GXSTRANS_MSG_META);
+
+	RsTokReqOptions opts;
+	opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
+
+    std::list<RsGxsGroupId> grouplist ;
+	grouplist.push_back(grpId) ;
+
+	uint32_t token;
+	rsGxsTrans->getTokenService()->requestMsgInfo(token,  RS_TOKREQ_ANSTYPE_SUMMARY, opts, grouplist);
+	mTransQueue->queueRequest(token, 0, RS_TOKREQ_ANSTYPE_ACK, GXSTRANS_MSG_META);
+}
+
+void GxsTransportStatistics::loadGroupStat(const uint32_t &token)
+{
+#ifdef DEBUG_GXSTRANS_STATS
+    std::cerr << "GxsTransportStatistics::loadGroupStat." << std::endl;
+#endif
+	GxsGroupStatistic stats;
+	rsGxsTrans->getGroupStatistic(token, stats);
+
+    dynamic_cast<GxsGroupStatistic&>(mGroupStats[stats.mGrpId]) = stats ;
+}
+
+void GxsTransportStatistics::loadGroupMeta(const uint32_t& token)
+{
+	mStateHelper->setLoading(GXSTRANS_GROUP_META, false);
+
+#ifdef DEBUG_GXSTRANS_STATS
+	std::cerr << "GxsTransportStatisticsWidget::loadGroupMeta()";
+	std::cerr << std::endl;
+#endif
+
+	std::list<RsGroupMetaData> groupInfo;
+	std::list<RsGroupMetaData>::iterator vit;
+
+	if (!rsGxsTrans->getGroupSummary(token,groupInfo))
+	{
+		std::cerr << "GxsTransportStatistics::loadGroupMeta() Error getting GroupMeta";
+		std::cerr << std::endl;
+		mStateHelper->setActive(GXSTRANS_GROUP_META, false);
+		return;
+	}
+
+	mStateHelper->setActive(GXSTRANS_GROUP_META, true);
+
+	std::set<RsGxsGroupId> existing_groups ;
+
+	for(vit = groupInfo.begin(); vit != groupInfo.end(); ++vit)
+	{
+        existing_groups.insert(vit->mGroupId) ;
+
+		/* Add Widget, and request Pages */
+#ifdef DEBUG_GXSTRANS_STATS
+		std::cerr << "GxsTransportStatisticsWidget::loadGroupMeta() GroupId: " << vit->mGroupId << " Group: " << vit->mGroupName << std::endl;
+#endif
+
+        requestGroupStat(vit->mGroupId) ;
+        requestMsgMeta(vit->mGroupId) ;
+
+        RsGxsTransGroupStatistics& s(mGroupStats[vit->mGroupId]);
+        s.popularity = vit->mPop ;
+        s.subscribed = IS_GROUP_SUBSCRIBED(vit->mSubscribeFlags) ;
+	}
+
+    // remove group stats for group that do not exist anymore
+
+    for(std::map<RsGxsGroupId,RsGxsTransGroupStatistics>::iterator it(mGroupStats.begin());it!=mGroupStats.end();)
+        if(existing_groups.find(it->first) == existing_groups.end())
+			it = mGroupStats.erase(it);
+		else
+			++it;
+}
+
+void GxsTransportStatistics::loadMsgMeta(const uint32_t& token)
+{
+	mStateHelper->setLoading(GXSTRANS_MSG_META, false);
+
+    GxsMsgMetaMap m ;
+
+	if (!rsGxsTrans->getMsgSummary(token,m))
+        return ;
+
+    for(GxsMsgMetaMap::const_iterator it(m.begin());it!=m.end();++it)
+        mGroupStats[it->first].messages_metas = it->second ;
+}
+

retroshare-gui/src/gui/statistics/GxsTransportStatistics.h

@@ -0,0 +1,90 @@
+/****************************************************************
+ *  RetroShare is distributed under the following license:
+ *
+ *  Copyright (C) 20011, RetroShare Team
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#pragma once
+
+#include <map>
+
+#include <QPoint>
+#include <retroshare/rsgrouter.h>
+#include <retroshare/rstypes.h>
+
+#include "util/TokenQueue.h"
+#include "RsAutoUpdatePage.h"
+#include "ui_GxsTransportStatistics.h"
+
+class GxsTransportStatisticsWidget ;
+class UIStateHelper;
+
+class RsGxsTransGroupStatistics: public GxsGroupStatistic
+{
+public:
+	RsGxsTransGroupStatistics() {}
+
+	bool subscribed ;
+	int  popularity ;
+
+    std::vector<RsMsgMetaData> messages_metas ;
+};
+
+class GxsTransportStatistics: public RsAutoUpdatePage, public TokenResponse, public Ui::GxsTransportStatistics
+{
+	Q_OBJECT
+
+public:
+	GxsTransportStatistics(QWidget *parent = NULL) ;
+	~GxsTransportStatistics();
+
+	// Cache for peer names.
+	static QString getPeerName(const RsPeerId& peer_id) ;
+
+	virtual void loadRequest(const TokenQueue *queue, const TokenRequest &req) ;
+
+	void updateContent() ;
+
+private slots:
+	/** Create the context popup menu and it's submenus */
+	void CustomPopupMenu( QPoint point );
+	void personDetails();
+
+private:
+	void loadGroupMeta(const uint32_t& token);
+	void loadGroupStat(const uint32_t& token);
+	void loadMsgMeta(const uint32_t& token);
+
+	void requestGroupMeta();
+	void requestMsgMeta(const RsGxsGroupId& groupId);
+	void requestGroupStat(const RsGxsGroupId &groupId);
+
+	void processSettings(bool bLoad);
+	bool m_bProcessSettings;
+
+	virtual void updateDisplay() ;
+
+	GxsTransportStatisticsWidget *_tst_CW ;
+	TokenQueue *mTransQueue ;
+	UIStateHelper *mStateHelper;
+	uint32_t mLastGroupReqTS ;
+
+	// temporary storage of retrieved data, for display (useful because this is obtained from the async token system)
+
+	std::map<RsGxsGroupId,RsGxsTransGroupStatistics> mGroupStats ;	// stores the list of active groups and statistics about each of them.
+} ;

retroshare-gui/src/gui/statistics/GxsTransportStatistics.ui

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>GxsTransportStatistics</class>
+ <widget class="QWidget" name="GxsTransportStatistics">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>1468</width>
+    <height>659</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Router Statistics</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="1" column="0">
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Gxs Transport Groups:</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout">
+      <item>
+       <widget class="QSplitter" name="splitter">
+        <property name="orientation">
+         <enum>Qt::Vertical</enum>
+        </property>
+        <widget class="QTreeWidget" name="groupTreeWidget">
+         <column>
+          <property name="text">
+           <string>Group ID / Author</string>
+          </property>
+         </column>
+         <column>
+          <property name="text">
+           <string>Number of messages / Publish TS</string>
+          </property>
+         </column>
+         <column>
+          <property name="text">
+           <string>Total size of messages</string>
+          </property>
+         </column>
+         <column>
+          <property name="text">
+           <string>Subscribed</string>
+          </property>
+         </column>
+         <column>
+          <property name="text">
+           <string>Popularity</string>
+          </property>
+         </column>
+        </widget>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="0" column="0">
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>GroupBox</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
+      <item row="0" column="0">
+       <widget class="QTreeWidget" name="treeWidget">
+        <property name="contextMenuPolicy">
+         <enum>Qt::CustomContextMenu</enum>
+        </property>
+        <property name="sortingEnabled">
+         <bool>true</bool>
+        </property>
+        <attribute name="headerShowSortIndicator" stdset="0">
+         <bool>false</bool>
+        </attribute>
+        <column>
+         <property name="text">
+          <string>ID</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Destination</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Data status</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Data size</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Data hash</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Sending time (secs ago)</string>
+         </property>
+        </column>
+        <column>
+         <property name="text">
+          <string>Group ID</string>
+         </property>
+        </column>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

retroshare-gui/src/gui/statistics/StatisticsWindow.cpp

@@ -34,8 +34,11 @@
 #include "retroshare/rspeers.h"
 #include <retroshare/rsplugin.h>
 
+#include <gui/settings/rsharesettings.h>
+
 #include <gui/statistics/TurtleRouterStatistics.h>
 #include <gui/statistics/GlobalRouterStatistics.h>
+#include <gui/statistics/GxsTransportStatistics.h>
 #include <gui/statistics/BwCtrlWindow.h>
 #include <gui/statistics/DhtWindow.h>
 
@@ -48,12 +51,14 @@
 	#include "gui/statistics/RttStatistics.h"
 #endif
 
-#define IMAGE_DHT           ":/images/dht32.png"
+#define IMAGE_DHT           ":/icons/DHT128.png"
-#define IMAGE_TURTLE        ":images/turtle.png"
+#define IMAGE_TURTLE        ":/icons/turtle128.png"
-#define IMAGE_BWGRAPH           ":/images/ksysguard.png"
+#define IMAGE_BWGRAPH       ":/icons/bandwidth128.png"
-#define IMAGE_GLOBALROUTER           ":/images/network32.png"
+#define IMAGE_GLOBALROUTER  ":/icons/GRouter128.png"
-#define IMAGE_BANDWIDTH       ":images/office-chart-area-stacked.png"
+#define IMAGE_GXSTRANSPORT  ":/icons/transport128.png"
-#define IMAGE_RTT             ":images/office-chart-line.png"
+#define IMAGE_RTT           ":/icons/RTT128.png"
+
+//#define IMAGE_BANDWIDTH     ":images/office-chart-area-stacked.png"
 
 /********************************************** STATIC WINDOW *************************************/
 StatisticsWindow * StatisticsWindow::mInstance = NULL;
@@ -94,6 +99,9 @@ StatisticsWindow::StatisticsWindow(QWidget *parent) :
     connect(ui->stackPages, SIGNAL(currentChanged(int)), this, SLOT(setNewPage(int)));
     ui->stackPages->setCurrentIndex(0);
 
+	int toolSize = Settings->getToolButtonSize();
+	ui->toolBar->setToolButtonStyle(Settings->getToolButtonStyle());
+	ui->toolBar->setIconSize(QSize(toolSize,toolSize));
 }
 
 StatisticsWindow::~StatisticsWindow()
@@ -125,7 +133,7 @@ void StatisticsWindow::initStackedPage()
   QAction *action;
   
   ui->stackPages->add(bwdlg = new BwCtrlWindow(ui->stackPages),
-                   action = createPageAction(QIcon(IMAGE_BANDWIDTH), tr("Bandwidth"), grp));
+                   action = createPageAction(QIcon(IMAGE_BWGRAPH), tr("Bandwidth"), grp));
                    
   ui->stackPages->add(trsdlg = new TurtleRouterStatistics(ui->stackPages),
                    action = createPageAction(QIcon(IMAGE_TURTLE), tr("Turtle Router"), grp));
@@ -133,6 +141,9 @@ void StatisticsWindow::initStackedPage()
   ui->stackPages->add(grsdlg = new GlobalRouterStatistics(ui->stackPages),
                    action = createPageAction(QIcon(IMAGE_GLOBALROUTER), tr("Global Router"), grp)); 
                    
+  ui->stackPages->add(gxsdlg = new GxsTransportStatistics(ui->stackPages),
+                   action = createPageAction(QIcon(IMAGE_GXSTRANSPORT), tr("Gxs Transport"), grp));
+
   ui->stackPages->add(rttdlg = new RttStatistics(ui->stackPages),
                       action = createPageAction(QIcon(IMAGE_RTT), tr("RTT Statistics"), grp));
                    
@@ -179,7 +190,7 @@ QAction *StatisticsWindow::createPageAction(const QIcon &icon, const QString &te
     font = action->font();
     font.setPointSize(9);
     action->setCheckable(true);
-    action->setFont(font);
+//    action->setFont(font);
     return action;
 }
 

retroshare-gui/src/gui/statistics/StatisticsWindow.h

@@ -37,6 +37,7 @@ class DhtWindow;
 class BwCtrlWindow;
 class TurtleRouterStatistics;
 class GlobalRouterStatistics;
+class GxsTransportStatistics;
 class RttStatistics;
 
 class StatisticsWindow : public QMainWindow {
@@ -53,6 +54,7 @@ public:
 
   DhtWindow *dhtw;
   GlobalRouterStatistics *grsdlg;
+  GxsTransportStatistics *gxsdlg;
   BwCtrlWindow *bwdlg;
   TurtleRouterStatistics *trsdlg;
   RttStatistics *rttdlg;

retroshare-gui/src/retroshare-gui.pro

@@ -374,6 +374,7 @@ HEADERS +=  rshare.h \
             gui/statistics/BandwidthStatsWidget.h \
             gui/statistics/DhtWindow.h \
             gui/statistics/GlobalRouterStatistics.h \
+            gui/statistics/GxsTransportStatistics.h \
             gui/statistics/StatisticsWindow.h \
             gui/statistics/BwCtrlWindow.h \
             gui/statistics/RttStatistics.h \
@@ -680,6 +681,7 @@ FORMS +=    gui/StartDialog.ui \
             gui/statistics/TurtleRouterDialog.ui \
             gui/statistics/TurtleRouterStatistics.ui \
             gui/statistics/GlobalRouterStatistics.ui \
+            gui/statistics/GxsTransportStatistics.ui \
             gui/statistics/StatisticsWindow.ui \
             gui/statistics/BwCtrlWindow.ui \
             gui/statistics/RttStatistics.ui \
@@ -917,6 +919,7 @@ SOURCES +=  main.cpp \
             gui/statistics/TurtleRouterDialog.cpp \
             gui/statistics/TurtleRouterStatistics.cpp \
             gui/statistics/GlobalRouterStatistics.cpp \
+            gui/statistics/GxsTransportStatistics.cpp \
             gui/statistics/StatisticsWindow.cpp \
             gui/statistics/BwCtrlWindow.cpp \
             gui/statistics/RttStatistics.cpp \

retroshare.pri

@@ -65,6 +65,18 @@ rs_nodeprecatedwarning:CONFIG -= no_rs_nodeprecatedwarning
 CONFIG *= no_rs_nocppwarning
 rs_nocppwarning:CONFIG -= no_rs_nocppwarning
 
+# To disable GXS mail append the following assignation to qmake command line
+# "CONFIG+=no_rs_gxs_trans"
+CONFIG *= rs_gxs_trans
+#no_rs_gxs_trans:CONFIG -= rs_gxs_trans ## Disabing not supported ATM
+
+# To enable GXS based async chat append the following assignation to qmake
+# command line "CONFIG+=rs_async_chat"
+CONFIG *= no_rs_async_chat
+rs_async_chat:CONFIG -= no_rs_async_chat
+
+
+
 unix {
 	isEmpty(PREFIX)   { PREFIX   = "/usr" }
 	isEmpty(BIN_DIR)  { BIN_DIR  = "$${PREFIX}/bin" }
@@ -173,13 +185,24 @@ rs_nodeprecatedwarning {
     QMAKE_CXXFLAGS += -Wno-deprecated
     QMAKE_CXXFLAGS += -Wno-deprecated-declarations
     DEFINES *= RS_NO_WARN_DEPRECATED
-    warning("QMAKE: You have disable deprecated warnings.")
+    warning("QMAKE: You have disabled deprecated warnings.")
 }
 
 rs_nocppwarning {
     QMAKE_CXXFLAGS += -Wno-cpp
     DEFINES *= RS_NO_WARN_CPP
-    warning("QMAKE: You have disable cpp warnings.")
+    warning("QMAKE: You have disabled C preprocessor warnings.")
 }
 
-rs_gxs_mail:DEFINES *= RS_GXS_MAIL
+rs_gxs_trans {
+    DEFINES *= RS_GXS_TRANS
+    greaterThan(QT_MAJOR_VERSION, 4) {
+        CONFIG += c++11
+    } else {
+        QMAKE_CXXFLAGS += -std=c++0x
+    }
+}
+
+rs_async_chat {
+    DEFINES *= RS_ASYNC_CHAT
+}