Merge remote-tracking branch 'RetroShare/master' into pr_rsconfig

libretroshare/src/chat/distributedchat.cc

@@ -219,9 +219,16 @@ bool DistributedChatService::checkSignature(RsChatLobbyBouncingObject *obj,const
 
     // network pre-request key to allow message authentication.
 
-    mGixs->requestKey(obj->signature.keyId,peer_list,RsIdentityUsage(RS_SERVICE_TYPE_CHAT,RsIdentityUsage::CHAT_LOBBY_MSG_VALIDATION,RsGxsGroupId(),RsGxsMessageId(),obj->lobby_id));
+    mGixs->requestKey(obj->signature.keyId,peer_list,RsIdentityUsage(RsServiceType::CHAT,
+                                                                     RsIdentityUsage::CHAT_LOBBY_MSG_VALIDATION,
+                                                                     RsGxsGroupId(),
+                                                                     RsGxsMessageId(),
+                                                                     RsGxsMessageId(),
+                                                                     RsGxsMessageId(),
+                                                                     obj->lobby_id));
 
-    uint32_t size = RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).size(dynamic_cast<RsItem*>(obj)) ;
+    uint32_t size = RsChatSerialiser(RsSerializationFlags::SIGNATURE)
+            .size(dynamic_cast<RsItem*>(obj));
     RsTemporaryMemory memory(size) ;
 
 #ifdef DEBUG_CHAT_LOBBIES
@@ -229,14 +236,21 @@ bool DistributedChatService::checkSignature(RsChatLobbyBouncingObject *obj,const
     std::cerr << "   signature id: " << obj->signature.keyId << std::endl;
 #endif
 
-    if(!RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).serialise(dynamic_cast<RsItem*>(obj),memory,&size))
+	if( !RsChatSerialiser(RsSerializationFlags::SIGNATURE)
+	        .serialise(dynamic_cast<RsItem*>(obj),memory,&size) )
     {
 	    std::cerr << "  (EE) Cannot serialise message item. " << std::endl;
 	    return false ;
     }
 
     uint32_t error_status ;
-    RsIdentityUsage use_info(RS_SERVICE_TYPE_CHAT,RsIdentityUsage::CHAT_LOBBY_MSG_VALIDATION,RsGxsGroupId(),RsGxsMessageId(),obj->lobby_id) ;
+    RsIdentityUsage use_info(RsServiceType::CHAT,
+                             RsIdentityUsage::CHAT_LOBBY_MSG_VALIDATION,
+                             RsGxsGroupId(),
+                             RsGxsMessageId(),
+                             RsGxsMessageId(),
+                             RsGxsMessageId(),
+                             obj->lobby_id) ;
 
     if(!mGixs->validateData(memory,size,obj->signature,false,use_info,error_status))
     {
@@ -1003,10 +1017,12 @@ bool DistributedChatService::locked_initLobbyBouncableObject(const ChatLobbyId&
 
     // now sign the object, if the lobby expects it
 
-        uint32_t size = RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).size(dynamic_cast<RsItem*>(&item)) ;
+	uint32_t size = RsChatSerialiser(RsSerializationFlags::SIGNATURE)
+	        .size(dynamic_cast<RsItem*>(&item));
         RsTemporaryMemory memory(size) ;
 
-        if(!RsChatSerialiser(RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE).serialise(dynamic_cast<RsItem*>(&item),memory,&size))
+	if( !RsChatSerialiser(RsSerializationFlags::SIGNATURE)
+	        .serialise(dynamic_cast<RsItem*>(&item),memory,&size) )
         {
             std::cerr << "(EE) Cannot sign message item. " << std::endl;
             return false ;
@@ -1600,6 +1616,12 @@ ChatLobbyId DistributedChatService::createChatLobby(const std::string& lobby_nam
 #endif
 	ChatLobbyId lobby_id ;
 	{
+		if (!rsIdentity->isOwnId(lobby_identity))
+		{
+			RsErr() << __PRETTY_FUNCTION__ << " lobby_identity RsGxsId id must be own" << std::endl;
+			return 0;
+		}
+
 		RsStackMutex stack(mDistributedChatMtx); /********** STACK LOCKED MTX ******/
 
 		// create a unique id.

libretroshare/src/chat/p3chatservice.cc

@@ -722,6 +722,23 @@ bool p3ChatService::initiateDistantChatConnexion( const RsGxsId& to_gxs_id,
                                                   uint32_t& error_code,
                                                   bool notify )
 {
+
+	if(to_gxs_id.isNull())
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " Destination RsGxsId is invalid" << std::endl;
+		return false;
+	}
+	if (from_gxs_id.isNull())
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " Origin RsGxsId is invalid" << std::endl;
+		return false;
+	}
+	if (!rsIdentity->isOwnId(from_gxs_id))
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " Origin RsGxsId id must be own" << std::endl;
+		return false;
+	}
+
 	if(DistantChatService::initiateDistantChatConnexion( to_gxs_id,
 	                                                     from_gxs_id, pid,
 	                                                     error_code, notify ))
@@ -881,6 +898,13 @@ bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *& ci)
     RsServer::notify()->notifyChatMessage(cm);
     
 	mHistoryMgr->addMessage(cm);
+
+	if(rsEvents)
+	{
+		auto ev = std::make_shared<RsChatMessageEvent>();
+		ev->mChatMessage = cm;
+		rsEvents->postEvent(ev);
+	}
     
     return true ;
 }

libretroshare/src/chat/p3chatservice.h

@@ -42,6 +42,22 @@ class p3HistoryMgr;
 
 typedef RsPeerId ChatLobbyVirtualPeerId ;
 
+struct RsChatMessageEvent : RsEvent
+{
+	RsChatMessageEvent() : RsEvent(RsEventType::CHAT_MESSAGE) {}
+	~RsChatMessageEvent() override = default;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mChatMessage);
+	}
+
+	ChatMessage mChatMessage;
+};
+
 //!The basic Chat service.
  /**
   *

libretroshare/src/chat/rschatitems.cc

@@ -70,23 +70,14 @@ void RsChatMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGeneric
 
 /*************************************************************************/
 
-RsChatAvatarItem::~RsChatAvatarItem()
-{
-	if(image_data != NULL)
-	{
-		free(image_data) ;
-		image_data = NULL ;
-	}
-}
-
 void RsChatLobbyBouncingObject::serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process(j,ctx,lobby_id,"lobby_id") ;
     RsTypeSerializer::serial_process(j,ctx,msg_id  ,"msg_id") ;
     RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_NAME,nick,"nick") ;
 
-    if(!(ctx.mFlags & RsServiceSerializer::SERIALIZATION_FLAG_SIGNATURE))
-    	RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,signature,"signature") ;
+	if(!(ctx.mFlags & RsSerializationFlags::SIGNATURE))
+		RS_SERIAL_PROCESS(signature);
 }
 
 void RsChatLobbyMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)

libretroshare/src/chat/rschatitems.h

@@ -99,8 +99,6 @@ class RsChatItem: public RsItem
 		}
 
 		virtual ~RsChatItem() {}
-		virtual std::ostream& print(std::ostream &out, uint16_t /*indent*/ = 0) { return out; }	// derived from RsItem, but should be removed
-
         virtual void clear() {}
 };
 
@@ -348,14 +346,20 @@ class RsChatStatusItem: public RsChatItem
 //
 class RsChatAvatarItem: public RsChatItem
 {
-	public:
-		RsChatAvatarItem() :RsChatItem(RS_PKT_SUBTYPE_CHAT_AVATAR) {setPriorityLevel(QOS_PRIORITY_RS_CHAT_AVATAR_ITEM) ;}
+public:
+	RsChatAvatarItem():
+	    RsChatItem(RS_PKT_SUBTYPE_CHAT_AVATAR),
+	    image_size(0), image_data(nullptr)
+	{ setPriorityLevel(QOS_PRIORITY_RS_CHAT_AVATAR_ITEM); }
 
-		virtual ~RsChatAvatarItem() ;
-		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+	~RsChatAvatarItem() override { free(image_data); }
 
-		uint32_t image_size ;				// size of data in bytes
-		unsigned char *image_data ;		// image
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx) override;
+
+	uint32_t image_size; /// size of data in bytes
+	unsigned char* image_data ; /// image data
 };
 
 
@@ -371,9 +375,8 @@ struct PrivateOugoingMapItem : RsChatItem
 
 struct RsChatSerialiser : RsServiceSerializer
 {
-	RsChatSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE) :
-	    RsServiceSerializer( RS_SERVICE_TYPE_CHAT,
-	                         RsGenericSerializer::FORMAT_BINARY, flags ) {}
+	RsChatSerialiser(RsSerializationFlags flags = RsSerializationFlags::NONE):
+	    RsServiceSerializer(RS_SERVICE_TYPE_CHAT, flags) {}
 
 	virtual RsItem *create_item(uint16_t service_id,uint8_t item_sub_id) const;
 };

libretroshare/src/file_sharing/file_tree.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Retroshare Team <retroshare.project@gmail.com>            *
+ * Copyright (C) 2018  Retroshare Team <contact@retroshare.cc>                 *
+ * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -20,40 +22,100 @@
  *                                                                             *
  ******************************************************************************/
 #include <iomanip>
-#include <util/radix64.h>
-#include <util/rsdir.h>
 
+#include "util/radix64.h"
+#include "util/rsbase64.h"
+#include "util/rsdir.h"
+#include "retroshare/rsfiles.h"
 #include "file_sharing_defaults.h"
 #include "filelist_io.h"
-#include "file_tree.h"
+#include "serialiser/rstypeserializer.h"
 
-std::string FileTreeImpl::toRadix64() const
+void RsFileTree::DirData::serial_process(
+            RsGenericSerializer::SerializeJob j,
+            RsGenericSerializer::SerializeContext& ctx )
 {
-	unsigned char *buff = NULL ;
-	uint32_t size = 0 ;
-
-	serialise(buff,size) ;
-
-	std::string res ;
-
-	Radix64::encode(buff,size,res) ;
-
-	free(buff) ;
-	return res ;
+	RS_SERIAL_PROCESS(name);
+	RS_SERIAL_PROCESS(subdirs);
+	RS_SERIAL_PROCESS(subfiles);
 }
 
-FileTree *FileTree::create(const std::string& radix64_string)
+void RsFileTree::FileData::serial_process(
+            RsGenericSerializer::SerializeJob j,
+            RsGenericSerializer::SerializeContext& ctx )
 {
-	FileTreeImpl *ft = new FileTreeImpl ;
+	RS_SERIAL_PROCESS(name);
+	RS_SERIAL_PROCESS(size);
+	RS_SERIAL_PROCESS(hash);
+}
 
+/*static*/ std::tuple<std::unique_ptr<RsFileTree>, std::error_condition>
+RsFileTree::fromBase64(const std::string& base64)
+{
+	const auto failure = [](std::error_condition ec)
+	{ return std::make_tuple(nullptr, ec); };
+
+	if(base64.empty()) return failure(std::errc::invalid_argument);
+
+	std::error_condition ec;
+	std::vector<uint8_t> mem;
+	if( (ec = RsBase64::decode(base64, mem)) ) return failure(ec);
+
+	RsGenericSerializer::SerializeContext ctx(
+	            mem.data(), static_cast<uint32_t>(mem.size()),
+	            RsSerializationFlags::INTEGER_VLQ );
+	std::unique_ptr<RsFileTree> ft(new RsFileTree);
+	ft->serial_process(
+	            RsGenericSerializer::SerializeJob::DESERIALIZE, ctx);
+	if(ctx.mOk) return std::make_tuple(std::move(ft), std::error_condition());
+
+	return failure(std::errc::invalid_argument);
+}
+
+std::string RsFileTree::toBase64() const
+{
+	RsGenericSerializer::SerializeContext ctx;
+	ctx.mFlags = RsSerializationFlags::INTEGER_VLQ;
+	RsFileTree* ncThis = const_cast<RsFileTree*>(this);
+	ncThis->serial_process(
+	            RsGenericSerializer::SerializeJob::SIZE_ESTIMATE, ctx );
+
+	std::vector<uint8_t> buf(ctx.mOffset);
+	ctx.mSize = ctx.mOffset; ctx.mOffset = 0; ctx.mData = buf.data();
+
+	ncThis->serial_process(
+	            RsGenericSerializer::SerializeJob::SERIALIZE, ctx );
+	std::string result;
+	RsBase64::encode(ctx.mData, ctx.mSize, result, false, true);
+	return result;
+}
+
+std::string RsFileTree::toRadix64() const
+{
+	unsigned char* buff = nullptr;
+	uint32_t size = 0;
+	serialise(buff, size);
+	std::string res;
+	Radix64::encode(buff,size,res);
+	free(buff);
+	return res;
+}
+
+std::unique_ptr<RsFileTree> RsFileTree::fromRadix64(
+        const std::string& radix64_string )
+{
+	std::unique_ptr<RsFileTree> ft(new RsFileTree);
 	std::vector<uint8_t> mem = Radix64::decode(radix64_string);
-	ft->deserialise(mem.data(),mem.size()) ;
-
-	return ft ;
+	if(ft->deserialise(mem.data(), static_cast<uint32_t>(mem.size())))
+		return ft;
+	return nullptr;
 }
 
-void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const DirDetails& dd,bool remote,bool remove_top_dirs)
+void RsFileTree::recurs_buildFileTree(
+        RsFileTree& ft, uint32_t index, const DirDetails& dd, bool remote,
+        bool remove_top_dirs )
 {
+	RsDbg() << __PRETTY_FUNCTION__ << " index: " << index << std::endl;
 	if(ft.mDirs.size() <= index)
 		ft.mDirs.resize(index+1) ;
 
@@ -67,14 +129,14 @@ void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const Di
 
 	DirDetails dd2 ;
 
-	FileSearchFlags flags = remote?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL ;
+	FileSearchFlags flags = remote ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL;
 
 	for(uint32_t i=0;i<dd.children.size();++i)
 		if(rsFiles->RequestDirDetails(dd.children[i].ref,dd2,flags))
 		{
 			if(dd.children[i].type == DIR_TYPE_FILE)
 			{
-				FileTree::FileData f ;
+				FileData f ;
 				f.name = dd2.name ;
 				f.size = dd2.count ;
 				f.hash = dd2.hash ;
@@ -97,33 +159,50 @@ void FileTreeImpl::recurs_buildFileTree(FileTreeImpl& ft,uint32_t index,const Di
 			std::cerr << "(EE) Cannot request dir details for pointer " << dd.children[i].ref << std::endl;
 }
 
-bool FileTreeImpl::getDirectoryContent(uint32_t index,std::string& name,std::vector<uint32_t>& subdirs,std::vector<FileData>& subfiles) const
+bool RsFileTree::getDirectoryContent(
+        std::string& name, std::vector<uint64_t>& subdirs,
+        std::vector<FileData>& subfiles, uint64_t index_p ) const
 {
-	if(index >= mDirs.size())
-		return false ;
+	// Avoid warnings on Android armv7
+	using sz_t = std::vector<FileData>::size_type;
+	sz_t index = static_cast<sz_t>(index_p);
+
+	if(index >= mDirs.size()) return false;
 
 	name = mDirs[index].name;
 	subdirs = mDirs[index].subdirs ;
 
 	subfiles.clear() ;
-	for(uint32_t i=0;i<mDirs[index].subfiles.size();++i)
-		subfiles.push_back(mFiles[mDirs[index].subfiles[i]]);
+	for(sz_t i=0; i < mDirs[index].subfiles.size(); ++i)
+		subfiles.push_back(mFiles[static_cast<sz_t>(mDirs[index].subfiles[i])]);
 
-	return true ;
+	return true;
 }
 
-FileTree *FileTree::create(const DirDetails& dd, bool remote,bool remove_top_dirs)
+std::unique_ptr<RsFileTree> RsFileTree::fromDirDetails(
+        const DirDetails& dd, bool remote ,bool remove_top_dirs )
 {
-	FileTreeImpl *ft = new FileTreeImpl ;
+	std::unique_ptr<RsFileTree>ft(new RsFileTree);
+	if(dd.type == DIR_TYPE_FILE)
+	{
+		FileData fd;
+		fd.name = dd.name; fd.hash = dd.hash; fd.size = dd.count;
+		ft->mFiles.push_back(fd);
+		ft->mTotalFiles = 1;
+		ft->mTotalSize = fd.size;
 
-	FileTreeImpl::recurs_buildFileTree(*ft,0,dd,remote,remove_top_dirs) ;
-
-	return ft ;
+		DirData dd;
+		dd.name = "/";
+		dd.subfiles.push_back(0);
+		ft->mDirs.push_back(dd);
+	}
+	else recurs_buildFileTree(*ft, 0, dd, remote, remove_top_dirs );
+	return ft;
 }
 
 typedef FileListIO::read_error read_error ;
 
-bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size)
+bool RsFileTree::deserialise(unsigned char *buffer,uint32_t buffer_size)
 {
     uint32_t buffer_offset = 0 ;
 
@@ -218,7 +297,7 @@ bool FileTreeImpl::deserialise(unsigned char *buffer,uint32_t buffer_size)
 	return true ;
 }
 
-bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) const
+bool RsFileTree::serialise(unsigned char *& buffer,uint32_t& buffer_size) const
 {
 	buffer = 0 ;
     uint32_t buffer_offset = 0 ;
@@ -234,7 +313,11 @@ bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) cons
     {
         // Write some header
 
-        if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,(uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001)) throw std::runtime_error("Write error") ;
+		if(!FileListIO::writeField(
+		            buffer, buffer_size, buffer_offset,
+		            FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,
+		            (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_TREE_VERSION_0001 ) )
+			throw std::runtime_error("Write error") ;
         if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mFiles.size())) throw std::runtime_error("Write error") ;
         if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t) mDirs.size())) throw std::runtime_error("Write error") ;
 
@@ -292,28 +375,4 @@ bool FileTreeImpl::serialise(unsigned char *& buffer,uint32_t& buffer_size) cons
     }
 }
 
-void FileTreeImpl::print() const
-{
-	std::cerr << "File hierarchy: name=" << mDirs[0].name << " size=" << mTotalSize << std::endl;
-	recurs_print(0,"  ") ;
-}
-
-void FileTreeImpl::recurs_print(uint32_t index,const std::string& indent) const
-{
-	if(index >= mDirs.size())
-	{
-		std::cerr << "(EE) inconsistent FileTree structure" << std::endl;
-		return;
-	}
-	std::cerr << indent << mDirs[index].name << std::endl;
-
-	for(uint32_t i=0;i<mDirs[index].subdirs.size();++i)
-		recurs_print(mDirs[index].subdirs[i],indent+"  ") ;
-
-	for(uint32_t i=0;i<mDirs[index].subfiles.size();++i)
-	{
-		const FileData& fd(mFiles[mDirs[index].subfiles[i]]) ;
-
-		std::cerr << indent << "  " << fd.hash << " " << std::setprecision(8) << fd.size << " " << fd.name << std::endl;
-	}
-}
+RsFileTree::~RsFileTree() = default;

libretroshare/src/file_sharing/p3filelists.cc

@@ -1424,9 +1424,15 @@ int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::lis
     return !results.empty() ;
 
 }
-bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const
+
+bool p3FileDatabase::search(
+        const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const
 {
-    RS_STACK_MUTEX(mFLSMtx) ;
+	RS_STACK_MUTEX(mFLSMtx);
+
+	if( (hintflags & RS_FILE_HINTS_EXTRA) &&
+	        mExtraFiles->search(hash, hintflags, info) )
+		return true;
 
     if(hintflags & RS_FILE_HINTS_LOCAL)
     {

libretroshare/src/ft/ftcontroller.cc

@@ -38,7 +38,7 @@
 #include "util/rsdiscspace.h"
 #include "util/rsmemory.h"
 #include "util/rstime.h"
-
+#include "util/cxx17retrocompat.h"
 #include "ft/ftcontroller.h"
 
 #include "ft/ftfilecreator.h"
@@ -923,10 +923,15 @@ bool ftController::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
 	return false ;
 }
 
-bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash,
-			uint64_t size, const std::string& dest, TransferRequestFlags flags,
-                                                                const std::list<RsPeerId> &_srcIds, uint16_t state)
+bool ftController::FileRequest(
+        const std::string& fname, const RsFileHash& hash, uint64_t size,
+        const std::string& dest, TransferRequestFlags flags,
+        const std::list<RsPeerId> &_srcIds, uint16_t state )
 {
+	/* TODO: To support collections faithfully we need to be able to save
+	 * the same file to multiple locations, both if already downloaded or
+	 * incomplete */
+
     std::list<RsPeerId> srcIds(_srcIds) ;
 
 	/* check if we have the file */
@@ -965,8 +970,9 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 
 		// create void file with the target name.
 		FILE *f = RsDirUtil::rs_fopen(destination.c_str(),"w") ;
-		if(f == NULL)
-			std::cerr << "Could not open file " << destination << " for writting." << std::endl ;
+		if(!f)
+			RsErr() << __PRETTY_FUNCTION__ << " Could not write file "
+			        << destination << std::endl;
 		else
 			fclose(f) ;
 
@@ -979,12 +985,13 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 	//
 	if(size >= 1024ull*1024ull*((1ull << 32) - 1))
 	{
-		std::cerr << "FileRequest Error: unexpected size. This is probably a bug." << std::endl;
-		std::cerr << "  name  = " << fname << std::endl ;
-		std::cerr << "  flags = " << flags << std::endl ;
-		std::cerr << "  dest  = " << dest << std::endl ;
-		std::cerr << "  size  = " << size << std::endl ;
-		return false ;
+		RsErr() << __PRETTY_FUNCTION__
+		        << " unexpected size. This is probably a bug."
+		        << " name  = " << fname
+		        << " flags = " << flags
+		        << " dest  = " << dest
+		        << " size  = " << size << std::endl;
+		return false;
 	}
 
 	/* If file transfer is not enabled ....
@@ -1004,8 +1011,8 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 		}
 	}
 
-	// remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files.
-	//
+	/* remove the sources from the list, if they don't have clearance for direct
+	 * transfer. This happens only for non cache files. */
 	for(std::list<RsPeerId>::iterator it = srcIds.begin(); it != srcIds.end(); )
 	{
 		bool bAllowDirectDL = false;
@@ -1052,11 +1059,10 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 	 * This is important as some guis request duplicate files regularly.
 	 */
 
-	{ 
-		RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
-
-        std::map<RsFileHash, ftFileControl*>::const_iterator dit = mDownloads.find(hash);
+	{
+		RS_STACK_MUTEX(ctrlMutex);
 
+		auto dit = std::as_const(mDownloads).find(hash);
 		if (dit != mDownloads.end())
 		{
 			/* we already have it! */
@@ -1110,7 +1116,7 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 
 			return true;
 		}
-	} /******* UNLOCKED ********/
+	} // RS_STACK_MUTEX(ctrlMutex); unlocked
 
 
 	if(mSearch && !(flags & RS_FILE_REQ_NO_SEARCH))

libretroshare/src/ft/ftextralist.cc

@@ -4,7 +4,7 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2008  Robert Fernie <retroshare@lunamutt.com>                 *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -21,6 +21,9 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <limits>
+#include <system_error>
+
 #ifdef WINDOWS_SYS
 #include "util/rswin.h"
 #endif
@@ -245,12 +248,8 @@ bool	ftExtraList::cleanupOldFiles()
 		/* remove items */
 		for(std::list<RsFileHash>::iterator rit = toRemove.begin(); rit != toRemove.end(); ++rit)
         {
-			if (mFiles.end() != (it = mFiles.find(*rit)))
-			{
-				cleanupEntry(it->second.info.path, it->second.info.transfer_info_flags);
-				mFiles.erase(it);
-			}
-			mHashOfHash.erase(makeEncryptedHash(*rit)) ;
+			if (mFiles.end() != (it = mFiles.find(*rit))) mFiles.erase(it);
+			mHashOfHash.erase(makeEncryptedHash(*rit));
         }
 
 		IndicateConfigChanged();
@@ -258,46 +257,39 @@ bool	ftExtraList::cleanupOldFiles()
 	return true;
 }
 
-
-bool	ftExtraList::cleanupEntry(std::string /*path*/, TransferRequestFlags /*flags*/)
-{
-//	if (flags & RS_FILE_CONFIG_CLEANUP_DELETE)
-//	{
-//		/* Delete the file? - not yet! */
-//	}
-	return true;
-}
-
-		/***
-		 * Hash file, and add to the files, 
-		 * file is removed after period.
-		 **/
-
 bool ftExtraList::hashExtraFile(
         std::string path, uint32_t period, TransferRequestFlags flags )
 {
-#ifdef  DEBUG_ELIST
-	std::cerr << "ftExtraList::hashExtraFile() path: " << path;
-	std::cerr << " period: " << period;
-	std::cerr << " flags: " << flags;
+	constexpr rstime_t max_int = std::numeric_limits<int>::max();
+	const rstime_t now = time(nullptr);
+	const rstime_t timeOut = now + period;
 
-	std::cerr << std::endl;
-#endif
-
-	auto failure = [](std::string errMsg)
+	if(timeOut > max_int)
 	{
-		RsErr() << __PRETTY_FUNCTION__ << " " << errMsg << std::endl;
+		/* Under the hood period is stored as int FileInfo::age so we do this
+		 * check here to detect 2038 year problem
+		 * https://en.wikipedia.org/wiki/Year_2038_problem */
+		RsErr() << __PRETTY_FUNCTION__ << " period: " << period  << " > "
+		        << max_int - now << std::errc::value_too_large << std::endl;
 		return false;
-	};
+	}
 
 	if(!RsDirUtil::fileExists(path))
-		return failure("file: " + path + "not found");
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " path: " << path
+		        << std::errc::no_such_file_or_directory << std::endl;
+		return false;
+	}
 
 	if(RsDirUtil::checkDirectory(path))
-		return failure("Cannot add a directory: " + path + "as extra file");
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " path: " << path
+		        << std::errc::is_a_directory << std::endl;
+		return false;
+	}
 
 	FileDetails details(path, period, flags);
-	details.info.age = static_cast<int>(time(nullptr) + period);
+	details.info.age = static_cast<int>(timeOut);
 
 	{
 		RS_STACK_MUTEX(extMutex);
@@ -492,8 +484,7 @@ bool    ftExtraList::loadList(std::list<RsItem *>& load)
 
 		if (ts > (rstime_t)fi->file.age)
 		{
-			/* to old */
-			cleanupEntry(fi->file.path, TransferRequestFlags(fi->flags));
+			/* too old */
 			delete (*it);
 			continue ;
 		}

libretroshare/src/ft/ftextralist.h

@@ -60,7 +60,7 @@
 #include "pqi/p3cfgmgr.h"
 #include "util/rstime.h"
 
-class FileDetails
+class RS_DEPRECATED_FOR(FileInfo) FileDetails
 {
 	public:
 		FileDetails()
@@ -130,7 +130,11 @@ public:
 		 * file is removed after period.
 		 **/
 
-	bool 		hashExtraFile(std::string path, uint32_t period, TransferRequestFlags flags);
+	/**
+	 * Hash file, and add to the files, file is removed after period.
+	 */
+	bool hashExtraFile(
+	        std::string path, uint32_t period, TransferRequestFlags flags );
 	bool	 	hashExtraFileDone(std::string path, FileInfo &info);
 
 	/***
@@ -165,7 +169,6 @@ private:
 	/* Worker Functions */
 	void	hashAFile();
 	bool	cleanupOldFiles();
-	bool    cleanupEntry(std::string path, TransferRequestFlags flags);
 
 	mutable RsMutex extMutex;
 

libretroshare/src/ft/ftserver.cc

@@ -20,6 +20,11 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <system_error>
+
 #include "crypto/chacha20.h"
 //const int ftserverzone = 29539;
 
@@ -40,18 +45,16 @@
 #include "retroshare/rstypes.h"
 #include "retroshare/rspeers.h"
 #include "retroshare/rsinit.h"
-
+#include "util/cxx17retrocompat.h"
 #include "rsitems/rsfiletransferitems.h"
 #include "rsitems/rsserviceids.h"
-
+#include "util/rsmemory.h"
 #include "rsserver/p3face.h"
 #include "turtle/p3turtle.h"
-
+#include "util/rsurl.h"
 #include "util/rsdebug.h"
 #include "util/rsdir.h"
 #include "util/rsprint.h"
-
-#include <iostream>
 #include "util/rstime.h"
 
 #ifdef RS_DEEP_FILES_INDEX
@@ -66,6 +69,18 @@
 #define FTSERVER_DEBUG() std::cerr << time(NULL) << " : FILE_SERVER : " << __FUNCTION__ << " : "
 #define FTSERVER_ERROR() std::cerr << "(EE) FILE_SERVER ERROR : "
 
+/*static*/ const RsFilesErrorCategory RsFilesErrorCategory::instance;
+
+/*static*/ const std::string RsFiles::DEFAULT_FILES_BASE_URL =
+        "retroshare:///files";
+/*static*/ const std::string RsFiles::FILES_URL_COUNT_FIELD = "filesCount";
+/*static*/ const std::string RsFiles::FILES_URL_DATA_FIELD = "filesData";
+// Use a 5 character word so there is no mistake it isn't base64 as 5%4=1
+/*static*/ const std::string RsFiles::FILES_URL_FAGMENT_FORWARD = "fragm";
+/*static*/ const std::string RsFiles::FILES_URL_NAME_FIELD = "filesName";
+/*static*/ const std::string RsFiles::FILES_URL_SIZE_FIELD = "filesSize";
+
+
 static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ;           // low priority tasks handling every 5 seconds
 static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most.
 
@@ -280,10 +295,14 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re
 
 bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
 {
-	return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info);
+	return mFileDatabase->search(
+	            hash, RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL, info );
 }
 
-bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds)
+bool ftServer::FileRequest(
+        const std::string& fname, const RsFileHash& hash, uint64_t size,
+        const std::string& dest, TransferRequestFlags flags,
+        const std::list<RsPeerId>& srcIds )
 {
 #ifdef SERVER_DEBUG
 	FTSERVER_DEBUG() << "Requesting " << fname << std::endl ;
@@ -295,6 +314,93 @@ bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uin
 	return true ;
 }
 
+std::error_condition ftServer::requestFiles(
+        const RsFileTree& collection, const std::string& destPath,
+        const std::vector<RsPeerId>& srcIds, FileRequestFlags flags )
+{
+	constexpr auto fname = __PRETTY_FUNCTION__;
+	const auto dirsCount = collection.mDirs.size();
+	const auto filesCount = collection.mFiles.size();
+
+	Dbg2() << fname << " dirsCount: " << dirsCount
+	       << " filesCount: " << filesCount << std::endl;
+
+	if(!dirsCount)
+	{
+		RsErr() << fname << " Directories list empty in collection "
+		        << std::endl;
+		return std::errc::not_a_directory;
+	}
+
+	if(!filesCount)
+	{
+		RsErr() << fname << " Files list empty in collection " << std::endl;
+		return std::errc::invalid_argument;
+	}
+
+	if(filesCount != collection.mTotalFiles)
+	{
+		RsErr() << fname << " Files count mismatch" << std::endl;
+		return std::errc::invalid_argument;
+	}
+
+	std::string basePath = destPath.empty() ? getDownloadDirectory() : destPath;
+	// Track how many time a directory have been explored
+	std::vector<uint32_t> dirsSeenCnt(dirsCount, 0);
+	//                          <directory handle, parent path>
+	using StackEntry = std::tuple<uint64_t, std::string>;
+	std::deque<StackEntry> dStack = { std::make_tuple(0, basePath) };
+
+	const auto exploreDir = [&](const StackEntry& se)-> std::error_condition
+	{
+		uint64_t dirHandle; std::string parentPath;
+		std::tie(dirHandle, parentPath) = se;
+
+		const auto& dirData = collection.mDirs[dirHandle];
+		auto& seenTimes = dirsSeenCnt[dirHandle];
+		std::string dirPath = RsDirUtil::makePath(parentPath, dirData.name);
+
+		/* This check is not perfect but is cheap and interrupt loop exploration
+		 * before it becomes pathological */
+		if(seenTimes++ > dirsCount)
+		{
+			RsErr() << fname << " loop detected! dir: "
+			        << dirHandle << " \"" << dirPath
+			        << "\" explored too many times" << std::endl;
+			return std::errc::too_many_symbolic_link_levels;
+		}
+
+		for(auto fHandle: dirData.subfiles)
+		{
+			if(fHandle >= filesCount) return std::errc::argument_out_of_domain;
+
+			const RsFileTree::FileData& fData = collection.mFiles[fHandle];
+
+			bool fr =
+			FileRequest( fData.name, fData.hash, fData.size,
+			             dirPath,
+			             TransferRequestFlags::fromEFT<FileRequestFlags>(flags),
+			             std::list<RsPeerId>(srcIds.begin(), srcIds.end()) );
+
+			Dbg2() << fname << " requested: " << fr << " "
+			       << fData.hash << " -> " << dirPath << std::endl;
+		}
+
+		for(auto dHandle: dirData.subdirs)
+			dStack.push_back(std::make_tuple(dHandle, dirPath));
+
+		return std::error_condition();
+	};
+
+	while(!dStack.empty())
+	{
+		if(std::error_condition ec = exploreDir(dStack.front())) return ec;
+		dStack.pop_front();
+	}
+
+	return std::error_condition();
+}
+
 bool ftServer::activateTunnels(const RsFileHash& hash,uint32_t encryption_policy,TransferRequestFlags flags,bool onoff)
 {
 	RsFileHash hash_of_hash ;
@@ -716,6 +822,14 @@ bool ftServer::ExtraFileRemove(const RsFileHash& hash)
 bool ftServer::ExtraFileHash(
         std::string localpath, rstime_t period, TransferRequestFlags flags )
 {
+	constexpr rstime_t uintmax = std::numeric_limits<uint32_t>::max();
+	if(period > uintmax)
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " period: " << period << " > "
+		        << uintmax << std::errc::value_too_large << std::endl;
+		return false;
+	}
+
 	return mFtExtra->hashExtraFile(
 	            localpath, static_cast<uint32_t>(period), flags );
 }
@@ -740,7 +854,7 @@ bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFl
 }
 
 bool ftServer::requestDirDetails(
-        DirDetails &details, std::uintptr_t handle, FileSearchFlags flags )
+        DirDetails &details, uint64_t handle, FileSearchFlags flags )
 { return RequestDirDetails(reinterpret_cast<void*>(handle), details, flags); }
 
 int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags)
@@ -1872,15 +1986,16 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
 			hasCallback = true;
 
 			std::vector<TurtleFileInfoV2> cRes;
-			for( const auto& tfiold : item->result)
-				cRes.push_back(tfiold);
+			for(auto& res: std::as_const(item->result))
+				cRes.push_back(TurtleFileInfoV2(res));
 
 			cbpt->second.first(cRes);
 		}
 	} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
 
 	if(!hasCallback)
-		RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result );
+		RsServer::notify()->notifyTurtleSearchResult(
+		            item->PeerId(), item->request_id, item->result );
 }
 
 bool ftServer::receiveSearchRequest(
@@ -2101,3 +2216,144 @@ RsFileItem::RsFileItem(RsFileItemType subtype) :
 void RsFileSearchRequestItem::clear() { queryString.clear(); }
 
 void RsFileSearchResultItem::clear() { mResults.clear(); }
+
+
+std::error_condition RsFilesErrorCategory::default_error_condition(int ev)
+const noexcept
+{
+	switch(static_cast<RsFilesErrorNum>(ev))
+	{
+	case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND:
+		return std::errc::invalid_argument;
+	default:
+		return std::error_condition(ev, *this);
+	}
+}
+
+std::error_condition ftServer::dirDetailsToLink(
+        std::string& link,
+        const DirDetails& dirDetails, bool fragSneak, const std::string& baseUrl )
+{
+	std::unique_ptr<RsFileTree> tFileTree =
+	        RsFileTree::fromDirDetails(dirDetails, false);
+	if(!tFileTree) return std::errc::invalid_argument;
+
+	link = tFileTree->toBase64();
+
+	if(!baseUrl.empty())
+	{
+		RsUrl tUrl(baseUrl);
+		tUrl.setQueryKV(FILES_URL_COUNT_FIELD,
+		                std::to_string(tFileTree->mTotalFiles) )
+		    .setQueryKV(FILES_URL_NAME_FIELD, dirDetails.name)
+		    .setQueryKV( FILES_URL_SIZE_FIELD,
+		                 std::to_string(tFileTree->mTotalSize) );
+		if(fragSneak)
+			tUrl.setQueryKV(FILES_URL_DATA_FIELD, FILES_URL_FAGMENT_FORWARD)
+			        .setFragment(link);
+		else tUrl.setQueryKV(FILES_URL_DATA_FIELD, link);
+		link = tUrl.toString();
+	}
+
+	return std::error_condition();
+}
+
+std::error_condition ftServer::exportCollectionLink(
+        std::string& link, uint64_t handle, bool fragSneak,
+        const std::string& baseUrl )
+{
+	DirDetails tDirDet;
+	if(!requestDirDetails(tDirDet, handle))
+		return RsFilesErrorNum::FILES_HANDLE_NOT_FOUND;
+
+	return dirDetailsToLink(link, tDirDet, fragSneak, baseUrl);
+}
+
+/// @see RsFiles
+std::error_condition ftServer::exportFileLink(
+        std::string& link, const RsFileHash& fileHash, uint64_t fileSize,
+        const std::string& fileName, bool fragSneak, const std::string& baseUrl )
+{
+	if(fileHash.isNull() || !fileSize || fileName.empty())
+		return std::errc::invalid_argument;
+
+	DirDetails tDirDet;
+	tDirDet.type = DIR_TYPE_FILE;
+	tDirDet.name = fileName;
+	tDirDet.hash = fileHash;
+	tDirDet.count = fileSize;
+
+	return dirDetailsToLink(link, tDirDet, fragSneak, baseUrl);
+}
+
+std::error_condition ftServer::parseFilesLink(
+        const std::string& link, RsFileTree& collection )
+{
+	RsUrl tUrl(link);
+
+	{
+		/* Handle retrocompatibility with old stile single file links
+		 * retroshare://file?name=$FILE_NAME&size=$FILE_SIZE&hash=$FILE_HASH
+		 */
+		if( tUrl.scheme() == "retroshare" && tUrl.host() == "file"
+		        && tUrl.hasQueryK("name") && !tUrl.getQueryV("name")->empty()
+		        && tUrl.hasQueryK("size") && !tUrl.getQueryV("size")->empty()
+		        && tUrl.hasQueryK("hash") && !tUrl.getQueryV("hash")->empty() )
+		{
+			DirDetails dt;
+			dt.type = DIR_TYPE_FILE;
+			dt.hash = RsFileHash(*tUrl.getQueryV("hash"));
+			dt.name = *tUrl.getQueryV("name");
+			try
+			{
+				dt.count = std::stoull(*tUrl.getQueryV("size"));
+				std::unique_ptr<RsFileTree> ft;
+				if( !dt.hash.isNull() &&
+				        (ft = RsFileTree::fromDirDetails(dt, true)) )
+				{
+					collection = *ft;
+					return std::error_condition();
+				}
+			}
+			catch (...) {}
+		}
+	}
+
+	{
+		/* Handle retrocompatibility with old stile collection links
+		 * retroshare://collection?name=$NAME&size=$SIZE&radix=$RADIX&files=$COUNT
+		 */
+		if( tUrl.scheme() == "retroshare" && tUrl.host() == "collection"
+		        && tUrl.hasQueryK("name") && !tUrl.getQueryV("name")->empty()
+		        && tUrl.hasQueryK("size") && !tUrl.getQueryV("size")->empty()
+		        && tUrl.hasQueryK("radix") && !tUrl.getQueryV("radix")->empty()
+		        && tUrl.hasQueryK("files") && !tUrl.getQueryV("files")->empty() )
+		{
+			try
+			{
+				// Don't need the values just check they are valid numbers
+				std::stoull(*tUrl.getQueryV("size"));
+				std::stoull(*tUrl.getQueryV("files"));
+
+				auto ft = RsFileTree::fromRadix64(*tUrl.getQueryV("radix"));
+				if(ft)
+				{
+					collection = *ft;
+					return std::error_condition();
+				}
+			}
+			catch (...) {}
+		}
+	}
+
+	// Finaly handle the new files link format
+	rs_view_ptr<const std::string> radixPtr =
+	        tUrl.getQueryV(FILES_URL_DATA_FIELD);
+	if(!radixPtr) radixPtr = &link;
+	else if(*radixPtr == FILES_URL_FAGMENT_FORWARD) radixPtr = &tUrl.fragment();
+
+	std::unique_ptr<RsFileTree> tft; std::error_condition ec;
+	std::tie(tft, ec) = RsFileTree::fromBase64(*radixPtr);
+	if(tft) collection = *tft;
+	return ec;
+}

libretroshare/src/ft/ftserver.h

@@ -207,14 +207,39 @@ public:
 	virtual uint32_t filePermDirectDL() ;
 
 	/// @see RsFiles
-	virtual bool turtleSearchRequest(
+	std::error_condition requestFiles(
+	        const RsFileTree& collection,
+	        const std::string& destPath = "",
+	        const std::vector<RsPeerId>& srcIds = std::vector<RsPeerId>(),
+	        FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING
+	        ) override;
+
+	/// @see RsFiles
+	bool turtleSearchRequest(
 	        const std::string& matchString,
 	        const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
-	        rstime_t maxWait = 300 );
+	        rstime_t maxWait = 300 ) override;
 
 	virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
 	virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
 
+	/// @see RsFiles
+	std::error_condition exportCollectionLink(
+	        std::string& link, uint64_t handle, bool fragSneak = false,
+	        const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL
+	        ) override;
+
+	/// @see RsFiles
+	std::error_condition exportFileLink(
+	        std::string& link, const RsFileHash& fileHash, uint64_t fileSize,
+	        const std::string& fileName, bool fragSneak = false,
+	        const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL
+	        ) override;
+
+	/// @see RsFiles
+	std::error_condition parseFilesLink(
+	        const std::string& link, RsFileTree& collection ) override;
+
     /***
          * Control of Downloads Priority.
          ***/
@@ -254,7 +279,7 @@ public:
 
 	/// @see RsFiles::RequestDirDetails
 	virtual bool requestDirDetails(
-	        DirDetails &details, std::uintptr_t handle = 0,
+	        DirDetails &details, uint64_t handle = 0,
 	        FileSearchFlags flags = RS_FILE_HINTS_LOCAL );
 
     virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ;
@@ -365,11 +390,11 @@ protected:
     bool findEncryptedHash(const RsPeerId& virtual_peer_id, RsFileHash& encrypted_hash);
 
 	bool checkUploadLimit(const RsPeerId& pid,const RsFileHash& hash);
-private:
 
-    /**** INTERNAL FUNCTIONS ***/
-    //virtual int 	reScanDirs();
-    //virtual int 	check_dBUpdate();
+	std::error_condition dirDetailsToLink(
+	        std::string& link,
+	        const DirDetails& dirDetails, bool fragSneak,
+	        const std::string& baseUrl );
 
 private:
 

libretroshare/src/gossipdiscovery/gossipdiscoveryitems.cc

@@ -37,6 +37,7 @@ RsItem *RsDiscSerialiser::create_item(
 	{
 	case RsGossipDiscoveryItemType::PGP_LIST: return new RsDiscPgpListItem();
 	case RsGossipDiscoveryItemType::PGP_CERT_BINARY: return new RsDiscPgpKeyItem();
+	case RsGossipDiscoveryItemType::PGP_CERT: return new RsDiscPgpCertItem();	// deprecated, hanlde to suppress "unkown item" warning
 	case RsGossipDiscoveryItemType::CONTACT:  return new RsDiscContactItem();
 	case RsGossipDiscoveryItemType::IDENTITY_LIST: return new RsDiscIdentityListItem();
     default:
@@ -74,8 +75,8 @@ void RsDiscPgpKeyItem::clear()
 {
 	pgpKeyId.clear();
 	free(bin_data);
-    bin_data = nullptr;
-    bin_len=0;
+	bin_data = nullptr;
+	bin_len = 0;
 }
 
 void 	RsDiscContactItem::clear()

libretroshare/src/gossipdiscovery/gossipdiscoveryitems.h

@@ -84,17 +84,44 @@ class RsDiscPgpKeyItem: public RsDiscItem
 {
 public:
 
-	RsDiscPgpKeyItem() : RsDiscItem(RsGossipDiscoveryItemType::PGP_CERT_BINARY)
+	RsDiscPgpKeyItem() :
+	    RsDiscItem(RsGossipDiscoveryItemType::PGP_CERT_BINARY),
+	    bin_data(nullptr), bin_len(0)
 	{ setPriorityLevel(QOS_PRIORITY_RS_DISC_PGP_CERT); }
 
-    virtual ~RsDiscPgpKeyItem() { delete[](bin_data);bin_data=nullptr;bin_len=0;}
-
+	~RsDiscPgpKeyItem() override { free(bin_data); }
 	void clear() override;
-	void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx) override;
 
-	RsPgpId pgpKeyId;				// duplicate information for practical reasons
-    unsigned char *bin_data;					// binry key data allocated with new unsigned char[]
-    uint32_t bin_len;
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override;
+
+	/// duplicate information for practical reasons
+	RsPgpId pgpKeyId;
+
+	unsigned char* bin_data;
+	uint32_t bin_len;
+};
+
+class RS_DEPRECATED_FOR(RsDiscPgpKeyItem) RsDiscPgpCertItem: public RsDiscItem
+{
+public:
+    RsDiscPgpCertItem() : RsDiscItem(RsGossipDiscoveryItemType::PGP_CERT)
+    { setPriorityLevel(QOS_PRIORITY_RS_DISC_PGP_CERT); }
+
+    void clear() override
+    {
+        pgpId.clear();
+        pgpCert.clear();
+    }
+    void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx) override
+    {
+        RsTypeSerializer::serial_process(j,ctx,pgpId,"pgpId") ;
+        RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_PGPCERT,pgpCert,"pgpCert") ;
+    }
+
+    RsPgpId pgpId;
+    std::string pgpCert;
 };
 
 class RsDiscContactItem: public RsDiscItem

libretroshare/src/gossipdiscovery/p3gossipdiscovery.cc

@@ -108,14 +108,6 @@ p3discovery2::p3discovery2(
 
 	// Add self into PGP FriendList.
 	mFriendList[AuthGPG::getAuthGPG()->getGPGOwnId()] = DiscPgpInfo();
-
-	if(rsEvents)
-		rsEvents->registerEventsHandler(
-                    RsEventType::GOSSIP_DISCOVERY,
-		            [this](std::shared_ptr<const RsEvent> event)
-		{
-			rsEventsHandler(*event);
-		}, mRsEventsHandle ); // mRsEventsHandle is zeroed in initializer list
 }
 
 
@@ -260,6 +252,7 @@ int p3discovery2::handleIncoming()
 	{
 		RsDiscPgpListItem*      pgplist  = nullptr;
 		RsDiscPgpKeyItem*       pgpkey   = nullptr;
+		RsDiscPgpCertItem*      pgpcert	 = nullptr;	 // deprecated, hanlde for retro compability
 		RsDiscContactItem*      contact  = nullptr;
 		RsDiscIdentityListItem* gxsidlst = nullptr;
 
@@ -282,6 +275,9 @@ int p3discovery2::handleIncoming()
 		}
 		else if((pgpkey = dynamic_cast<RsDiscPgpKeyItem *>(item)) != nullptr)
 			recvPGPCertificate(item->PeerId(), pgpkey);
+		else if((pgpcert = dynamic_cast<RsDiscPgpCertItem *>(item)) != nullptr)
+			// sink
+			delete pgpcert;
 		else if((pgplist = dynamic_cast<RsDiscPgpListItem *>(item)) != nullptr)
 		{
 			if (pgplist->mode == RsGossipDiscoveryPgpListMode::FRIENDS)
@@ -392,7 +388,12 @@ void p3discovery2::recvOwnContactInfo(const RsPeerId &fromId, const RsDiscContac
 
 	setPeerVersion(fromId, item->version);
 
-	updatePeerAddresses(item);
+    // Hidden nodes do not need IP information. So that information is dropped.
+    // However, that doesn't mean hidden nodes do not know that information. Normally
+    // normal nodes should not send it, but old nodes still do.
+
+    if(!mPeerMgr->isHiddenNode(rsPeers->getOwnId()))
+        updatePeerAddresses(item);
 
     // if the peer is not validated, we stop the exchange here
 
@@ -454,7 +455,7 @@ void p3discovery2::recvIdentityList(const RsPeerId& pid,const std::list<RsGxsId>
     std::cerr << "p3discovery2::recvIdentityList(): from peer " << pid << ": " << ids.size() << " identities" << std::endl;
 #endif
 
-    RsIdentityUsage use_info(RS_SERVICE_TYPE_DISC,RsIdentityUsage::IDENTITY_DATA_UPDATE);
+    RsIdentityUsage use_info(RsServiceType::GOSSIP_DISCOVERY,RsIdentityUsage::IDENTITY_NEW_FROM_DISCOVERY);
 
 	for(auto it(ids.begin());it!=ids.end();++it)
     {
@@ -1280,11 +1281,6 @@ bool p3discovery2::setPeerVersion(const RsPeerId &peerId, const std::string &ver
 	return true;
 }
 
-void p3discovery2::rsEventsHandler(const RsEvent& event)
-{
-	Dbg3() << __PRETTY_FUNCTION__ << " " << static_cast<uint32_t>(event.mType) << std::endl;
-}
-
 
 /*************************************************************************************/
 /*			AuthGPGService						     */

libretroshare/src/grouter/grouteritems.cc

@@ -99,12 +99,10 @@ void RsGRouterGenericDataItem::serial_process(RsGenericSerializer::SerializeJob
     RsTypeSerializer::serial_process          (j,ctx,destination_key,"destination_key") ;
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,service_id,"service_id") ;
 
-    RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ;
+	RsTypeSerializer::RawMemoryWrapper prox(data_bytes, data_size);
+	RsTypeSerializer::serial_process(j, ctx, prox, "data");
 
-    RsTypeSerializer::serial_process(j,ctx,prox,"data") ;
-
-    if(ctx.mFlags & RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE)
-        return ;
+	if(!!(ctx.mFlags & RsSerializationFlags::SIGNATURE)) return;
 
     RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,signature,"signature") ;
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,duplication_factor,"duplication_factor") ;
@@ -133,8 +131,7 @@ void RsGRouterSignedReceiptItem::serial_process(RsGenericSerializer::SerializeJo
     RsTypeSerializer::serial_process<uint32_t> (j,ctx,service_id,"service_id") ;
     RsTypeSerializer::serial_process           (j,ctx,data_hash,"data_hash") ;
 
-    if(ctx.mFlags &  RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE)
-        return ;
+	if(!!(ctx.mFlags & RsSerializationFlags::SIGNATURE)) return;
 
     RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,signature,"signature") ;
 }

libretroshare/src/grouter/grouteritems.h

@@ -286,7 +286,9 @@ class RsGRouterRoutingInfoItem: public RsGRouterItem, public GRouterRoutingInfo,
 class RsGRouterSerialiser: public RsServiceSerializer
 {
 public:
-    explicit RsGRouterSerialiser(SerializationFlags flags = SERIALIZATION_FLAG_NONE) : RsServiceSerializer(RS_SERVICE_TYPE_GROUTER,RsGenericSerializer::FORMAT_BINARY,flags) {}
+	explicit RsGRouterSerialiser(
+	        RsSerializationFlags flags = RsSerializationFlags::NONE ):
+	    RsServiceSerializer(RS_SERVICE_TYPE_GROUTER, flags) {}
 
     virtual RsItem *create_item(uint16_t service,uint8_t subtype) const ;
 };

libretroshare/src/grouter/p3grouter.cc

@@ -1658,7 +1658,9 @@ void p3GRouter::handleIncomingReceiptItem(const RsGRouterSignedReceiptItem *rece
 
 Sha1CheckSum p3GRouter::computeDataItemHash(const RsGRouterGenericDataItem *data_item)
 {
-	RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER);
+	RsGRouterSerialiser signature_serializer(
+	            RsSerializationFlags::SIGNATURE |
+	            RsSerializationFlags::SKIP_HEADER );
 
     uint32_t signed_data_size = signature_serializer.size(const_cast<RsGRouterGenericDataItem*>(data_item));
     uint32_t total_size = signed_data_size + data_item->signature.TlvSize() ;
@@ -2034,7 +2036,9 @@ bool p3GRouter::signDataItem(RsGRouterAbstractMsgItem *item,const RsGxsId& signi
         std::cerr << "     Key ID = " << signing_id << std::endl;
         std::cerr << "     Getting key material..." << std::endl;
 //#endif
-        RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER) ;
+		RsGRouterSerialiser signature_serializer(
+		            RsSerializationFlags::SIGNATURE |
+		            RsSerializationFlags::SKIP_HEADER );
 		uint32_t data_size = signature_serializer.size(item) ;
 		RsTemporaryMemory data(data_size) ;
 
@@ -2092,8 +2096,8 @@ bool p3GRouter::verifySignedDataItem(const RsGRouterAbstractMsgItem *item,const
 		}
 
 		RsGRouterSerialiser signature_serializer(
-		            RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE |
-		            RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER );
+		            RsSerializationFlags::SIGNATURE |
+		            RsSerializationFlags::SKIP_HEADER );
 
 		uint32_t data_size = signature_serializer.size(const_cast<RsGRouterAbstractMsgItem*>(item));	// the const cast shouldn't be necessary if size() took a const.
 		RsTemporaryMemory data(data_size);
@@ -2104,7 +2108,7 @@ bool p3GRouter::verifySignedDataItem(const RsGRouterAbstractMsgItem *item,const
 		if(!signature_serializer.serialise(const_cast<RsGRouterAbstractMsgItem*>(item),data,&data_size))
 			throw std::runtime_error("Cannot serialise signed data.");
 
-		RsIdentityUsage use(RS_SERVICE_TYPE_GROUTER,info);
+		RsIdentityUsage use(RsServiceType::GROUTER,info);
 
 		if(!mGixs->validateData( data, data_size, item->signature, true, use, error_status ))
 		{

libretroshare/src/gxs/gxssecurity.cc

@@ -289,7 +289,7 @@ bool GxsSecurity::generateKeyPair(RsTlvPublicRSAKey& public_key,RsTlvPrivateRSAK
     if(!(private_key.checkKey() && public_key.checkKey()))
     {
         std::cerr << "(EE) ERROR while generating keys. Something inconsistent in flags. This is probably a bad sign!" << std::endl;
-	return false ;
+		return false ;
     }
                      
     return true ;
@@ -418,13 +418,11 @@ bool GxsSecurity::validateNxsMsg(const RsNxsMsg& msg, const RsTlvKeySignature& s
 
             /* check signature timeperiod */
             if ((msgMeta.mPublishTs < key.startTS) || (key.endTS != 0 && msgMeta.mPublishTs > key.endTS))
-            {
-    #ifdef GXS_SECURITY_DEBUG
-                    std::cerr << " GxsSecurity::validateNxsMsg() TS out of range";
-                    std::cerr << std::endl;
-    #endif
-                    return false;
-            }
+			{
+				RsWarn() << __PRETTY_FUNCTION__ << " GxsSecurity::validateNxsMsg() TS out of range for key " << msgMeta.mAuthorId
+				         << " usage is limited to TS=[" << key.startTS << "," << key.endTS << "] and msg publish time is " << msgMeta.mPublishTs << std::endl;
+				return false;
+			}
 
             /* decode key */
             const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data;
@@ -653,6 +651,9 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u
 
 	try
 	{
+        if(keys.empty())
+			throw std::runtime_error("EVP_SealInit will not be called with 0 keys. GxsSecurity::encrypt() was called with an empty set of destination keys!") ;
+
 		for(uint32_t i=0;i<keys.size();++i)
 		{
 			RSA *tmpkey = ::extractPublicKey(keys[i]) ;

libretroshare/src/gxs/rsdataservice.cc

@@ -110,8 +110,6 @@ const std::string RsGeneralDataService::MSG_META_STATUS = KEY_MSG_STATUS;
 
 const uint32_t RsGeneralDataService::GXS_MAX_ITEM_SIZE = 1572864; // 1.5 Mbytes
 
-static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // 10 minutes
-
 static int addColumn(std::list<std::string> &list, const std::string &attribute)
 {
     list.push_back(attribute);
@@ -123,7 +121,6 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d
     : RsGeneralDataService(), mDbMutex("RsDataService"), mServiceDir(serviceDir), mDbName(dbName), mDbPath(mServiceDir + "/" + dbName), mServType(serviceType), mDb(NULL)
 {
     bool isNewDatabase = !RsDirUtil::fileExists(mDbPath);
-    mGrpMetaDataCache_ContainsAllDatabase = false ;
 
     mDb = new RetroDb(mDbPath, RetroDb::OPEN_READWRITE_CREATE, key);
 
@@ -488,8 +485,7 @@ bool RsDataService::finishReleaseUpdate(int release, bool result)
 RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset,bool use_cache)
 {
 #ifdef RS_DATA_SERVICE_DEBUG
-    std::cerr << "RsDataService::locked_getGrpMeta()";
-    std::cerr << std::endl;
+    std::cerr << "RsDataService::locked_getGrpMeta()" << std::endl;
 #endif
 
     bool ok = true;
@@ -506,21 +502,17 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset
     RsGxsGrpMetaData* grpMeta ;
 	RsGxsGroupId grpId(tempId) ;
 
-	if(use_cache)
-	{
-		auto it = mGrpMetaDataCache.find(grpId) ;
+    if(grpId.isNull())			// not in the DB!
+        return nullptr;
 
-		if(it != mGrpMetaDataCache.end())
-			grpMeta = it->second ;
-		else
-		{
-			grpMeta = new RsGxsGrpMetaData();
-			mGrpMetaDataCache[grpId] = grpMeta ;
-		}
-	}
+	if(use_cache)
+        grpMeta = mGrpMetaDataCache.getOrCreateMeta(grpId);
 	else
 		grpMeta = new RsGxsGrpMetaData();
 
+    if(!grpMeta->mGroupId.isNull())	// the grpMeta is already initialized because it comes from the cache
+        return grpMeta;
+
     grpMeta->mGroupId = RsGxsGroupId(tempId);
     c.getString(mColGrpMeta_NxsIdentity + colOffset, tempId);
     grpMeta->mAuthorId = RsGxsId(tempId);
@@ -653,24 +645,40 @@ RsNxsGrp* RsDataService::locked_getGroup(RetroCursor &c)
     return NULL;
 }
 
-RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset)
+RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset,bool use_cache)
 {
 
-    RsGxsMsgMetaData* msgMeta = new RsGxsMsgMetaData();
-
     bool ok = true;
     uint32_t data_len = 0,
     offset = 0;
     char* data = NULL;
 
+    RsGxsGroupId group_id;
+    RsGxsMessageId msg_id;
+
     std::string gId;
     c.getString(mColMsgMeta_GrpId + colOffset, gId);
-    msgMeta->mGroupId = RsGxsGroupId(gId);
+    group_id = RsGxsGroupId(gId);
     std::string temp;
     c.getString(mColMsgMeta_MsgId + colOffset, temp);
-    msgMeta->mMsgId = RsGxsMessageId(temp);
+    msg_id = RsGxsMessageId(temp);
+
     // without these, a msg is meaningless
-    ok &= (!msgMeta->mGroupId.isNull()) && (!msgMeta->mMsgId.isNull());
+    if(group_id.isNull() || msg_id.isNull())
+        return nullptr;
+
+    RsGxsMsgMetaData* msgMeta = nullptr;
+
+	if(use_cache)
+        msgMeta = mMsgMetaDataCache[group_id].getOrCreateMeta(msg_id);
+	else
+		msgMeta = new RsGxsMsgMetaData();
+
+    if(!msgMeta->mGroupId.isNull())	// we cannot do that because the cursor needs to advance. Is there a method to skip some data in the db?
+        return msgMeta;
+
+	msgMeta->mGroupId = group_id;
+	msgMeta->mMsgId = msg_id;
 
     c.getString(mColMsgMeta_OrigMsgId + colOffset, temp);
     msgMeta->mOrigMsgId = RsGxsMessageId(temp);
@@ -704,7 +712,7 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset
 
     if(ok)
         return msgMeta;
-    else
+    else if(!use_cache)
         delete msgMeta;
 
     return NULL;
@@ -834,7 +842,8 @@ int RsDataService::storeMessage(const std::list<RsNxsMsg*>& msg)
 
         // This is needed so that mLastPost is correctly updated in the group meta when it is re-loaded.
 
-        locked_clearGrpMetaCache(msgMetaPtr->mGroupId);
+        mGrpMetaDataCache.clear(msgMetaPtr->mGroupId);
+		mMsgMetaDataCache[msgMetaPtr->mGroupId].updateMeta(msgMetaPtr->mMsgId,*msgMetaPtr);
     }
 
     // finish transaction
@@ -926,7 +935,7 @@ int RsDataService::storeGroup(const std::list<RsNxsGrp*>& grp)
 		cv.put(KEY_GRP_STATUS, (int32_t)grpMetaPtr->mGroupStatus);
 		cv.put(KEY_GRP_LAST_POST, (int32_t)grpMetaPtr->mLastPost);
 
-		locked_updateGrpMetaCache(*grpMetaPtr);
+		mGrpMetaDataCache.updateMeta(grpMetaPtr->mGroupId,*grpMetaPtr);
 
 		if (!mDb->sqlInsert(GRP_TABLE_NAME, "", cv))
 		{
@@ -942,54 +951,6 @@ int RsDataService::storeGroup(const std::list<RsNxsGrp*>& grp)
     return ret;
 }
 
-void RsDataService::locked_updateGrpMetaCache(const RsGxsGrpMetaData& meta)
-{
-	auto it = mGrpMetaDataCache.find(meta.mGroupId) ;
-
-	if(it != mGrpMetaDataCache.end())
-		*(it->second) = meta ;
-    else
-        mGrpMetaDataCache[meta.mGroupId] = new RsGxsGrpMetaData(meta) ;
-}
-
-void RsDataService::locked_clearGrpMetaCache(const RsGxsGroupId& gid)
-{
-	rstime_t now = time(NULL) ;
-    auto it = mGrpMetaDataCache.find(gid) ;
-
-	// We dont actually delete the item, because it might be used by a calling client.
-	// In this case, the memory will not be used for long, so we keep it into a list for a safe amount
-	// of time and delete it later. Using smart pointers here would be more elegant, but that would need
-	// to be implemented thread safe, which is difficult in this case.
-
-	if(it != mGrpMetaDataCache.end())
-	{
-#ifdef RS_DATA_SERVICE_DEBUG
-		std::cerr << "(II) moving database cache entry " << (void*)(*it).second << " to dead list." << std::endl;
-#endif
-
-		mOldCachedItems.push_back(std::make_pair(now,it->second)) ;
-
-		mGrpMetaDataCache.erase(it) ;
-		mGrpMetaDataCache_ContainsAllDatabase = false;
-	}
-
-	// We also take that opportunity to delete old entries.
-
-	auto it2(mOldCachedItems.begin());
-
-	while(it2!=mOldCachedItems.end() && (*it2).first + CACHE_ENTRY_GRACE_PERIOD < now)
-	{
-#ifdef RS_DATA_SERVICE_DEBUG
-		std::cerr << "(II) deleting old GXS database cache entry " << (void*)(*it2).second << ", " << now - (*it2).first << " seconds old." << std::endl;
-#endif
-
-		delete (*it2).second ;
-		it2 = mOldCachedItems.erase(it2) ;
-	}
-
-}
-
 int RsDataService::updateGroup(const std::list<RsNxsGrp *> &grp)
 {
 
@@ -1058,7 +1019,7 @@ int RsDataService::updateGroup(const std::list<RsNxsGrp *> &grp)
 
         mDb->sqlUpdate(GRP_TABLE_NAME, "grpId='" + grpPtr->grpId.toStdString() + "'", cv);
 
-        locked_updateGrpMetaCache(*grpMetaPtr);
+        mGrpMetaDataCache.updateMeta(grpMetaPtr->mGroupId,*grpMetaPtr);
     }
     // finish transaction
     bool ret = mDb->commitTransaction();
@@ -1275,7 +1236,7 @@ void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vector<RsNxsMsg
 
         if(m){
             if (metaOffset) {
-                m->metaData = locked_getMsgMeta(*c, metaOffset);
+                m->metaData = locked_getMsgMeta(*c, metaOffset,false);
             }
             msgs.push_back(m);
         }
@@ -1285,7 +1246,7 @@ void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vector<RsNxsMsg
     return;
 }
 
-int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaResult &msgMeta)
+int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaResult& msgMeta)
 {
     RsStackMutex stack(mDbMutex);
 
@@ -1294,77 +1255,112 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes
     int resultCount = 0;
 #endif
 
-    GxsMsgReq::const_iterator mit = reqIds.begin();
-
-    for(; mit != reqIds.end(); ++mit)
+    for(auto mit(reqIds.begin()); mit != reqIds.end(); ++mit)
     {
 
         const RsGxsGroupId& grpId = mit->first;
+        const std::set<RsGxsMessageId>& msgIdV = mit->second;
 
         // if vector empty then request all messages
-        const std::set<RsGxsMessageId>& msgIdV = mit->second;
-        std::vector<RsGxsMsgMetaData*> metaSet;
 
-        if(msgIdV.empty()){
-            RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", "");
+        t_MetaDataCache<RsGxsMessageId,RsGxsMsgMetaData>& cache(mMsgMetaDataCache[grpId]);
 
-            if (c)
-            {
-                locked_retrieveMsgMeta(c, metaSet);
+        if(msgIdV.empty())
+        {
+            if(cache.isCacheUpToDate())
+                cache.getFullMetaList(msgMeta[grpId]);
+            else
+			{
+				RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", "");
+
+				if (c)
+				{
+					locked_retrieveMsgMetaList(c, msgMeta[grpId]);
+					cache.setCacheUpToDate(true);
+				}
+                delete c;
+			}
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
-              std::cerr << "Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl;
+			std::cerr << mDbName << ": Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl;
 #endif
-            }
-        }else{
-
-            // request each grp
-            std::set<RsGxsMessageId>::const_iterator sit = msgIdV.begin();
-
-            for(; sit!=msgIdV.end(); ++sit){
-                const RsGxsMessageId& msgId = *sit;
-                RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString()
-                                               + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", "");
-
-                if (c)
-                {
-                    locked_retrieveMsgMeta(c, metaSet);
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-              std::cerr << "Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl;
-#endif
-                }
-            }
         }
+        else
+        {
+            // request each msg meta
+			auto& metaSet(msgMeta[grpId]);
 
-#ifdef RS_DATA_SERVICE_DEBUG_TIME
-        resultCount += metaSet.size();
+            for(auto sit(msgIdV.begin()); sit!=msgIdV.end(); ++sit)
+			{
+				const RsGxsMessageId& msgId = *sit;
+
+				RsGxsMsgMetaData *meta = cache.getMeta(msgId);
+
+                if(meta)
+                    metaSet.push_back(meta);
+                else
+				{
+					RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", "");
+
+                    c->moveToFirst();
+                    RsGxsMsgMetaData* meta = locked_getMsgMeta(*c, 0,true);
+
+                    if(meta)
+                        metaSet.push_back(meta);
+
+                    delete c;
+				}
+			}
+#ifdef RS_DATA_SERVICE_DEBUG_CACHE
+			std::cerr << mDbName << ": Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl;
 #endif
-
-        msgMeta[grpId] = metaSet;
+        }
     }
 
 #ifdef RS_DATA_SERVICE_DEBUG_TIME
+    if(mDbName==std::string("gxsforums_db"))
     std::cerr << "RsDataService::retrieveGxsMsgMetaData() " << mDbName << ", Requests: " << reqIds.size() << ", Results: " << resultCount << ", Time: " << timer.duration() << std::endl;
 #endif
 
     return 1;
 }
 
-void RsDataService::locked_retrieveMsgMeta(RetroCursor *c, std::vector<RsGxsMsgMetaData *> &msgMeta)
+void RsDataService::locked_retrieveGrpMetaList(RetroCursor *c, std::map<RsGxsGroupId,RsGxsGrpMetaData *>& grpMeta)
 {
+	if(!c)
+	{
+        RsErr() << __PRETTY_FUNCTION__ << ": attempt to retrieve Group Meta data from the DB with null cursor!" << std::endl;
+		return;
+	}
 
-    if(c)
-    {
-        bool valid = c->moveToFirst();
-        while(valid){
-            RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0);
+	bool valid = c->moveToFirst();
 
-            if(m != NULL)
-                msgMeta.push_back(m);
+	while(valid)
+	{
+		RsGxsGrpMetaData* m = locked_getGrpMeta(*c, 0,true);
 
-            valid = c->moveToNext();
-        }
-        delete c;
-    }
+		if(m)
+			grpMeta[m->mGroupId] = m;
+
+		valid = c->moveToNext();
+	}
+}
+void RsDataService::locked_retrieveMsgMetaList(RetroCursor *c, std::vector<const RsGxsMsgMetaData *>& msgMeta)
+{
+	if(!c)
+	{
+		RsErr() << __PRETTY_FUNCTION__ << ": attempt to retrieve Msg Meta data from the DB with null cursor!" << std::endl;
+		return;
+	}
+
+	bool valid = c->moveToFirst();
+	while(valid){
+		const RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0,true);
+
+		if(m != NULL)
+			msgMeta.push_back(m);
+
+		valid = c->moveToNext();
+	}
 }
 
 int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp)
@@ -1384,13 +1380,13 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp)
 
     if(grp.empty())
     {
-        if(mGrpMetaDataCache_ContainsAllDatabase)	// grab all the stash from the cache, so as to avoid decryption costs.
+        if(mGrpMetaDataCache.isCacheUpToDate())	// grab all the stash from the cache, so as to avoid decryption costs.
         {
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
         std::cerr << (void*)this << ": RsDataService::retrieveGxsGrpMetaData() retrieving all from cache!" << std::endl;
 #endif
 
-			grp = mGrpMetaDataCache ;
+			mGrpMetaDataCache.getFullMetaList(grp) ;
         }
         else
 		{
@@ -1401,93 +1397,103 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp)
 
 			RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", "");
 
-			if(c)
+            if(c)
 			{
-				bool valid = c->moveToFirst();
+				locked_retrieveGrpMetaList(c,grp);
 
-				while(valid)
-				{
-					RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true);
-
-					if(g)
-					{
-						grp[g->mGroupId] = g;
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-						std::cerr << (void *)this << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl;
-#endif
-					}
-					valid = c->moveToNext();
-
-#ifdef RS_DATA_SERVICE_DEBUG_TIME
-					++resultCount;
-#endif
-				}
-				delete c;
+				mGrpMetaDataCache.setCacheUpToDate(true);
 			}
+            delete c;
+#ifdef RS_DATA_SERVICE_DEBUG_TIME
+			resultCount += grp.size();
+#endif
 
-			mGrpMetaDataCache_ContainsAllDatabase = true ;
+//			if(c)
+//			{
+//				bool valid = c->moveToFirst();
+//
+//				while(valid)
+//				{
+//					RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true);
+//
+//					if(g)
+//					{
+//						grp[g->mGroupId] = g;
+//#ifdef RS_DATA_SERVICE_DEBUG_CACHE
+//						std::cerr << (void *)this << " " << mDbName << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl;
+//#endif
+//					}
+//					valid = c->moveToNext();
+//
+//				}
+//				delete c;
+//			}
 		}
-
     }
 	else
-    {
-		std::map<RsGxsGroupId, RsGxsGrpMetaData *>::iterator mit = grp.begin();
+	{
+		for(auto mit(grp.begin()); mit != grp.end(); ++mit)
+		{
+			RsGxsGrpMetaData *meta = mGrpMetaDataCache.getMeta(mit->first) ;
 
-          for(; mit != grp.end(); ++mit)
-          {
-              std::map<RsGxsGroupId, RsGxsGrpMetaData*>::const_iterator itt = mGrpMetaDataCache.find(mit->first) ;
-
-              if(itt != mGrpMetaDataCache.end())
-              {
+			if(meta)
+				mit->second = meta;
+			else
+			{
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
-          std::cerr << "Retrieving Grp metadata grpId=" << mit->first << " from cache!" << std::endl;
-#endif
-		  			grp[mit->first] = itt->second ;
-              }
-              else
-			  {
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-				  std::cerr << "Retrieving Grp metadata grpId=" << mit->first ;
+				std::cerr << mDbName << ": Retrieving Grp metadata grpId=" << mit->first ;
 #endif
 
-				  const RsGxsGroupId& grpId = mit->first;
-				  RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", "");
+				const RsGxsGroupId& grpId = mit->first;
+				RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", "");
 
-				  if(c)
-				  {
-					  bool valid = c->moveToFirst();
+				c->moveToFirst();
+				RsGxsGrpMetaData* meta = locked_getGrpMeta(*c, 0,true);
 
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-					  if(!valid)
-						  std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl;
-#endif
-					  while(valid)
-					  {
-						  RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true);
-
-						  if(g)
-						  {
-							  grp[g->mGroupId] = g;
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-							  std::cerr << ". Got it. Updating cache." << std::endl;
-#endif
-						  }
-						  valid = c->moveToNext();
+                if(meta)
+                    mit->second = meta;
 
 #ifdef RS_DATA_SERVICE_DEBUG_TIME
-						  ++resultCount;
+				++resultCount;
 #endif
-					  }
-					  delete c;
-				  }
-#ifdef RS_DATA_SERVICE_DEBUG_CACHE
-				  else
-					  std::cerr << ". not found!" << std::endl;
-#endif
-			  }
-          }
 
-      }
+                delete c;
+
+				//				  if(c)
+				//				  {
+				//					  bool valid = c->moveToFirst();
+				//
+				//#ifdef RS_DATA_SERVICE_DEBUG_CACHE
+				//					  if(!valid)
+				//						  std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl;
+				//#endif
+				//					  while(valid)
+				//					  {
+				//						  RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true);
+				//
+				//						  if(g)
+				//						  {
+				//							  grp[g->mGroupId] = g;
+				//#ifdef RS_DATA_SERVICE_DEBUG_CACHE
+				//							  std::cerr << ". Got it. Updating cache." << std::endl;
+				//#endif
+				//						  }
+				//						  valid = c->moveToNext();
+				//
+				//#ifdef RS_DATA_SERVICE_DEBUG_TIME
+				//						  ++resultCount;
+				//#endif
+				//					  }
+				//					  delete c;
+				//				  }
+#ifdef RS_DATA_SERVICE_DEBUG_CACHE
+				else
+				std::cerr << ". not found!" << std::endl;
+#endif
+			}
+		}
+
+	}
 
 #ifdef RS_DATA_SERVICE_DEBUG_TIME
     std::cerr << "RsDataService::retrieveGxsGrpMetaData() " << mDbName << ", Requests: " << requestedGroups << ", Results: " << resultCount << ", Time: " << timer.duration() << std::endl;
@@ -1524,35 +1530,37 @@ int RsDataService::resetDataStore()
     return 1;
 }
 
-int RsDataService::updateGroupMetaData(GrpLocMetaData &meta)
+int RsDataService::updateGroupMetaData(const GrpLocMetaData& meta)
 {
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
     std::cerr << (void*)this << ": Updating Grp Meta data: grpId = " << meta.grpId << std::endl;
 #endif
 
     RsStackMutex stack(mDbMutex);
-    RsGxsGroupId& grpId = meta.grpId;
+    const RsGxsGroupId& grpId = meta.grpId;
 
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
     std::cerr << (void*)this << ": erasing old entry from cache." << std::endl;
 #endif
 
-    locked_clearGrpMetaCache(meta.grpId);
+    mGrpMetaDataCache.clear(meta.grpId);
 
     return mDb->sqlUpdate(GRP_TABLE_NAME,  KEY_GRP_ID+ "='" + grpId.toStdString() + "'", meta.val) ? 1 : 0;
 }
 
-int RsDataService::updateMessageMetaData(MsgLocMetaData &metaData)
+int RsDataService::updateMessageMetaData(const MsgLocMetaData& metaData)
 {
 #ifdef RS_DATA_SERVICE_DEBUG_CACHE
     std::cerr << (void*)this << ": Updating Msg Meta data: grpId = " << metaData.msgId.first << " msgId = " << metaData.msgId.second << std::endl;
 #endif
 
     RsStackMutex stack(mDbMutex);
-    RsGxsGroupId& grpId = metaData.msgId.first;
-    RsGxsMessageId& msgId = metaData.msgId.second;
-    return mDb->sqlUpdate(MSG_TABLE_NAME,  KEY_GRP_ID+ "='" + grpId.toStdString()
-                          + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", metaData.val) ? 1 : 0;
+    const RsGxsGroupId& grpId = metaData.msgId.first;
+    const RsGxsMessageId& msgId = metaData.msgId.second;
+
+    mMsgMetaDataCache[grpId].clear(msgId);
+
+    return mDb->sqlUpdate(MSG_TABLE_NAME,  KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", metaData.val) ? 1 : 0;
 }
 
 int RsDataService::removeMsgs(const GxsMsgReq& msgIds)
@@ -1677,13 +1685,13 @@ bool RsDataService::locked_removeMessageEntries(const GxsMsgReq& msgIds)
     {
         const RsGxsGroupId& grpId = mit->first;
         const std::set<RsGxsMessageId>& msgsV = mit->second;
-        std::set<RsGxsMessageId>::const_iterator vit = msgsV.begin();
+        auto& cache(mMsgMetaDataCache[grpId]);
 
-        for(; vit != msgsV.end(); ++vit)
+        for(auto& msgId:msgsV)
         {
-            const RsGxsMessageId& msgId = *vit;
-            mDb->sqlDelete(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString()
-                    + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", "");
+            mDb->sqlDelete(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", "");
+
+            cache.clear(msgId);
         }
     }
 
@@ -1697,23 +1705,18 @@ bool RsDataService::locked_removeGroupEntries(const std::vector<RsGxsGroupId>& g
     // start a transaction
     bool ret = mDb->beginTransaction();
 
-    std::vector<RsGxsGroupId>::const_iterator vit = grpIds.begin();
-
-    for(; vit != grpIds.end(); ++vit)
+    for(auto grpId:grpIds)
     {
-
-        const RsGxsGroupId& grpId = *vit;
         mDb->sqlDelete(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", "");
 
 		// also remove the group meta from cache.
-		locked_clearGrpMetaCache(*vit) ;
+		mGrpMetaDataCache.clear(grpId) ;
     }
 
     ret &= mDb->commitTransaction();
-
-    mGrpMetaDataCache_ContainsAllDatabase = false ;
     return ret;
 }
+
 uint32_t RsDataService::cacheSize() const {
     return 0;
 }
@@ -1723,3 +1726,38 @@ int RsDataService::setCacheSize(uint32_t /* size */)
     return 0;
 }
 
+void RsDataService::debug_printCacheSize() const
+{
+    uint32_t nb_items,nb_items_on_deadlist;
+    uint64_t total_size,total_size_of_deadlist;
+
+    mGrpMetaDataCache.debug_computeSize(nb_items, nb_items_on_deadlist, total_size,total_size_of_deadlist);
+
+    RsDbg() << "Cache size: " << std::endl;
+    RsDbg() << "   Groups: " << " total: " << nb_items << " (dead: " << nb_items_on_deadlist << "), size: " << total_size << " (Dead: " << total_size_of_deadlist << ")" << std::endl;
+
+    nb_items = 0,nb_items_on_deadlist = 0;
+    total_size = 0,total_size_of_deadlist = 0;
+
+    for(auto it:mMsgMetaDataCache)
+    {
+		uint32_t tmp_nb_items,tmp_nb_items_on_deadlist;
+		uint64_t tmp_total_size,tmp_total_size_of_deadlist;
+
+		it.second.debug_computeSize(tmp_nb_items, tmp_nb_items_on_deadlist, tmp_total_size,tmp_total_size_of_deadlist);
+
+        nb_items += tmp_nb_items;
+        nb_items_on_deadlist += tmp_nb_items_on_deadlist;
+        total_size += tmp_total_size;
+        total_size_of_deadlist += tmp_total_size_of_deadlist;
+    }
+    RsDbg() << "   Msgs:   " << " total: " << nb_items << " (dead: " << nb_items_on_deadlist << "), size: " << total_size << " (Dead: " << total_size_of_deadlist << ")" << std::endl;
+}
+
+
+
+
+
+
+
+

libretroshare/src/gxs/rsdataservice.h

@@ -35,6 +35,124 @@ public:
 	ContentValue cv;
 };
 
+template<class ID, class MetaDataClass> class t_MetaDataCache
+{
+public:
+    t_MetaDataCache() : mCache_ContainsAllMetas(false) {}
+    virtual ~t_MetaDataCache()
+    {
+        for(auto it: mMetas)
+            delete it.second;
+
+        for(auto it: mOldCachedItems)
+            delete it.second;
+    }
+
+    bool isCacheUpToDate() const { return mCache_ContainsAllMetas ; }
+    void setCacheUpToDate(bool b) { mCache_ContainsAllMetas = b; }
+
+	void getFullMetaList(std::map<ID,MetaDataClass*>& mp) const { mp = mMetas ; }
+    void getFullMetaList(std::vector<const MetaDataClass*>& mp) const { for(auto& m:mMetas) mp.push_back(m.second) ; }
+
+    MetaDataClass *getMeta(const ID& id)
+    {
+		auto itt = mMetas.find(id);
+
+		if(itt != mMetas.end())
+			return itt->second ;
+        else
+            return NULL;
+    }
+
+    MetaDataClass *getOrCreateMeta(const ID& id)
+    {
+        MetaDataClass *meta = nullptr;
+		auto it = mMetas.find(id) ;
+
+		if(it != mMetas.end())
+		{
+#ifdef RS_DATA_SERVICE_DEBUG
+			RsDbg() << __PRETTY_FUNCTION__ << ": getting group meta " << grpId << " from cache." << std::endl;
+#endif
+			meta = it->second ;
+		}
+		else
+		{
+#ifdef RS_DATA_SERVICE_DEBUG
+			RsDbg() << __PRETTY_FUNCTION__ << ": group meta " << grpId << " not in cache. Loading it from DB..." << std::endl;
+#endif
+			meta = new MetaDataClass();
+			mMetas[id] = meta ;
+		}
+
+        return meta;
+    }
+
+	void updateMeta(const ID& id,const MetaDataClass& meta)
+	{
+		auto it = mMetas.find(id) ;
+
+		if(it != mMetas.end())
+			*(it->second) = meta ;
+		else
+			mMetas[id] = new MetaDataClass(meta) ;
+	}
+
+    void clear(const ID& id)
+	{
+		rstime_t now = time(NULL) ;
+		auto it = mMetas.find(id) ;
+
+		// We dont actually delete the item, because it might be used by a calling client.
+		// In this case, the memory will not be used for long, so we keep it into a list for a safe amount
+		// of time and delete it later. Using smart pointers here would be more elegant, but that would need
+		// to be implemented thread safe, which is difficult in this case.
+
+		if(it != mMetas.end())
+		{
+#ifdef RS_DATA_SERVICE_DEBUG
+			std::cerr << "(II) moving database cache entry " << (void*)(*it).second << " to dead list." << std::endl;
+#endif
+
+			mOldCachedItems.push_back(std::make_pair(now,it->second)) ;
+
+			mMetas.erase(it) ;
+			mCache_ContainsAllMetas = false;
+		}
+
+		// We also take that opportunity to delete old entries.
+
+		auto it2(mOldCachedItems.begin());
+
+		while(it2!=mOldCachedItems.end() && (*it2).first + CACHE_ENTRY_GRACE_PERIOD < now)
+		{
+#ifdef RS_DATA_SERVICE_DEBUG
+			std::cerr << "(II) deleting old GXS database cache entry " << (void*)(*it2).second << ", " << now - (*it2).first << " seconds old." << std::endl;
+#endif
+			delete (*it2).second ;
+			it2 = mOldCachedItems.erase(it2) ;
+		}
+	}
+
+    void debug_computeSize(uint32_t& nb_items, uint32_t& nb_items_on_deadlist, uint64_t& total_size,uint64_t& total_size_of_deadlist) const
+    {
+        nb_items = mMetas.size();
+        nb_items_on_deadlist = mOldCachedItems.size();
+        total_size = 0;
+        total_size_of_deadlist = 0;
+
+        for(auto it:mMetas) total_size += it.second->serial_size();
+        for(auto it:mOldCachedItems) total_size_of_deadlist += it.second->serial_size();
+    }
+private:
+	std::map<ID,MetaDataClass*> mMetas;
+	std::list<std::pair<rstime_t,MetaDataClass*> > mOldCachedItems ;	// dead list, where items get deleted after being unused for a while. This is due to not using smart ptrs.
+
+	static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // Unused items are deleted 10 minutes after last usage.
+
+    bool mCache_ContainsAllMetas ;
+};
+
 class RsDataService : public RsGeneralDataService
 {
 public:
@@ -147,13 +265,13 @@ public:
      * @param metaData The meta data item to update
      * @return error code
      */
-    int updateMessageMetaData(MsgLocMetaData& metaData);
+    int updateMessageMetaData(const MsgLocMetaData& metaData);
 
     /*!
      * @param metaData The meta data item to update
      * @return error code
      */
-    int updateGroupMetaData(GrpLocMetaData& meta);
+    int updateGroupMetaData(const GrpLocMetaData &meta);
 
     /*!
      * Completely clear out data stored in
@@ -174,6 +292,8 @@ public:
 
     int updateGroupKeys(const RsGxsGroupId& grpId,const RsTlvSecurityKeySet& keys, uint32_t subscribe_flags) ;
 
+    void debug_printCacheSize() const;
+
 private:
 
     /*!
@@ -194,15 +314,22 @@ private:
     /*!
      * Retrieves all the msg meta results from a cursor
      * @param c cursor to result set
-     * @param metaSet message metadata retrieved from cursor are stored here
+     * @param msgMeta message metadata retrieved from cursor are stored here
      */
-    void locked_retrieveMsgMeta(RetroCursor* c, std::vector<RsGxsMsgMetaData*>& msgMeta);
+    void locked_retrieveMsgMetaList(RetroCursor* c, std::vector<const RsGxsMsgMetaData*>& msgMeta);
+
+    /*!
+     * Retrieves all the grp meta results from a cursor
+     * @param c cursor to result set
+     * @param grpMeta group metadata retrieved from cursor are stored here
+     */
+	void locked_retrieveGrpMetaList(RetroCursor *c, std::map<RsGxsGroupId,RsGxsGrpMetaData *>& grpMeta);
 
     /*!
      * extracts a msg meta item from a cursor at its
      * current position
      */
-    RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c, int colOffset);
+    RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c, int colOffset, bool use_cache);
 
     /*!
      * extracts a grp meta item from a cursor at its
@@ -348,10 +475,8 @@ private:
     void locked_clearGrpMetaCache(const RsGxsGroupId& gid);
 	void locked_updateGrpMetaCache(const RsGxsGrpMetaData& meta);
 
-    std::map<RsGxsGroupId,RsGxsGrpMetaData*> mGrpMetaDataCache ;
-	std::list<std::pair<rstime_t,RsGxsGrpMetaData*> > mOldCachedItems ;
-
-    bool mGrpMetaDataCache_ContainsAllDatabase ;
+    t_MetaDataCache<RsGxsGroupId,RsGxsGrpMetaData> mGrpMetaDataCache;
+    std::map<RsGxsGroupId,t_MetaDataCache<RsGxsMessageId,RsGxsMsgMetaData> > mMsgMetaDataCache;
 };
 
 #endif // RSDATASERVICE_H

libretroshare/src/gxs/rsgds.h

@@ -149,7 +149,7 @@ public:
 	        bool withMeta = false ) = 0;
 
     /*!
-     * Retrieves all groups stored
+     * Retrieves all groups stored. Caller owns the memory and is supposed to delete the RsNxsGrp pointers after use.
      * @param grp retrieved groups
      * @param withMeta if true the meta handle of nxs grps is intitialised
      * @param cache whether to store retrieval in mem for faster later retrieval
@@ -239,12 +239,12 @@ public:
     /*!
      * @param metaData
      */
-    virtual int updateMessageMetaData(MsgLocMetaData& metaData) = 0;
+    virtual int updateMessageMetaData(const MsgLocMetaData& metaData) = 0;
 
     /*!
      * @param metaData
      */
-    virtual int updateGroupMetaData(GrpLocMetaData& meta) = 0;
+    virtual int updateGroupMetaData(const GrpLocMetaData& meta) = 0;
 
     virtual int updateGroupKeys(const RsGxsGroupId& grpId,const RsTlvSecurityKeySet& keys,uint32_t subscribed_flags) = 0 ;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -61,7 +61,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN        = 0x00000040; // admin key
 
 #define GXS_MASK "GXS_MASK_HACK"
 
-//#define GEN_EXCH_DEBUG	1
+#define GEN_EXCH_DEBUG	1
 
 static const uint32_t MSG_CLEANUP_PERIOD     = 60*59; // 59 minutes
 static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes
@@ -236,14 +236,17 @@ void RsGenExchange::tick()
 
 			if (!grpIds.empty())
 			{
-				RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PROCESSED, false);
-				gc->mGrpIdList = grpIds;
+                for(auto& groupId:grpIds)
+				{
+					RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_GROUP_DELETED,groupId, false);
+
 #ifdef GEN_EXCH_DEBUG
-				std::cerr << "  adding the following grp ids to notification: " << std::endl;
-				for(std::list<RsGxsGroupId>::const_iterator it(grpIds.begin());it!=grpIds.end();++it)
-					std::cerr << "    " << *it << std::endl;
+					std::cerr << "  adding the following grp ids to notification: " << std::endl;
+					for(std::list<RsGxsGroupId>::const_iterator it(grpIds.begin());it!=grpIds.end();++it)
+						std::cerr << "    " << *it << std::endl;
 #endif
-				mNotifications.push_back(gc);
+					mNotifications.push_back(gc);
+				}
 
 				// also notify the network exchange service that these groups no longer exist.
 
@@ -251,12 +254,12 @@ void RsGenExchange::tick()
 					mNetService->removeGroups(grpIds) ;
 			}
 
-			if (!msgIds.empty())
-			{
-				RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_PROCESSED, false);
-				c->msgChangeMap = msgIds;
-				mNotifications.push_back(c);
-			}
+            for(auto it(msgIds.begin());it!=msgIds.end();++it)
+                for(auto& msgId:it->second)
+				{
+					RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,it->first, msgId, false);
+					mNotifications.push_back(c);
+				}
 
 			delete mIntegrityCheck;
 			mIntegrityCheck = NULL;
@@ -279,7 +282,7 @@ bool RsGenExchange::messagePublicationTest(const RsGxsMsgMetaData& meta)
 
 	rstime_t storageTimeLimit = meta.mPublishTs + st;
 
-	return meta.mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP || st == 0 || storageTimeLimit >= time(NULL);
+	return meta.mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER || st == 0 || storageTimeLimit >= time(NULL);
 }
 
 bool RsGenExchange::acknowledgeTokenMsg(const uint32_t& token,
@@ -509,7 +512,7 @@ int RsGenExchange::createGroupSignatures(RsTlvKeySignatureSet& signSet, RsTlvBin
                 if(GxsSecurity::getSignature((char*)grpData.bin_data, grpData.bin_len, authorKey, sign))
                 {
                 	id_ret = SIGN_SUCCESS;
-					mGixs->timeStampKey(grpMeta.mAuthorId,RsIdentityUsage(mServType,RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_CREATION,grpMeta.mGroupId)) ;
+					mGixs->timeStampKey(grpMeta.mAuthorId,RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_CREATION,grpMeta.mGroupId)) ;
 					signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
                 }
                 else
@@ -677,7 +680,7 @@ int RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinar
 		    if(GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len, authorKey, sign))
 		    {
 			    id_ret = SIGN_SUCCESS;
-			    mGixs->timeStampKey(msgMeta.mAuthorId,RsIdentityUsage(mServType,RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_CREATION,msgMeta.mGroupId,msgMeta.mMsgId)) ;
+			    mGixs->timeStampKey(msgMeta.mAuthorId,RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_CREATION,msgMeta.mGroupId,msgMeta.mMsgId,msgMeta.mParentId,msgMeta.mThreadId)) ;
 			    signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
 		    }
 		    else
@@ -770,7 +773,7 @@ int RsGenExchange::createMessage(RsNxsMsg* msg)
 		hash.addData(allMsgData, allMsgDataLen);
 		RsFileHash hashId;
 		hash.Complete(hashId);
-		msg->msgId = hashId;
+		msg->msgId = RsGxsMessageId(hashId);
 
 		// assign msg id to msg meta
 		msg->metaData->mMsgId = msg->msgId;
@@ -902,7 +905,11 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin
 		    {
 			    RsTlvKeySignature sign = metaData.signSet.keySignSet[INDEX_AUTHEN_IDENTITY];
 			    idValidate &= GxsSecurity::validateNxsMsg(*msg, sign, authorKey);
-			    mGixs->timeStampKey(metaData.mAuthorId,RsIdentityUsage(mServType,RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId,metaData.mMsgId)) ;
+			    mGixs->timeStampKey(metaData.mAuthorId,RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_VALIDATION,
+                                                                       metaData.mGroupId,
+                                                                       metaData.mMsgId,
+                                                                       metaData.mParentId,
+                                                                       metaData.mThreadId)) ;
 		    }
 		    else
 		    {
@@ -934,7 +941,7 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin
 				        RsReputationLevel::LOCALLY_NEGATIVE )
 				{
 #ifdef GEN_EXCH_DEBUG	
-					std::cerr << "RsGenExchange::validateMsg(): message from " << metaData.mAuthorId << ", rejected because reputation level (" << details.mReputation.mOverallReputationLevel <<") indicate that you banned this ID." << std::endl;
+					std::cerr << "RsGenExchange::validateMsg(): message from " << metaData.mAuthorId << ", rejected because reputation level (" << static_cast<int>(details.mReputation.mOverallReputationLevel) <<") indicate that you banned this ID." << std::endl;
 #endif
 					idValidate = false ;
 				}
@@ -946,7 +953,12 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin
             {
                 std::list<RsPeerId> peers;
                 peers.push_back(msg->PeerId());
-                mGixs->requestKey(metaData.mAuthorId, peers, RsIdentityUsage(serviceType(),RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId,metaData.mMsgId));
+                mGixs->requestKey(metaData.mAuthorId, peers, RsIdentityUsage((RsServiceType)serviceType(),
+                                                                             RsIdentityUsage::MESSAGE_AUTHOR_SIGNATURE_VALIDATION,
+                                                                             metaData.mGroupId,
+                                                                             metaData.mMsgId,
+                                                                             metaData.mParentId,
+                                                                             metaData.mThreadId));
                 
 #ifdef GEN_EXCH_DEBUG
                 std::cerr << ", Key missing. Retry later." << std::endl;
@@ -1023,7 +1035,7 @@ int RsGenExchange::validateGrp(RsNxsGrp* grp)
 #ifdef GEN_EXCH_DEBUG
 				    std::cerr << "  key ID validation result: " << idValidate << std::endl;
 #endif
-					mGixs->timeStampKey(metaData.mAuthorId,RsIdentityUsage(mServType,RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId));
+					mGixs->timeStampKey(metaData.mAuthorId,RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId));
 			    }
 			    else
 			    {
@@ -1041,7 +1053,7 @@ int RsGenExchange::validateGrp(RsNxsGrp* grp)
 #endif
 			    std::list<RsPeerId> peers;
 			    peers.push_back(grp->PeerId());
-			    mGixs->requestKey(metaData.mAuthorId, peers,RsIdentityUsage(mServType,RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId));
+			    mGixs->requestKey(metaData.mAuthorId, peers,RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::GROUP_AUTHOR_SIGNATURE_VALIDATION,metaData.mGroupId));
 			    return VALIDATE_FAIL_TRY_LATER;
 		    }
 	    }
@@ -1110,8 +1122,10 @@ static void addMessageChanged(std::map<RsGxsGroupId, std::set<RsGxsMessageId> >
     }
 }
 
+#ifdef TO_REMOVE
 void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
 {
+    std::cerr << "***********************************  RsGenExchange::receiveChanges()" << std::endl;
 #ifdef GEN_EXCH_DEBUG
     std::cerr << "RsGenExchange::receiveChanges()" << std::endl;
 #endif
@@ -1154,6 +1168,7 @@ void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
 
 	if(rsEvents) rsEvents->postEvent(std::move(evt));
 }
+#endif
 
 bool RsGenExchange::subscribeToGroup(uint32_t& token, const RsGxsGroupId& grpId, bool subscribe)
 {
@@ -1192,7 +1207,7 @@ bool RsGenExchange::getGroupList(const uint32_t &token, std::list<RsGxsGroupId>
 bool RsGenExchange::getMsgList(const uint32_t &token,
                                GxsMsgIdResult &msgIds)
 {
-	return mDataAccess->getMsgList(token, msgIds);
+	return mDataAccess->getMsgIdList(token, msgIds);
 }
 
 bool RsGenExchange::getMsgRelatedList(const uint32_t &token, MsgRelatedIdResult &msgIds)
@@ -1268,18 +1283,18 @@ bool RsGenExchange::getMsgMeta(const uint32_t &token,
 
 	for(; mit != result.end(); ++mit)
 	{
-		std::vector<RsGxsMsgMetaData*>& metaV = mit->second;
+		std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
 
 		std::vector<RsMsgMetaData>& msgInfoV = msgInfo[mit->first];
 
-		std::vector<RsGxsMsgMetaData*>::iterator vit = metaV.begin();
+		std::vector<const RsGxsMsgMetaData*>::iterator vit = metaV.begin();
 		RsMsgMetaData meta;
 		for(; vit != metaV.end(); ++vit)
 		{
-			RsGxsMsgMetaData& m = *(*vit);
+			const RsGxsMsgMetaData& m = *(*vit);
 			meta = m;
 			msgInfoV.push_back(meta);
-			delete *vit;
+			//delete *vit;
 		}
 		metaV.clear();
 	}
@@ -1296,18 +1311,18 @@ bool RsGenExchange::getMsgRelatedMeta(const uint32_t &token, GxsMsgRelatedMetaMa
 
         for(; mit != result.end(); ++mit)
         {
-                std::vector<RsGxsMsgMetaData*>& metaV = mit->second;
+                std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
 
                 std::vector<RsMsgMetaData>& msgInfoV = msgMeta[mit->first];
 
-                std::vector<RsGxsMsgMetaData*>::iterator vit = metaV.begin();
+                std::vector<const RsGxsMsgMetaData*>::iterator vit = metaV.begin();
                 RsMsgMetaData meta;
                 for(; vit != metaV.end(); ++vit)
                 {
-                        RsGxsMsgMetaData& m = *(*vit);
+                        const RsGxsMsgMetaData& m = *(*vit);
                         meta = m;
                         msgInfoV.push_back(meta);
-                        delete *vit;
+                        //delete *vit;
                 }
                 metaV.clear();
         }
@@ -1669,21 +1684,14 @@ void RsGenExchange::receiveNewMessages(std::vector<RsNxsMsg *>& messages)
 
 void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId)
 {
-	std::cerr << __PRETTY_FUNCTION__ << " received result for request "
-	          << std::hex << id << std::dec << std::endl;
-
-	RS_STACK_MUTEX(mGenMtx);
-
-	RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId);
-	mNotifications.push_back(gc);
+	std::cerr << __PRETTY_FUNCTION__ << " received result for request " << std::hex << id << std::dec << ": this method should be overloaded in the client service, but it is not. This is a bug!" << std::endl;
 }
 
 void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId)
 {
 	RS_STACK_MUTEX(mGenMtx);
 
-	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY, true);
-	gc->mGrpIdList.push_back(grpId);
+	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY,grpId, true);
 	mNotifications.push_back(gc);
 }
 
@@ -1691,8 +1699,8 @@ void RsGenExchange::notifyChangedGroupStats(const RsGxsGroupId &grpId)
 {
 	RS_STACK_MUTEX(mGenMtx);
 
-	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PROCESSED, false);
-	gc->mGrpIdList.push_back(grpId);
+	RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_STATISTICS_CHANGED,grpId, false);
+
 	mNotifications.push_back(gc);
 }
 
@@ -1789,19 +1797,20 @@ void RsGenExchange::publishGroup(uint32_t& token, RsGxsGrpItem *grpItem)
 
 void RsGenExchange::updateGroup(uint32_t& token, RsGxsGrpItem* grpItem)
 {
-    if(!checkGroupMetaConsistency(grpItem->meta))
-    {
-        std::cerr << "(EE) Cannot update group. Some information was not supplied." << std::endl;
-       return ;
-    }
+	if(!checkGroupMetaConsistency(grpItem->meta))
+	{
+		std::cerr << "(EE) Cannot update group. Some information was not supplied." << std::endl;
+		delete grpItem;
+		return ;
+	}
 
-					RS_STACK_MUTEX(mGenMtx) ;
+	RS_STACK_MUTEX(mGenMtx) ;
 	token = mDataAccess->generatePublicToken();
-        mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, token));
+	mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, token));
 
 #ifdef GEN_EXCH_DEBUG
-    std::cerr << "RsGenExchange::updateGroup() token: " << token;
-    std::cerr << std::endl;
+	std::cerr << "RsGenExchange::updateGroup() token: " << token;
+	std::cerr << std::endl;
 #endif
 }
 
@@ -2010,15 +2019,15 @@ void RsGenExchange::processMsgMetaChanges()
 
                 if(mit != result.end())
                 {
-                    std::vector<RsGxsMsgMetaData*>& msgMetaV = mit->second;
+                    std::vector<const RsGxsMsgMetaData*>& msgMetaV = mit->second;
 
                     if(!msgMetaV.empty())
                     {
-                        RsGxsMsgMetaData* meta = *(msgMetaV.begin());
+                        const RsGxsMsgMetaData* meta = *(msgMetaV.begin());
                         value = (meta->mMsgStatus & ~mask) | (mask & value);
 						changed = (static_cast<int64_t>(meta->mMsgStatus) != value);
                         m.val.put(RsGeneralDataService::MSG_META_STATUS, value);
-                        delete meta;
+                        //delete meta;
                         ok = true;
                     }
                 }
@@ -2048,11 +2057,13 @@ void RsGenExchange::processMsgMetaChanges()
         }
     }
 
-    if (!msgIds.empty()) {
+    if (!msgIds.empty())
+    {
         RS_STACK_MUTEX(mGenMtx);
-        RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_PROCESSED, false);
-        c->msgChangeMap = msgIds;
-        mNotifications.push_back(c);
+
+        for(auto it(msgIds.begin());it!=msgIds.end();++it)
+            for(auto& msg_id:it->second)
+				mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_PROCESSED, it->first, msg_id, false));
     }
 }
 
@@ -2098,17 +2109,11 @@ void RsGenExchange::processGrpMetaChanges()
         }
     }
 
-    if(!grpChanged.empty())
+    for(auto& groupId:grpChanged)
     {
         RS_STACK_MUTEX(mGenMtx);
-        RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PROCESSED, true);
-        gc->mGrpIdList = grpChanged;
-        mNotifications.push_back(gc);
-#ifdef GEN_EXCH_DEBUG
-                    			std::cerr << "  adding the following grp ids to notification: " << std::endl;
-                                	for(std::list<RsGxsGroupId>::const_iterator it(grpChanged.begin());it!=grpChanged.end();++it)
-                                        	std::cerr << "    " << *it << std::endl;
-#endif
+
+        mNotifications.push_back(new RsGxsGroupChange(RsGxsNotify::TYPE_PROCESSED,groupId, true));
     }
 }
 
@@ -2185,7 +2190,7 @@ void RsGenExchange::publishMsgs()
 		mMsgsToPublish.insert(std::make_pair(sign_it->first, item.mItem));
 	}
 
-	std::map<RsGxsGroupId, std::set<RsGxsMessageId> > msgChangeMap;
+	std::map<RsGxsGroupId, std::list<RsGxsMsgItem*> > msgChangeMap;
 	std::map<uint32_t, RsGxsMsgItem*>::iterator mit = mMsgsToPublish.begin();
 
 	for(; mit != mMsgsToPublish.end(); ++mit)
@@ -2314,9 +2319,12 @@ void RsGenExchange::publishMsgs()
 
                 mPublishedMsgs[token] = *msg->metaData;
 
+                RsGxsMsgItem *msg_item = dynamic_cast<RsGxsMsgItem*>(mSerialiser->deserialise(msg->msg.bin_data,&msg->msg.bin_len)) ;
+                msg_item->meta = *msg->metaData;
+
 				delete msg ;
 
-				msgChangeMap[grpId].insert(msgId);
+				msgChangeMap[grpId].push_back(msg_item);
 
 				delete[] metaDataBuff;
 
@@ -2355,13 +2363,14 @@ void RsGenExchange::publishMsgs()
 	// entries are invalid
 	mMsgsToPublish.clear();
 
-	if(!msgChangeMap.empty())
-	{
-		RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISHED, false);
-		ch->msgChangeMap = msgChangeMap;
-		mNotifications.push_back(ch);
-	}
+    for(auto it(msgChangeMap.begin());it!=msgChangeMap.end();++it)
+        for(auto& msg_item: it->second)
+		{
+			RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISHED,msg_item->meta.mGroupId, msg_item->meta.mMsgId, false);
+			ch->mNewMsgItem = msg_item;
 
+			mNotifications.push_back(ch);
+		}
 }
 
 RsGenExchange::ServiceCreate_Return RsGenExchange::service_CreateGroup(RsGxsGrpItem* /* grpItem */,
@@ -2380,15 +2389,14 @@ RsGenExchange::ServiceCreate_Return RsGenExchange::service_CreateGroup(RsGxsGrpI
 
 void RsGenExchange::processGroupUpdatePublish()
 {
-					RS_STACK_MUTEX(mGenMtx) ;
+	RS_STACK_MUTEX(mGenMtx) ;
 
 	// get keys for group update publish
 
 	// first build meta request map for groups to be updated
 	RsGxsGrpMetaTemporaryMap grpMeta;
-	std::vector<GroupUpdatePublish>::iterator vit = mGroupUpdatePublish.begin();
 
-	for(; vit != mGroupUpdatePublish.end(); ++vit)
+	for(auto vit = mGroupUpdatePublish.begin(); vit != mGroupUpdatePublish.end(); ++vit)
 	{
 		GroupUpdatePublish& gup = *vit;
 		const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId;
@@ -2401,8 +2409,7 @@ void RsGenExchange::processGroupUpdatePublish()
 	mDataStore->retrieveGxsGrpMetaData(grpMeta);
 
 	// now
-	vit = mGroupUpdatePublish.begin();
-	for(; vit != mGroupUpdatePublish.end(); ++vit)
+	for(auto vit = mGroupUpdatePublish.begin(); vit != mGroupUpdatePublish.end(); ++vit)
 	{
 		GroupUpdatePublish& gup = *vit;
 		const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId;
@@ -2420,14 +2427,14 @@ void RsGenExchange::processGroupUpdatePublish()
 			meta = mit->second;
 
 		//gup.grpItem->meta = *meta;
-        GxsGrpPendingSign ggps(gup.grpItem, gup.mToken);
+		GxsGrpPendingSign ggps(gup.grpItem, gup.mToken);
 
 		if(checkKeys(meta->keys))
 		{
 			ggps.mKeys = meta->keys;
-            
-            		GxsSecurity::createPublicKeysFromPrivateKeys(ggps.mKeys) ;
-                    
+
+			GxsSecurity::createPublicKeysFromPrivateKeys(ggps.mKeys) ;
+
 			ggps.mHaveKeys = true;
 			ggps.mStartTS = time(NULL);
 			ggps.mLastAttemptTS = 0;
@@ -2437,8 +2444,8 @@ void RsGenExchange::processGroupUpdatePublish()
 		}
 		else
 		{
-            		std::cerr << "(EE) publish group fails because RS cannot find the private publish and author keys" << std::endl;
-                    
+			std::cerr << "(EE) publish group fails because RS cannot find the private publish and author keys" << std::endl;
+
 			delete gup.grpItem;
 			mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::FAILED);
 		}
@@ -2493,15 +2500,15 @@ void RsGenExchange::processGroupDelete()
 			grpDeleted.push_back(note.second);
 	}
 
-	if(!grpDeleted.empty())
+    for(auto& groupId:grpDeleted)
 	{
-		RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHED, false);
-		gc->mGrpIdList = grpDeleted;
+		RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_GROUP_DELETED, groupId,false);
 		mNotifications.push_back(gc);
 	}
 
 	mGroupDeletePublish.clear();
 }
+
 void RsGenExchange::processMessageDelete()
 {
     RS_STACK_MUTEX(mGenMtx) ;
@@ -2520,31 +2527,24 @@ void RsGenExchange::processMessageDelete()
 		mDataStore->removeMsgs( (*vit).mMsgs );
 	}
 
+//	std::list<RsGxsGroupId> grpDeleted;
+//	std::map<uint32_t, GrpNote>::iterator mit = toNotify.begin();
+//	for(; mit != toNotify.end(); ++mit)
+//	{
+//		GrpNote& note = mit->second;
+//		uint8_t status = note.first ? RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE
+//		                            : RsTokenService::GXS_REQUEST_V2_STATUS_FAILED;
+//
+//		mGrpNotify.insert(std::make_pair(mit->first, note.second));
+//		mDataAccess->updatePublicRequestStatus(mit->first, status);
+//
+//		if(note.first)
+//			grpDeleted.push_back(note.second);
+//	}
 
-#warning csoler: TODO: notify for deleted messages
-#ifdef SUSPENDED
-	std::list<RsGxsGroupId> grpDeleted;
-	std::map<uint32_t, GrpNote>::iterator mit = toNotify.begin();
-	for(; mit != toNotify.end(); ++mit)
-	{
-		GrpNote& note = mit->second;
-		uint8_t status = note.first ? RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE
-		                            : RsTokenService::GXS_REQUEST_V2_STATUS_FAILED;
-
-		mGrpNotify.insert(std::make_pair(mit->first, note.second));
-		mDataAccess->updatePublicRequestStatus(mit->first, status);
-
-		if(note.first)
-			grpDeleted.push_back(note.second);
-	}
-
-	if(!grpDeleted.empty())
-	{
-		RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISH, false);
-		gc->mGrpIdList = grpDeleted;
-		mNotifications.push_back(gc);
-	}
-#endif
+    for(uint32_t i=0;i<mMsgDeletePublish.size();++i)
+        for(auto it(mMsgDeletePublish[i].mMsgs.begin());it!=mMsgDeletePublish[i].mMsgs.end();++it)
+			mNotifications.push_back(new RsGxsGroupChange(RsGxsNotify::TYPE_MESSAGE_DELETED,it->first, false));
 
 	mMsgDeletePublish.clear();
 }
@@ -2804,17 +2804,8 @@ void RsGenExchange::publishGrps()
 			    grpChanged.push_back(note.second);
 	    }
 
-	    if(!grpChanged.empty())
-	    {
-		    RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
-		    gc->mGrpIdList = grpChanged;
-		    mNotifications.push_back(gc);
-#ifdef GEN_EXCH_DEBUG
-		    std::cerr << "  adding the following grp ids to notification: " << std::endl;
-		    for(std::list<RsGxsGroupId>::const_iterator it(grpChanged.begin());it!=grpChanged.end();++it)
-			    std::cerr << "    " << *it << std::endl;
-#endif
-	    }
+        for(auto& groupId:grpChanged)
+		    mNotifications.push_back(new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW,groupId, true));
     }
 
     // This is done off-mutex to avoid possible cross deadlocks with the net service.
@@ -3071,11 +3062,19 @@ void RsGenExchange::processRecvdMessages()
 #ifdef GEN_EXCH_DEBUG
 		    std::cerr << "  storing remaining messages" << std::endl;
 #endif
-		    mDataStore->storeMessage(msgs_to_store);
 
-		    RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
-		    c->msgChangeMap = msgIds;
-		    mNotifications.push_back(c);
+            for(auto& nxs_msg: msgs_to_store)
+            {
+                RsGxsMsgItem *item = dynamic_cast<RsGxsMsgItem*>(mSerialiser->deserialise(nxs_msg->msg.bin_data,&nxs_msg->msg.bin_len));
+                item->meta = *nxs_msg->metaData;
+
+				RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVED_NEW, item->meta.mGroupId, item->meta.mMsgId,false);
+                c->mNewMsgItem = item;
+
+				mNotifications.push_back(c);
+			}
+
+		    mDataStore->storeMessage(msgs_to_store);	// we do that late because it destroys the items in msgs_to_store
 	    }
     }
 
@@ -3204,11 +3203,17 @@ void RsGenExchange::processRecvdGroups()
 		vit = tmp ;
 	}
 
-	if(!grpIds.empty())
+	if(!grps_to_store.empty())
 	{
-		RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
-		c->mGrpIdList = grpIds;
-		mNotifications.push_back(c);
+        for(auto Grp:grps_to_store)
+		{
+			RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, Grp->grpId, false);
+
+            c->mNewGroupItem = dynamic_cast<RsGxsGrpItem*>(mSerialiser->deserialise(Grp->grp.bin_data,&Grp->grp.bin_len));
+
+			mNotifications.push_back(c);
+		}
+
 		mDataStore->storeGroup(grps_to_store);
 #ifdef GEN_EXCH_DEBUG
                     			std::cerr << "  adding the following grp ids to notification: " << std::endl;
@@ -3229,39 +3234,42 @@ void RsGenExchange::performUpdateValidation()
 	std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl;
 #endif
 
-	RsGxsGrpMetaTemporaryMap grpMetas;
+	RsNxsGrpDataTemporaryMap grpDatas;
 
-	std::vector<GroupUpdate>::iterator vit = mGroupUpdates.begin();
-	for(; vit != mGroupUpdates.end(); ++vit)
-		grpMetas.insert(std::make_pair(vit->newGrp->grpId, (RsGxsGrpMetaData*)NULL));
+	for(auto vit(mGroupUpdates.begin()); vit != mGroupUpdates.end(); ++vit)
+		grpDatas.insert(std::make_pair(vit->newGrp->grpId, (RsNxsGrp*)NULL));
+
+	if(grpDatas.empty() || !mDataStore->retrieveNxsGrps(grpDatas,true,false))
+    {
+        if(grpDatas.empty())
+			RsErr() << __PRETTY_FUNCTION__ << " Validation of multiple group updates failed: no group in list!" << std::endl;
+        else
+            RsErr() << __PRETTY_FUNCTION__ << " Validation of multiple group updates failed: cannot retrieve froup data for these groups!" << std::endl;
 
-	if(!grpMetas.empty())
-		mDataStore->retrieveGxsGrpMetaData(grpMetas);
-	else
 		return;
-
-	vit = mGroupUpdates.begin();
-	for(; vit != mGroupUpdates.end(); ++vit)
-	{
-		GroupUpdate& gu = *vit;
-		std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator mit = grpMetas.find(gu.newGrp->grpId);
-		gu.oldGrpMeta = mit->second;
-		gu.validUpdate = updateValid(*(gu.oldGrpMeta), *(gu.newGrp));
-	}
+    }
 
 #ifdef GEN_EXCH_DEBUG
 	std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl;
 #endif
 
-	vit = mGroupUpdates.begin();
-
 	RsNxsGrpDataTemporaryList grps ;
 
-	for(; vit != mGroupUpdates.end(); ++vit)
+	for(auto vit(mGroupUpdates.begin()); vit != mGroupUpdates.end(); ++vit)
 	{
 		GroupUpdate& gu = *vit;
 
-		if(gu.validUpdate)
+		auto mit = grpDatas.find(gu.newGrp->grpId);
+
+        if(mit == grpDatas.end())
+        {
+            RsErr() << __PRETTY_FUNCTION__ << " Validation of group update failed for group " << gu.newGrp->grpId << " because previous grp version cannot be found." << std::endl;
+            continue;
+        }
+        RsGxsGrpMetaData *oldGrpMeta(mit->second->metaData);
+        RsNxsGrp *oldGrp(mit->second);
+
+		if(updateValid(*oldGrpMeta, *gu.newGrp))
 		{
 			if(gu.newGrp->metaData->mCircleType == GXS_CIRCLE_TYPE_YOUR_FRIENDS_ONLY)
 				gu.newGrp->metaData->mOriginator = gu.newGrp->PeerId();
@@ -3269,42 +3277,39 @@ void RsGenExchange::performUpdateValidation()
 			// Keep subscriptionflag to what it was. This avoids clearing off the flag when updates to group meta information
 			// is received.
 
-			gu.newGrp->metaData->mSubscribeFlags = gu.oldGrpMeta->mSubscribeFlags ;
+			gu.newGrp->metaData->mSubscribeFlags = oldGrpMeta->mSubscribeFlags ;
 
 			// Also keep private keys if present
 
 			if(!gu.newGrp->metaData->keys.private_keys.empty())
 				std::cerr << "(EE) performUpdateValidation() group " <<gu.newGrp->metaData->mGroupId << " has been received with private keys. This is very unexpected!" << std::endl;
 			else
-				gu.newGrp->metaData->keys.private_keys = gu.oldGrpMeta->keys.private_keys ;
+				gu.newGrp->metaData->keys.private_keys = oldGrpMeta->keys.private_keys ;
+
+            // Now prepare notification of the client
+
+			RsGxsGroupChange *c = new RsGxsGroupChange(RsGxsNotify::TYPE_UPDATED,gu.newGrp->metaData->mGroupId,false);
+
+            c->mNewGroupItem = dynamic_cast<RsGxsGrpItem*>(mSerialiser->deserialise(gu.newGrp->grp.bin_data,&gu.newGrp->grp.bin_len));
+            c->mNewGroupItem->meta = *gu.newGrp->metaData;		// gu.newGrp will be deleted because mDataStore will destroy it on update
+
+            c->mOldGroupItem = dynamic_cast<RsGxsGrpItem*>(mSerialiser->deserialise(oldGrp->grp.bin_data,&oldGrp->grp.bin_len));
+            c->mOldGroupItem->meta = *oldGrpMeta;				// no need to delete mit->second, as it will be deleted automatically in the temporary map
+
+			mNotifications.push_back(c);
+
+			// finally, add the group to the list to send to mDataStore
 
 			grps.push_back(gu.newGrp);
 		}
 		else
 		{
-			delete gu.newGrp;
+			delete gu.newGrp;    // delete here because mDataStore will not take care of this one.  no need to delete mit->second, as it will be deleted automatically in the temporary map
 			gu.newGrp = NULL ;
 		}
-
-		gu.oldGrpMeta = NULL ;
 	}
-	// notify the client
-
-	RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
-
-	for(uint32_t i=0;i<mGroupUpdates.size();++i)
-		if(mGroupUpdates[i].newGrp != NULL)
-		{
-			c->mGrpIdList.push_back(mGroupUpdates[i].newGrp->grpId) ;
-#ifdef GEN_EXCH_DEBUG
-			std::cerr << "    " << mGroupUpdates[i].newGrp->grpId << std::endl;
-#endif
-		}
-
-	mNotifications.push_back(c);
 
 	// Warning: updateGroup will destroy the objects in grps. Dont use it afterwards!
-
 	mDataStore->updateGroup(grps);
 
 #ifdef GEN_EXCH_DEBUG
@@ -3350,7 +3355,7 @@ bool RsGenExchange::updateValid(const RsGxsGrpMetaData& oldGrpMeta, const RsNxsG
 	// also check this is the latest published group
 	bool latest = newGrp.metaData->mPublishTs > oldGrpMeta.mPublishTs;
 
-    mGixs->timeStampKey(newGrp.metaData->mAuthorId, RsIdentityUsage(mServType,RsIdentityUsage::GROUP_ADMIN_SIGNATURE_CREATION, oldGrpMeta.mGroupId)) ;
+    mGixs->timeStampKey(newGrp.metaData->mAuthorId, RsIdentityUsage(RsServiceType(mServType),RsIdentityUsage::GROUP_ADMIN_SIGNATURE_CREATION, oldGrpMeta.mGroupId)) ;
 
     return GxsSecurity::validateNxsGrp(newGrp, adminSign, keyMit->second) && latest;
 }
@@ -3460,20 +3465,32 @@ bool RsGenExchange::exportGroupBase64(
 
 	if(groupId.isNull()) return failure("groupId cannot be null");
 
+    // We have no blocking API here, so we need to make a blocking request manually.
 	const std::list<RsGxsGroupId> groupIds({groupId});
 	RsTokReqOptions opts;
 	opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA;
 	uint32_t token;
-	mDataAccess->requestGroupInfo(
-	            token, RS_TOKREQ_ANSTYPE_DATA, opts, groupIds);
-	RsTokenService::GxsRequestStatus wtStatus = mDataAccess->waitToken(token);
-	if(wtStatus != RsTokenService::COMPLETE)
-		return failure( "waitToken(...) failed with: " +
-		                std::to_string(wtStatus) );
+	mDataAccess->requestGroupInfo( token, RS_TOKREQ_ANSTYPE_DATA, opts, groupIds);
+
+    // provide a sync response: actually wait for the token.
+    std::chrono::milliseconds maxWait = std::chrono::milliseconds(10000);
+    std::chrono::milliseconds checkEvery = std::chrono::milliseconds(100);
+
+	auto timeout = std::chrono::steady_clock::now() + maxWait;	// wait for 10 secs at most
+	auto st = mDataAccess->requestStatus(token);
+
+	while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE) && std::chrono::steady_clock::now() < timeout )
+	{
+		std::this_thread::sleep_for(checkEvery);
+		st = mDataAccess->requestStatus(token);
+	}
+	if(st != RsTokenService::COMPLETE)
+		return failure( "waitToken(...) failed with: " + std::to_string(st) );
 
 	uint8_t* buf = nullptr;
 	uint32_t size;
 	RsGxsGroupId grpId;
+
 	if(!getSerializedGroupData(token, grpId, buf, size))
 		return failure("failed retrieving GXS data");
 

libretroshare/src/gxs/rsgenexchange.h

@@ -33,6 +33,7 @@
 #include "rsnxsobserver.h"
 #include "retroshare/rsgxsservice.h"
 #include "rsitems/rsnxsitems.h"
+#include "gxs/rsgxsnotify.h"
 #include "rsgxsutil.h"
 
 template<class GxsItem, typename Identity = std::string>
@@ -262,12 +263,14 @@ public:
      */
     bool getPublishedMsgMeta(const uint32_t& token,RsMsgMetaData& meta);
 
+#ifdef TO_REMOVE
     /*!
      * Gxs services should call this for automatic handling of
      * changes, send
      * @param changes
      */
     virtual void receiveChanges(std::vector<RsGxsNotify*>& changes);
+#endif
 
     /*!
      * \brief acceptNewGroup
@@ -726,7 +729,7 @@ public:
     virtual void     setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ;
 	virtual bool     getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats);
 
-    uint16_t serviceType() const { return mServType ; }
+    uint16_t serviceType() const override { return mServType ; }
     uint32_t serviceFullType() const { return RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(mServType); }
 
 	virtual RsReputationLevel minReputationForForwardingMessages(
@@ -748,9 +751,6 @@ protected:
      */
     virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) = 0;
 
-
-
-
 private:
 
     void processRecvdData();

libretroshare/src/gxs/rsgxs.h

@@ -29,8 +29,8 @@
 
 /* data types used throughout Gxs from netservice to genexchange */
 
-typedef std::map<RsGxsGroupId,      std::vector<RsGxsMsgMetaData*> > GxsMsgMetaResult;
-typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsGxsMsgMetaData*> > MsgRelatedMetaResult;
+typedef std::map<RsGxsGroupId,      std::vector<const RsGxsMsgMetaData*> > GxsMsgMetaResult;
+typedef std::map<RsGxsGrpMsgIdPair, std::vector<const RsGxsMsgMetaData*> > MsgRelatedMetaResult;
 
 // Default values that are used throughout GXS code
 

libretroshare/src/gxs/rsgxsdata.cc

@@ -196,7 +196,6 @@ bool RsGxsGrpMetaData::deserialise(void *data, uint32_t &pktsize)
 
     return ok;
 }
-int RsGxsMsgMetaData::refcount = 0;
 
 RsGxsMsgMetaData::RsGxsMsgMetaData(){
 	clear();
@@ -209,7 +208,7 @@ RsGxsMsgMetaData::~RsGxsMsgMetaData(){
 	return;
 }
 
-uint32_t RsGxsMsgMetaData::serial_size()
+uint32_t RsGxsMsgMetaData::serial_size() const
 {
 
     uint32_t s = 8; // header size

libretroshare/src/gxs/rsgxsdata.h

@@ -48,6 +48,7 @@ public:
     bool deserialise(void *data, uint32_t &pktsize);
     bool serialise(void* data, uint32_t &pktsize, uint32_t api_version);
     uint32_t serial_size(uint32_t api_version) const;
+    uint32_t serial_size() const { return serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION); }
     void clear();
     void operator =(const RsGroupMetaData& rMeta);
 
@@ -94,12 +95,10 @@ public:
     ~RsGxsMsgMetaData();
     bool deserialise(void *data, uint32_t *size);
     bool serialise(void* data, uint32_t *size);
-    uint32_t serial_size();
+    uint32_t serial_size() const;
     void clear();
     void operator =(const RsMsgMetaData& rMeta);
 
-    static int refcount;
-
     //Sort data in same order than serialiser and deserializer
     RsGxsGroupId mGroupId;
     RsGxsMessageId mMsgId;

libretroshare/src/gxs/rsgxsdataaccess.h

@@ -22,14 +22,17 @@
 #ifndef RSGXSDATAACCESS_H
 #define RSGXSDATAACCESS_H
 
+#include <queue>
 #include "retroshare/rstokenservice.h"
 #include "rsgxsrequesttypes.h"
 #include "rsgds.h"
 
 
-typedef std::map< RsGxsGroupId, std::map<RsGxsMessageId, RsGxsMsgMetaData*> > MsgMetaFilter;
+typedef std::map< RsGxsGroupId, std::map<RsGxsMessageId, const RsGxsMsgMetaData*> > MsgMetaFilter;
 typedef std::map< RsGxsGroupId, RsGxsGrpMetaData* > GrpMetaFilter;
 
+bool operator<(const std::pair<uint32_t,GxsRequest*>& p1,const std::pair<uint32_t,GxsRequest*>& p2);
+
 class RsGxsDataAccess : public RsTokenService
 {
 public:
@@ -56,7 +59,7 @@ public:
      * @param groupIds group id to request info for
      * @return
      */
-    bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId> &groupIds);
+    bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId> &groupIds) override;
 
     /*!
      * Use this to request all group related info
@@ -65,7 +68,7 @@ public:
      * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
      * @return
      */
-    bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts);
+    bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts) override;
 
     /*!
      * Use this to get msg information (id, meta, or data), store token value to poll for request completion
@@ -75,7 +78,7 @@ public:
      * @param groupIds The ids of the groups to get, second entry of map empty to query for all msgs
      * @return true if request successful false otherwise
      */
-    bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const GxsMsgReq& msgIds);
+    bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const GxsMsgReq& msgIds) override;
 
     /*!
      * Use this to get message information (id, meta, or data), store token value to poll for request completion
@@ -86,7 +89,7 @@ public:
      * all messages for all groups are retrieved
      * @return true if request successful false otherwise
      */
-    bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId>& grpIds);
+    bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId>& grpIds) override;
 
     /*!
      * For requesting msgs related to a given msg id within a group
@@ -96,7 +99,7 @@ public:
      * @param groupIds The ids of the groups to get, second entry of map empty to query for all msgs
      * @return true if request successful false otherwise
      */
-    bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::vector<RsGxsGrpMsgIdPair> &msgIds);
+    bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::vector<RsGxsGrpMsgIdPair> &msgIds) override;
 
     /*!
      * This request statistics on amount of data held
@@ -107,19 +110,20 @@ public:
      * total size of messages
      * total size of groups
      * @param token
+     * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
      */
-    void requestServiceStatistic(uint32_t& token);
+    void requestServiceStatistic(uint32_t& token, const RsTokReqOptions &opts) override;
 
 	/*!
 	 * To request statistic on a group
 	 * @param token set to value to be redeemed to get statistic
 	 * @param grpId the id of the group
+     * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
 	 */
-    void requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId);
-
+    void requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId, const RsTokReqOptions &opts) override;
 
     /* Poll */
-	GxsRequestStatus requestStatus(const uint32_t token);
+	GxsRequestStatus requestStatus(uint32_t token);
 
     /* Cancel Request */
     bool cancelRequest(const uint32_t &token);
@@ -200,7 +204,8 @@ public:
      * @param token request token to be redeemed
      * @param msgIds
      */
-    bool getMsgList(const uint32_t &token, GxsMsgIdResult &msgIds);
+    bool getMsgIdList(const uint32_t &token, GxsMsgIdResult &msgIds);
+
 
     /*!
      * Retrieve msg list for a given token for message related info
@@ -271,7 +276,7 @@ private:
      * @param token the value of the token for the request object handle wanted
      * @return the request associated to this token
      */
-    GxsRequest* locked_retrieveRequest(const uint32_t& token);
+    GxsRequest* locked_retrieveCompetedRequest(const uint32_t& token);
 
     /*!
      * Add a gxs request to queue
@@ -323,11 +328,11 @@ private:
      */
     void tokenList(std::list<uint32_t> &tokens);
 
-    /*!
-     * Convenience function to delete the ids
-     * @param filter the meta filter to clean
-     */
-    void cleanseMsgMetaMap(GxsMsgMetaResult& result);
+    //  /*!
+    //   * Convenience function to delete the ids
+    //   * @param filter the meta filter to clean
+    //   */
+    //  void cleanseMsgMetaMap(GxsMsgMetaResult& result);
 
 public:
 
@@ -378,8 +383,18 @@ private:
      * @param req
      * @return false if unsuccessful, true otherwise
      */
-    bool getMsgList(MsgIdReq* req);
+    bool getMsgIdList(MsgIdReq* req);
 
+    /*!
+     * Attempts to retrieve msg Meta list from data store
+     * Computationally/CPU-Bandwidth expensive
+     *
+     * @param msgIds List of message Ids for the Message Metas to retrieve
+     * @param opts   GxsRequest options
+     * @param result Map of Meta information for messages
+     *
+     */
+	bool getMsgMetaDataList( const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgMetaResult& result );
 
     /*!
      * Attempts to retrieve group meta data from data store
@@ -445,7 +460,7 @@ private:
      * @param opts the request options set by user
      * @param meta The accompanying meta information for msg, ids
      */
-    void filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const;
+    void filterMsgIdList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const;
 
     /*!
      * This filter msgs based of options supplied (at the moment just status masks)
@@ -482,9 +497,10 @@ private:
      * @param opts the options used to parameterise the id filter
      * @param msgIdsOut the left overs ids after filter is applied to msgIds
      */
-    bool getMsgList(const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgReq& msgIdsOut);
+    bool getMsgIdList(const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgReq& msgIdsOut);
 
 private:
+    bool locked_clearRequest(const uint32_t &token);
 
     RsGeneralDataService* mDataStore;
 
@@ -492,10 +508,9 @@ private:
 
     uint32_t mNextToken;
 	std::map<uint32_t, GxsRequestStatus> mPublicToken;
-    std::map<uint32_t, GxsRequest*> mRequests;
-
-
 
+    std::set<std::pair<uint32_t,GxsRequest*> > mRequestQueue;
+    std::map<uint32_t, GxsRequest*> mCompletedRequests;
 };
 
 #endif // RSGXSDATAACCESS_H

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -272,6 +272,7 @@
     NXS_NET_DEBUG_6		group sync statistics (e.g. number of posts at nighbour nodes, etc)
  	NXS_NET_DEBUG_7		encryption/decryption of transactions
  	NXS_NET_DEBUG_8		gxs distant sync
+ 	NXS_NET_DEBUG_9		gxs distant search
 
  ***/
 //#define NXS_NET_DEBUG_0 	1
@@ -283,6 +284,7 @@
 //#define NXS_NET_DEBUG_6 	1
 //#define NXS_NET_DEBUG_7 	1
 //#define NXS_NET_DEBUG_8 	1
+//#define NXS_NET_DEBUG_9 	1
 
 //#define NXS_FRAG
 
@@ -306,6 +308,7 @@ static const uint32_t GROUP_STATS_UPDATE_DELAY                =          240; //
 static const uint32_t GROUP_STATS_UPDATE_NB_PEERS             =            2; // number of peers to which the group stats are asked
 static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE            =       199000; // 200,000 bytes including signature and headers
 static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH          =           40; // dont search same group more than every 40 secs.
+static const uint32_t SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE    =         1800; // avoid re-sending the same msg list to a peer who asks twice for the same update in less than this time
 
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN             = 0x00 ;
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR            = 0x01 ;
@@ -318,11 +321,11 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING     = 0x05 ;
 
 #if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2)  || defined(NXS_NET_DEBUG_3) \
  || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6)  || defined(NXS_NET_DEBUG_7) \
- || defined(NXS_NET_DEBUG_8)
+ || defined(NXS_NET_DEBUG_8) || defined(NXS_NET_DEBUG_9)
 
-static const RsPeerId     peer_to_print     = RsPeerId(std::string(""))   ;
-static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ;	// use this to allow to this group id only, or "" for all IDs
-static const uint32_t     service_to_print  = RS_SERVICE_GXS_TYPE_CHANNELS ;                       	// use this to allow to this service id only, or 0 for all services
+static const RsPeerId     peer_to_print     = RsPeerId();//std::string("a97fef0e2dc82ddb19200fb30f9ac575"))   ;
+static const RsGxsGroupId group_id_to_print = RsGxsGroupId();//std::string("66052380f5d1d0c5992e2b55dc402ce6")) ;	// use this to allow to this group id only, or "" for all IDs
+static const uint32_t     service_to_print  = RS_SERVICE_GXS_TYPE_GXSID;                       	// use this to allow to this service id only, or 0 for all services
 										// warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums)
 
 class nullstream: public std::ostream {};
@@ -590,32 +593,30 @@ void RsGxsNetService::syncWithPeers()
         return;
     }
 
-    std::set<RsPeerId>::iterator sit = peers.begin();
+	// for now just grps
+	for(auto sit = peers.begin(); sit != peers.end(); ++sit)
+	{
 
-    // for now just grps
-    for(; sit != peers.end(); ++sit)
-    {
+		const RsPeerId peerId = *sit;
 
-        const RsPeerId peerId = *sit;
+		ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId);
+		uint32_t updateTS = 0;
 
-        ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId);
-        uint32_t updateTS = 0;
-
-        if(cit != mClientGrpUpdateMap.end())
-        {
-            const RsGxsGrpUpdate *gui = &cit->second;
-            updateTS = gui->grpUpdateTS;
-        }
-        RsNxsSyncGrpReqItem *grp = new RsNxsSyncGrpReqItem(mServType);
-        grp->clear();
-        grp->PeerId(*sit);
-        grp->updateTS = updateTS;
+		if(cit != mClientGrpUpdateMap.end())
+		{
+			const RsGxsGrpUpdate *gui = &cit->second;
+			updateTS = gui->grpUpdateTS;
+		}
+		RsNxsSyncGrpReqItem *grp = new RsNxsSyncGrpReqItem(mServType);
+		grp->clear();
+		grp->PeerId(*sit);
+		grp->updateTS = updateTS;
 
 #ifdef NXS_NET_DEBUG_5
-	GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << "  sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl;
+		GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << "  sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl;
 #endif
-        generic_sendItem(grp);
-    }
+		generic_sendItem(grp);
+	}
 
     if(!mAllowMsgSync)
         return ;
@@ -641,15 +642,13 @@ void RsGxsNetService::syncWithPeers()
 	    }
     }
 
-    sit = peers.begin();
-
     // Synchronise group msg for groups which we're subscribed to
     // For each peer and each group, we send to the peer the time stamp of the most
     // recent modification the peer has sent. If the peer has more recent messages he will send them, because its latest
     // modifications will be more recent. This ensures that we always compare timestamps all taken in the same
     // computer (the peer's computer in this case)
 
-    for(; sit != peers.end(); ++sit)
+    for(auto sit = peers.begin(); sit != peers.end(); ++sit)
     {
         const RsPeerId& peerId = *sit;
 
@@ -953,7 +952,7 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs)
 #endif
 	    mDataStore->retrieveGxsMsgMetaData(reqIds, result);
 
-	    const std::vector<RsGxsMsgMetaData*>& vec(result[grs->grpId]) ;
+	    const std::vector<const RsGxsMsgMetaData*>& vec(result[grs->grpId]) ;
 
 	    if(vec.empty())	// that means we don't have any, or there isn't any, but since the default is always 0, no need to send.
 		    return ;
@@ -970,12 +969,9 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs)
 														// be used to discard groups that are not used.
 
 	    for(uint32_t i=0;i<vec.size();++i)
-	    {
 		    if(grs_resp->last_post_TS < vec[i]->mPublishTs)
 			    grs_resp->last_post_TS = vec[i]->mPublishTs;
 
-		    delete vec[i] ;
-	    }
 #ifdef NXS_NET_DEBUG_6
 	    GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "  sending back statistics item with " << vec.size() << " elements." << std::endl;
 #endif
@@ -2953,21 +2949,19 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr)
     reqIds[grpId] = std::set<RsGxsMessageId>();
     GxsMsgMetaResult result;
     mDataStore->retrieveGxsMsgMetaData(reqIds, result);
-    std::vector<RsGxsMsgMetaData*> &msgMetaV = result[grpId];
+    std::vector<const RsGxsMsgMetaData*> &msgMetaV = result[grpId];
 
 #ifdef NXS_NET_DEBUG_1
     GXSNETDEBUG_PG(item->PeerId(),grpId) << "  retrieving grp message list..." << std::endl;
     GXSNETDEBUG_PG(item->PeerId(),grpId) << "  grp locally contains " << msgMetaV.size() << " messsages." << std::endl;
 #endif
-    std::vector<RsGxsMsgMetaData*>::const_iterator vit = msgMetaV.begin();
+    std::vector<const RsGxsMsgMetaData*>::const_iterator vit = msgMetaV.begin();
     std::set<RsGxsMessageId> msgIdSet;
 
     // put ids in set for each searching
     for(; vit != msgMetaV.end(); ++vit)
-    {
         msgIdSet.insert((*vit)->mMsgId);
-        delete(*vit);
-    }
+
     msgMetaV.clear();
 
 #ifdef NXS_NET_DEBUG_1
@@ -3185,7 +3179,8 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr)
         {
             grpItemL.push_back(item);
             grpMetaMap[item->grpId] = NULL;
-        }else
+        }
+        else
         {
 #ifdef NXS_NET_DEBUG_0
             GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "RsGxsNetService::genReqGrpTransaction(): item failed to caste to RsNxsSyncMsgItem* " << std::endl;
@@ -3228,7 +3223,7 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr)
         RsNxsSyncGrpItem*& grpSyncItem = *llit;
         const RsGxsGroupId& grpId = grpSyncItem->grpId;
 
-	std::map<RsGxsGroupId, RsGxsGrpMetaData*>::const_iterator metaIter = grpMetaMap.find(grpId);
+		std::map<RsGxsGroupId, RsGxsGrpMetaData*>::const_iterator metaIter = grpMetaMap.find(grpId);
         bool haveItem = false;
         bool latestVersion = false;
 
@@ -3239,19 +3234,21 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr)
         }
         // FIXTESTS global variable rsReputations not available in unittests!
 
-#warning csoler 2016-12-23: Update the code below to correctly send/recv dependign on reputation
-		if( !grpSyncItem->authorId.isNull() &&
-		        mReputations->overallReputationLevel(grpSyncItem->authorId) ==
-		        RsReputationLevel::LOCALLY_NEGATIVE )
+		if( mReputations->overallReputationLevel(RsGxsId(grpSyncItem->grpId)) == RsReputationLevel::LOCALLY_NEGATIVE )
 		{
 #ifdef NXS_NET_DEBUG_0
-			GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "  Identity " << grpSyncItem->authorId << " is banned. Not syncing group." << std::endl;
+			GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "  Identity " << grpSyncItem->grpId << " is banned. Not GXS-syncing group." << std::endl;
 #endif
 			continue ;
 		}
 
         if( (mGrpAutoSync && !haveItem) || latestVersion)
+        {
+#ifdef NXS_NET_DEBUG_0
+			GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "  Identity " << grpId << " will be sync-ed using GXS. mGrpAutoSync:" << mGrpAutoSync << " haveItem:" << haveItem << " latest_version: " << latestVersion << std::endl;
+#endif
 			addGroupItemToList(tr, grpId, transN, reqList);
+        }
     }
 
     if(!reqList.empty())
@@ -3598,6 +3595,10 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr)
 		    msg->count = 1;	// only one piece. This is to keep compatibility if we ever implement fragmenting in the future.
 		    msg->pos = 0;
 
+#ifdef NXS_NET_DEBUG_0
+			GXSNETDEBUG_PG(tr->mTransaction->PeerId(),msg->grpId) << "   sending msg Id " << msg->msgId << " in Group " << msg->grpId << std::endl;
+#endif
+
 		    newTr->mItems.push_back(msg);
 		    msgSize++;
 #endif
@@ -3994,7 +3995,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item)
 	GXSNETDEBUG_P_(item->PeerId()) << "  local modification time stamp: " << std::dec<< time(NULL) - mGrpServerUpdate.grpUpdateTS << " secs ago. Update sent: " <<
 	                                  ((item->updateTS < mGrpServerUpdate.grpUpdateTS)?"YES":"NO")  << std::endl;
 #endif
-	return item->updateTS < mGrpServerUpdate.grpUpdateTS;
+	return item->updateTS < mGrpServerUpdate.grpUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),RsGxsGroupId(),item->updateTS,mGrpServerUpdate.grpUpdateTsRecords[item->PeerId()]) ;
 }
 
 void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrpReqItem *item)
@@ -4239,10 +4240,33 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs
     return true;
 }
 
+bool RsGxsNetService::locked_checkResendingOfUpdates(const RsPeerId& pid,const RsGxsGroupId& grpId,rstime_t incoming_ts,RsPeerUpdateTsRecord& rec)
+{
+	rstime_t now = time(NULL);
+
+	// Now we check if the peer is sending the same outdated TS for the same time in a short while. This would mean the peer
+	// hasn't finished processing the updates we're sending and we shouldn't send new data anymore. Of course the peer might
+	// have disconnected or so, which means that we need to be careful about not sending. As a compromise we still send, but
+	// after waiting for a while (See
+
+	if(rec.mLastTsReceived == incoming_ts && rec.mTs + SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE > now)
+    {
+#ifdef NXS_NET_DEBUG_0
+		GXSNETDEBUG_PG(pid,grpId) << "(II) peer " << pid << " already sent the same TS " << (long int)now-(long int)rec.mTs << " secs ago for that group ID. Will not send msg list again for a while to prevent clogging..." << std::endl;
+#endif
+		return false;
+    }
+
+	rec.mLastTsReceived = incoming_ts;
+	rec.mTs = now;
+
+	return true;
+}
+
 bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& grp_is_known)
 {
     // Do we have new updates for this peer?
-    // Here we compare times in the same clock: the friend's clock, so it should be fine.
+    // Here we compare times in the same clock: our own clock, so it should be fine.
 
     grp_is_known = false ;
 
@@ -4251,7 +4275,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr
 	    // Item contains the hashed group ID in order to protect is from friends who don't know it. So we de-hash it using bruteforce over known group IDs for this peer.
 	    // We could save the de-hash result. But the cost is quite light, since the number of encrypted groups per service is usually low.
 
-	    for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
+	    for(ServerMsgMap::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
 		    if(item->grpId == hashGrpId(it->first,item->PeerId()))
 		    {
 			    item->grpId = it->first ;
@@ -4261,20 +4285,25 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr
 #endif
 			    grp_is_known = true ;
 
-			    return item->updateTS < it->second.msgUpdateTS ;
+                // The order of tests below is important because we want to only modify the map of requests records if the request actually is a valid requests instead of
+                // a simple check that nothing's changed.
+
+			    return item->updateTS < it->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,it->second.msgUpdateTsRecords[item->PeerId()]) ;
 		    }
 
 	    return false ;
     }
 
-    ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId);
+    ServerMsgMap::iterator cit = mServerMsgUpdateMap.find(item->grpId);
+
     if(cit != mServerMsgUpdateMap.end())
     {
 #ifdef NXS_NET_DEBUG_0
 	    GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "  local time stamp: " << std::dec<< time(NULL) - cit->second.msgUpdateTS << " secs ago. Update sent: " << (item->updateTS < cit->second.msgUpdateTS) << std::endl;
 #endif
 	    grp_is_known = true ;
-	    return item->updateTS < cit->second.msgUpdateTS ;
+
+		return item->updateTS < cit->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,cit->second.msgUpdateTsRecords[item->PeerId()]) ;
     }
 
 #ifdef NXS_NET_DEBUG_0
@@ -4295,6 +4324,9 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
     bool grp_is_known = false;
     bool was_circle_protected = item_was_encrypted || bool(item->flag & RsNxsSyncMsgReqItem::FLAG_USE_HASHED_GROUP_ID);
 
+    // This call determines if the peer can receive updates from us, meaning that our last TS is larger than what the peer sent.
+    // It also changes the items' group id into the un-hashed group ID if the group is a distant group.
+
     bool peer_can_receive_update = locked_CanReceiveUpdate(item, grp_is_known);
 
     if(item_was_encrypted)
@@ -4310,7 +4342,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 
     // We update suppliers in two cases:
     // Case 1: the grp is known because it is the hash of an existing group, but it's not yet in the server config map
-    // Case 2: the gtp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only
+    // Case 2: the grp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only
     // 		 gathered from known suppliers. So statistics never add new suppliers. These are only added here.
 
     if(grp_is_known || mServerGrpConfigMap.find(item->grpId)!=mServerGrpConfigMap.end())
@@ -4363,7 +4395,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 
     GxsMsgMetaResult metaResult;
     mDataStore->retrieveGxsMsgMetaData(req, metaResult);
-    std::vector<RsGxsMsgMetaData*>& msgMetas = metaResult[item->grpId];
+    std::vector<const RsGxsMsgMetaData*>& msgMetas = metaResult[item->grpId];
 
 #ifdef NXS_NET_DEBUG_0
     GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "   retrieving message meta data." << std::endl;
@@ -4391,9 +4423,9 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 
     if(canSendMsgIds(msgMetas, *grpMeta, peer, should_encrypt_to_this_circle_id))
     {
-	    for(std::vector<RsGxsMsgMetaData*>::iterator vit = msgMetas.begin();vit != msgMetas.end(); ++vit)
+	    for(auto vit = msgMetas.begin();vit != msgMetas.end(); ++vit)
 		{
-			RsGxsMsgMetaData* m = *vit;
+			const RsGxsMsgMetaData* m = *vit;
 
             // Check reputation
 
@@ -4412,7 +4444,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 				if(details.mReputation.mOverallReputationLevel < minReputationForForwardingMessages(grpMeta->mSignFlags, details.mFlags))
 				{
 #ifdef NXS_NET_DEBUG_0
-					GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " not sending item ID " << (*vit)->mMsgId << ", because the author is flags " << std::hex << details.mFlags << std::dec << " and reputation level " << details.mReputation.mOverallReputationLevel << std::endl;
+					GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " not sending item ID " << (*vit)->mMsgId << ", because the author is flags " << std::hex << details.mFlags << std::dec << " and reputation level " << (int) details.mReputation.mOverallReputationLevel << std::endl;
 #endif
 					continue ;
 				}
@@ -4493,8 +4525,8 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 
 
     // release meta resource
-    for(std::vector<RsGxsMsgMetaData*>::iterator vit = msgMetas.begin(); vit != msgMetas.end(); ++vit)
-	    delete *vit;
+	//   for(std::vector<RsGxsMsgMetaData*>::iterator vit = msgMetas.begin(); vit != msgMetas.end(); ++vit)
+	//     delete *vit;
 }
 
 void RsGxsNetService::locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, const RsPeerId& sslId, const RsGxsGroupId& grp_id,const uint32_t& transN)
@@ -4538,7 +4570,7 @@ void RsGxsNetService::locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, c
 	}
 }
 
-bool RsGxsNetService::canSendMsgIds(std::vector<RsGxsMsgMetaData*>& msgMetas, const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId,RsGxsCircleId& should_encrypt_id)
+bool RsGxsNetService::canSendMsgIds(std::vector<const RsGxsMsgMetaData*>& msgMetas, const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId,RsGxsCircleId& should_encrypt_id)
 {
 #ifdef NXS_NET_DEBUG_4
     GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl;
@@ -4605,7 +4637,7 @@ bool RsGxsNetService::canSendMsgIds(std::vector<RsGxsMsgMetaData*>& msgMetas, co
                 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "   deleting MsgMeta entry for msg ID " << msgMetas[i]->mMsgId << " signed by " << msgMetas[i]->mAuthorId << " who is not in group circle " << circleId << std::endl;
 #endif
 
-                delete msgMetas[i] ;
+                //delete msgMetas[i] ;
                 msgMetas[i] = msgMetas[msgMetas.size()-1] ;
                 msgMetas.pop_back() ;
             }
@@ -5156,7 +5188,7 @@ static bool termSearch(const std::string& src, const std::string& substring)
 }
 #endif // ndef RS_DEEP_CHANNEL_INDEX
 
-bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
+bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
     RS_STACK_MUTEX(mNxsMutex) ;
 
@@ -5168,7 +5200,7 @@ bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<
     group_infos = it->second;
     return true ;
 }
-bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs)
+bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSearchResults& gs)
 {
 	RS_STACK_MUTEX(mNxsMutex) ;
     for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it)
@@ -5190,8 +5222,7 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
     return true ;
 }
 
-void RsGxsNetService::receiveTurtleSearchResults(
-        TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
+void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
 {
 	std::set<RsGxsGroupId> groupsToNotifyResults;
 
@@ -5199,20 +5230,43 @@ void RsGxsNetService::receiveTurtleSearchResults(
 		RS_STACK_MUTEX(mNxsMutex);
 
 		RsGxsGrpMetaTemporaryMap grpMeta;
-		std::map<RsGxsGroupId,RsGxsGroupSummary>&
-		        search_results_map(mDistantSearchResults[req]);
+		std::map<RsGxsGroupId,RsGxsGroupSearchResults>& search_results_map(mDistantSearchResults[req]);
+
+#ifdef NXS_NET_DEBUG_9
+        std::cerr << "Received group summary through turtle search for the following groups:" << std::endl;
+#endif
 
 		for(const RsGxsGroupSummary& gps : group_infos)
-			if(search_results_map.find(gps.mGroupId) == search_results_map.end())
-				grpMeta[gps.mGroupId] = nullptr;
+        {
+            std::cerr <<"  " << gps.mGroupId << "  \"" << gps.mGroupName << "\"" << std::endl;
+			grpMeta[gps.mGroupId] = nullptr;
+        }
+
 		mDataStore->retrieveGxsGrpMetaData(grpMeta);
 
+#ifdef NXS_NET_DEBUG_9
+        std::cerr << "Retrieved data store group data for the following groups:" <<std::endl;
+        for(auto& it:grpMeta)
+            std::cerr << "  " << it.first << " : " << it.second->mGroupName << std::endl;
+#endif
+
 		for (const RsGxsGroupSummary& gps : group_infos)
 		{
 #ifndef RS_DEEP_CHANNEL_INDEX
 			/* Only keep groups that are not locally known, and groups that are
-			 * not already in the mDistantSearchResults structure. */
-			if(grpMeta[gps.mGroupId]) continue;
+			 * not already in the mDistantSearchResults structure.
+			 * mDataStore may in some situations allocate an empty group meta data, so it's important
+			 * to test that the group meta is both non null and actually corresponds to the group id we seek. */
+
+            auto& meta(grpMeta[gps.mGroupId]);
+
+			if(meta != nullptr && meta->mGroupId == gps.mGroupId)
+                continue;
+
+#ifdef NXS_NET_DEBUG_9
+            std::cerr << "  group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
+#endif
+
 #else // ndef RS_DEEP_CHANNEL_INDEX
 			/* When deep search is enabled search results may bring more info
 			 * then we already have also about post that are indexed by xapian,
@@ -5221,22 +5275,32 @@ void RsGxsNetService::receiveTurtleSearchResults(
 			const RsGxsGroupId& grpId(gps.mGroupId);
 
 			groupsToNotifyResults.insert(grpId);
-			auto it2 = search_results_map.find(grpId);
-			if(it2 != search_results_map.end())
-			{
-				// update existing data
-				RsGxsGroupSummary& eGpS(it2->second);
-				eGpS.mPopularity++;
-				eGpS.mNumberOfMessages = std::max(
-				            eGpS.mNumberOfMessages,
-				            gps.mNumberOfMessages );
-			}
-			else
-			{
-				search_results_map[grpId] = gps;
-				// number of results so far
-				search_results_map[grpId].mPopularity = 1;
-			}
+
+            // Find search results place for this particular group
+
+#ifdef NXS_NET_DEBUG_9
+            std::cerr << "  Adding gps=" << gps.mGroupId << " name=\"" << gps.mGroupName << "\" gps.mSearchContext=\"" << gps.mSearchContext << "\"" << std::endl;
+#endif
+			RsGxsGroupSearchResults& eGpS(search_results_map[grpId]);
+
+            if(eGpS.mGroupId != grpId)	// not initialized yet. So we do it now.
+            {
+                eGpS.mGroupId   = gps.mGroupId;
+				eGpS.mGroupName = gps.mGroupName;
+				eGpS.mAuthorId  = gps.mAuthorId;
+				eGpS.mPublishTs = gps.mPublishTs;
+				eGpS.mSignFlags = gps.mSignFlags;
+            }
+            // We should check that the above values are always the same for all info that is received. In the end, we'll
+            // request the group meta and check the signature, but it may be misleading to receive a forged information
+            // that is not the real one.
+
+            ++eGpS.mPopularity;	// increase popularity. This is not a real counting, but therefore some heuristic estimate.
+            eGpS.mNumberOfMessages = std::max( eGpS.mNumberOfMessages, gps.mNumberOfMessages );
+			eGpS.mLastMessageTs    = std::max( eGpS.mLastMessageTs, gps.mLastMessageTs );
+
+            if(gps.mSearchContext != gps.mGroupName)			// this is a bit of a hack. We should have flags to tell where the search hit happens
+				eGpS.mSearchContexts.insert(gps.mSearchContext);
 		}
 	} // end RS_STACK_MUTEX(mNxsMutex);
 

libretroshare/src/gxs/rsgxsnetservice.h

@@ -140,9 +140,9 @@ public:
 	virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos);
 	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len);
 
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos);
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos);
 	virtual bool clearDistantSearchResults(const TurtleRequestId& id);
-    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&);
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&, RsGxsGroupSearchResults &);
 
     /*!
      * pauses synchronisation of subscribed groups and request for group id
@@ -395,7 +395,7 @@ private:
      * @return false, if you cannot send to this peer, true otherwise
      */
     bool canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData& grpMeta, std::vector<GrpIdCircleVet>& toVet, bool &should_encrypt);
-    bool canSendMsgIds(std::vector<RsGxsMsgMetaData*>& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId, RsGxsCircleId &should_encrypt_id);
+    bool canSendMsgIds(std::vector<const RsGxsMsgMetaData*>& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId, RsGxsCircleId &should_encrypt_id);
 
     /*!
      * \brief checkPermissionsForFriendGroup
@@ -439,6 +439,7 @@ private:
     bool locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item);
     bool locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item, bool &grp_is_known);
 	void locked_resetClientTS(const RsGxsGroupId& grpId);
+	bool locked_checkResendingOfUpdates(const RsPeerId& pid, const RsGxsGroupId &grpId, rstime_t incoming_ts, RsPeerUpdateTsRecord& rec);
 
     static RsGxsGroupId hashGrpId(const RsGxsGroupId& gid,const RsPeerId& pid) ;
     
@@ -609,7 +610,7 @@ private:
     std::set<RsGxsGroupId> mNewPublishKeysToNotify ;
 
     // Distant search result map
-    std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSummary> > mDistantSearchResults ;
+    std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSearchResults> > mDistantSearchResults ;
 
     void debugDump();
 

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -167,7 +167,12 @@ public:
 class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
 {
 public:
-    explicit RsGxsNetTunnelTurtleSearchGroupDataItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA) {}
+    explicit RsGxsNetTunnelTurtleSearchGroupDataItem()
+        : RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA),
+          encrypted_group_data(NULL),
+          encrypted_group_data_len(0)
+    {}
+
     virtual ~RsGxsNetTunnelTurtleSearchGroupDataItem() {}
 
     uint16_t service ;
@@ -1090,8 +1095,10 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id
 	{
 		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
 
+#ifdef DEBUG_RSGXSNETTUNNEL
 		for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
 			std::cerr << "   group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
+#endif
 
 		auto it = mSearchableServices.find(result_gs->service) ;
 

libretroshare/src/gxs/rsgxsnotify.h

@@ -0,0 +1,97 @@
+/*******************************************************************************
+ * libretroshare/src/gxs/: rsgxsnotify.h                                       *
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright (C) 2015  Retroshare Team <retroshare.project@gmail.com>          *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#pragma once
+
+/*!
+ * The aim of this class is to implement notifications internally to GXS, which are
+ * mostly used by RsGenExchange to send information to specific services. These services
+ * then interpret these changes and turn them into human-readable/processed service-specific changes.
+ */
+
+#include "retroshare/rsids.h"
+
+class RsGxsNotify
+{
+public:
+    RsGxsNotify(const RsGxsGroupId& gid): mGroupId(gid){}
+    virtual ~RsGxsNotify()=default;
+
+	enum NotifyType
+	{
+        TYPE_UNKNOWN                           = 0x00,
+        TYPE_PUBLISHED                         = 0x01,
+        TYPE_RECEIVED_NEW                      = 0x02,
+        TYPE_PROCESSED                         = 0x03,
+        TYPE_RECEIVED_PUBLISHKEY               = 0x04,
+        TYPE_RECEIVED_DISTANT_SEARCH_RESULTS   = 0x05,
+        TYPE_STATISTICS_CHANGED                = 0x06,
+        TYPE_UPDATED                           = 0x07,
+        TYPE_MESSAGE_DELETED                   = 0x08,
+        TYPE_GROUP_DELETED                     = 0x09,
+	};
+
+	virtual NotifyType getType() = 0;
+
+    RsGxsGroupId mGroupId;			// Group id of the group we're talking about. When the group is deleted, it's useful to know which group
+    								// that was although there is no pointers to the actual group data anymore.
+};
+
+/*!
+ * Relevant to group changes
+ */
+class RsGxsGroupChange : public RsGxsNotify
+{
+public:
+	RsGxsGroupChange(NotifyType type, const RsGxsGroupId& gid,bool metaChange) : RsGxsNotify(gid),mNewGroupItem(nullptr),mOldGroupItem(nullptr), mNotifyType(type), mMetaChange(metaChange) {}
+    virtual ~RsGxsGroupChange() override { delete mOldGroupItem; delete mNewGroupItem ; }
+
+    NotifyType getType() override { return mNotifyType;}
+    bool metaChange() { return mMetaChange; }
+
+    RsGxsGrpItem *mNewGroupItem;	// Valid when a group has changed, or a new group is received.
+    RsGxsGrpItem *mOldGroupItem;	// only valid when mNotifyType is TYPE_UPDATED
+
+protected:
+    NotifyType mNotifyType;
+    bool mMetaChange;
+};
+
+/*!
+ * Relevant to message changes
+ */
+class RsGxsMsgChange : public RsGxsNotify
+{
+public:
+	RsGxsMsgChange(NotifyType type, const RsGxsGroupId& gid, const RsGxsMessageId& msg_id,bool metaChange)
+        : RsGxsNotify(gid), mMsgId(msg_id), mNewMsgItem(nullptr),NOTIFY_TYPE(type), mMetaChange(metaChange) {}
+
+    RsGxsMessageId mMsgId;
+    RsGxsMsgItem *mNewMsgItem;
+
+	NotifyType getType(){ return NOTIFY_TYPE;}
+    bool metaChange() { return mMetaChange; }
+private:
+    const NotifyType NOTIFY_TYPE;
+    bool mMetaChange;
+};
+

libretroshare/src/gxs/rsgxsrequesttypes.cc

@@ -23,6 +23,128 @@
 #include "rsgxsrequesttypes.h"
 #include "util/rsstd.h"
 
+std::ostream& operator<<(std::ostream& o,const GxsRequest& g)
+{
+    return g.print(o);
+}
+
+
+std::ostream& GroupMetaReq::print(std::ostream& o) const
+{
+	o << "[Request type=GroupMeta groupIds (size=" << mGroupIds.size() << "): " ;
+
+    if(!mGroupIds.empty())
+    {
+        o << *mGroupIds.begin() ;
+
+		if(mGroupIds.size() > 1)
+			o << " ..." ;
+    }
+
+	o << "]" ;
+
+	return o;
+}
+std::ostream& GroupIdReq::print(std::ostream& o) const
+{
+    return o << "[Request type=GroupIdReq" << "]" ;
+}
+
+std::ostream& GroupSerializedDataReq::print(std::ostream& o) const
+{
+    return o << "[Request type=GroupSerializedData" << "]" ;
+}
+
+std::ostream& GroupDataReq::print(std::ostream& o) const
+{
+	o << "[Request type=GroupDataReq groupIds (size=" << mGroupIds.size() << "): " ;
+
+    if(!mGroupIds.empty())
+	{
+		o << *mGroupIds.begin() ;
+
+		if(mGroupIds.size() > 1)
+			o << " ..." ;
+	}
+
+	o << "]" ;
+
+	return o;
+}
+
+std::ostream& MsgIdReq::print(std::ostream& o) const
+{
+    return o << "[Request type=MsgId" << "]" ;
+}
+
+std::ostream& MsgMetaReq::print(std::ostream& o) const
+{
+	o << "[Request type=MsgMetaReq groups (size=" << mMsgIds.size() << "): " ;
+
+    if(!mMsgIds.empty())
+    {
+        o << mMsgIds.begin()->first << " (" << mMsgIds.begin()->second.size() << " messages)";
+
+		if(mMsgIds.size() > 1)
+			o << " ..." ;
+    }
+
+	o << "]" ;
+
+	return o;
+}
+
+std::ostream& MsgDataReq::print(std::ostream& o) const
+{
+	o << "[Request type=MsgDataReq groups (size=" << mMsgIds.size() << "): " ;
+
+    if(!mMsgIds.empty())
+    {
+        o << mMsgIds.begin()->first << " (" << mMsgIds.begin()->second.size() << " messages)";
+
+		if(mMsgIds.size() > 1)
+			o << " ..." ;
+    }
+
+	o << "]" ;
+
+	return o;
+}
+
+std::ostream& MsgRelatedInfoReq::print(std::ostream& o) const
+{
+	o << "[Request type=MsgRelatedInfo msgIds (size=" << mMsgIds.size() << "): " ;
+
+    if(!mMsgIds.empty())
+    {
+        o << mMsgIds.begin()->first ;
+
+		if(mMsgIds.size() > 1)
+			o << " ..." ;
+    }
+
+	o << "]" ;
+
+	return o;
+}
+
+std::ostream& GroupSetFlagReq::print(std::ostream& o) const
+{
+	return o << "[Request type=GroupFlagSet grpId=" <<  grpId << "]" ;
+}
+
+
+
+std::ostream& ServiceStatisticRequest::print(std::ostream& o) const
+{
+    return o << "[Request type=ServiceStatistics" << "]" ;
+}
+
+std::ostream& GroupStatisticRequest::print(std::ostream& o) const
+{
+	return o << "[Request type=GroupStatistics grpId=" << mGrpId << "]" ;
+}
+
 GroupMetaReq::~GroupMetaReq()
 {
 	//rsstd::delete_all(mGroupMetaData.begin(), mGroupMetaData.end());	// now memory ownership is kept by the cache.
@@ -57,3 +179,8 @@ MsgRelatedInfoReq::~MsgRelatedInfoReq()
 		rsstd::delete_all(dataIt->second.begin(), dataIt->second.end());
 	}
 }
+std::ostream& MessageSetFlagReq::print(std::ostream& o) const
+{
+	return o << "[Request type=MsgFlagSet" << "]" ;
+}
+

libretroshare/src/gxs/rsgxsrequesttypes.h

@@ -29,25 +29,30 @@
 struct GxsRequest
 {
 	GxsRequest() :
-	    token(0), reqTime(0), ansType(0), reqType(0),
+	    token(0), reqTime(0), clientAnswerType(0), reqType(0),
 	    status(RsTokenService::FAILED) {}
 	virtual ~GxsRequest() {}
 
 	uint32_t token;
 	uint32_t reqTime;
 
-	RS_DEPRECATED uint32_t ansType; /// G10h4ck: This is of no use
+	uint32_t clientAnswerType; /// This is made available to the clients in order to keep track of why specific requests where sent..
 	uint32_t reqType;
 	RsTokReqOptions Options;
 
 	RsTokenService::GxsRequestStatus status;
+
+    virtual std::ostream& print(std::ostream& o) const = 0;
 };
 
+std::ostream& operator<<(std::ostream& o,const GxsRequest& g);
+
 class GroupMetaReq : public GxsRequest
 {
 public:
 	virtual ~GroupMetaReq();
 
+    virtual std::ostream& print(std::ostream& o) const override;
 public:
 	std::list<RsGxsGroupId> mGroupIds;
 	std::list<const RsGxsGrpMetaData*> mGroupMetaData;
@@ -56,12 +61,16 @@ public:
 class GroupIdReq : public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
+
 	std::list<RsGxsGroupId> mGroupIds;
 	std::list<RsGxsGroupId> mGroupIdResult;
 };
 class GroupSerializedDataReq : public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
+
 	std::list<RsGxsGroupId> mGroupIds;
 	std::list<RsNxsGrp*> mGroupData;
 };
@@ -71,6 +80,7 @@ class GroupDataReq : public GxsRequest
 public:
 	virtual ~GroupDataReq();
 
+	virtual std::ostream& print(std::ostream& o) const override;
 public:
 	std::list<RsGxsGroupId> mGroupIds;
 	std::list<RsNxsGrp*> mGroupData;
@@ -79,6 +89,8 @@ public:
 class MsgIdReq : public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
+
 	GxsMsgReq mMsgIds;
 	GxsMsgIdResult mMsgIdResult;
 };
@@ -88,6 +100,8 @@ class MsgMetaReq : public GxsRequest
 public:
 	virtual ~MsgMetaReq();
 
+	virtual std::ostream& print(std::ostream& o) const override;
+
 public:
 	GxsMsgReq mMsgIds;
 	GxsMsgMetaResult mMsgMetaData;
@@ -98,6 +112,7 @@ class MsgDataReq : public GxsRequest
 public:
 	virtual ~MsgDataReq();
 
+	virtual std::ostream& print(std::ostream& o) const override;
 public:
 	GxsMsgReq mMsgIds;
 	NxsMsgDataResult mMsgData;
@@ -106,12 +121,15 @@ public:
 class ServiceStatisticRequest: public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
 	GxsServiceStatistic mServiceStatistic;
 };
 
 struct GroupStatisticRequest: public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
+
 	RsGxsGroupId mGrpId;
 	GxsGroupStatistic mGroupStatistic;
 };
@@ -121,6 +139,7 @@ class MsgRelatedInfoReq : public GxsRequest
 public:
 	virtual ~MsgRelatedInfoReq();
 
+	std::ostream& print(std::ostream& o) const override;
 public:
 	std::vector<RsGxsGrpMsgIdPair> mMsgIds;
 	MsgRelatedIdResult mMsgIdResult;
@@ -131,6 +150,8 @@ public:
 class GroupSetFlagReq : public GxsRequest
 {
 public:
+    virtual std::ostream& print(std::ostream& o) const override ;
+
 	const static uint32_t FLAG_SUBSCRIBE;
 	const static uint32_t FLAG_STATUS;
 
@@ -145,6 +166,7 @@ class MessageSetFlagReq : public GxsRequest
 public:
 	const static uint32_t FLAG_STATUS;
 
+    virtual std::ostream& print(std::ostream& o) const override ;
 	uint8_t type;
 	uint32_t flag;
 	uint32_t flagMask;

libretroshare/src/gxs/rsgxsutil.cc

@@ -85,7 +85,7 @@ bool RsGxsMessageCleanUp::clean()
 
 		for(; mit != result.end(); ++mit)
 		{
-			std::vector<RsGxsMsgMetaData*>& metaV = mit->second;
+			std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
 
             // First, make a map of which message have a child message. This allows to only delete messages that dont have child messages.
             // A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the
@@ -99,7 +99,7 @@ bool RsGxsMessageCleanUp::clean()
 
 			for( uint32_t i=0;i<metaV.size();++i)
 			{
-				RsGxsMsgMetaData* meta = metaV[i];
+				const RsGxsMsgMetaData* meta = metaV[i];
 
 				bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end());
 
@@ -107,7 +107,7 @@ bool RsGxsMessageCleanUp::clean()
 				bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids;
 
 				// check client does not want the message kept regardless of age
-				remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP);
+				remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER);
 
 				// if not subscribed remove messages (can optimise this really)
 				remove = remove ||  (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED);
@@ -132,7 +132,7 @@ bool RsGxsMessageCleanUp::clean()
 					std::cerr << std::endl;
 #endif
 
-				delete meta;
+				//delete meta;
 			}
 		}
 
@@ -215,7 +215,8 @@ bool RsGxsIntegrityCheck::check()
 						        rsReputations->overallReputationLevel(
 						            grp->metaData->mAuthorId ) >
 						        RsReputationLevel::LOCALLY_NEGATIVE )
-							used_gxs_ids.insert(std::make_pair(grp->metaData->mAuthorId, RsIdentityUsage(mGenExchangeClient->serviceType(), RsIdentityUsage::GROUP_AUTHOR_KEEP_ALIVE,grp->grpId)));
+							used_gxs_ids.insert(std::make_pair(grp->metaData->mAuthorId, RsIdentityUsage(RsServiceType(mGenExchangeClient->serviceType()),
+                                                                                                         RsIdentityUsage::GROUP_AUTHOR_KEEP_ALIVE,grp->grpId)));
 					}
 				}
 			}
@@ -404,7 +405,12 @@ bool RsGxsIntegrityCheck::check()
 					        rsReputations->overallReputationLevel(
 					            msg->metaData->mAuthorId ) >
 					        RsReputationLevel::LOCALLY_NEGATIVE )
-						used_gxs_ids.insert(std::make_pair(msg->metaData->mAuthorId,RsIdentityUsage(mGenExchangeClient->serviceType(),RsIdentityUsage::MESSAGE_AUTHOR_KEEP_ALIVE,msg->metaData->mGroupId,msg->metaData->mMsgId))) ;
+						used_gxs_ids.insert(std::make_pair(msg->metaData->mAuthorId,RsIdentityUsage(RsServiceType(mGenExchangeClient->serviceType()),
+                                                                                                    RsIdentityUsage::MESSAGE_AUTHOR_KEEP_ALIVE,
+                                                                                                    msg->metaData->mGroupId,
+                                                                                                    msg->metaData->mMsgId,
+                                                                                                    msg->metaData->mParentId,
+                                                                                                    msg->metaData->mThreadId))) ;
 				}
 			}
 

libretroshare/src/gxs/rsgxsutil.h

@@ -111,27 +111,6 @@ typedef t_RsGxsGenericDataTemporaryMapVector<RsNxsMsg>                RsNxsMsgDa
 typedef t_RsGxsGenericDataTemporaryList<RsNxsGrp>                     RsNxsGrpDataTemporaryList ;
 typedef t_RsGxsGenericDataTemporaryList<RsNxsMsg>                     RsNxsMsgDataTemporaryList ;
 
-#ifdef UNUSED
-template<class T>
-class RsGxsMetaDataTemporaryMapVector: public std::vector<T*>
-{
-public:
-    virtual ~RsGxsMetaDataTemporaryMapVector()
-    {
-        clear() ;
-    }
-
-    virtual void clear()
-    {
-        for(typename RsGxsMetaDataTemporaryMapVector<T>::iterator it = this->begin();it!=this->end();++it)
-            if(it->second != NULL)
-		    delete it->second ;
-        std::vector<T*>::clear() ;
-    }
-};
-#endif
-
-
 inline RsGxsGrpMsgIdPair getMsgIdPair(RsNxsMsg& msg)
 {
 	return RsGxsGrpMsgIdPair(std::make_pair(msg.grpId, msg.msgId));
@@ -146,7 +125,7 @@ inline RsGxsGrpMsgIdPair getMsgIdPair(RsGxsMsgItem& msg)
  * Does message clean up based on individual group expirations first
  * if avialable. If not then deletion s
  */
-class RsGxsMessageCleanUp //: public RsThread
+class RsGxsMessageCleanUp
 {
 public:
 
@@ -166,11 +145,6 @@ public:
 	 */
 	bool clean();
 
-	/*!
-	 * TODO: Rather than manual progressions consider running through a thread
-	 */
-    //virtual void data_tick(){}
-
 private:
 
 	RsGeneralDataService* const mDs;

libretroshare/src/gxs/rsnxs.h

@@ -128,7 +128,7 @@ public:
      * \return
      * 			false when the request is unknown.
      */
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos)=0;
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)=0;
     /*!
      * \brief getDistantSearchResults
      * \param id
@@ -136,7 +136,7 @@ public:
      * \return
      */
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
-    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0;
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
 
     virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
 	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;

libretroshare/src/gxstrans/p3gxstrans.cc

@@ -43,7 +43,7 @@ p3GxsTrans::~p3GxsTrans()
 	}
 }
 
-bool p3GxsTrans::getStatistics(GxsTransStatistics& stats)
+bool p3GxsTrans::getDataStatistics(GxsTransStatistics& stats)
 {
 	{
 		RS_STACK_MUTEX(mDataMutex);
@@ -656,6 +656,9 @@ void p3GxsTrans::notifyChanges(std::vector<RsGxsNotify*>& changes)
 #ifdef DEBUG_GXSTRANS
 	std::cout << "p3GxsTrans::notifyChanges(...)" << std::endl;
 #endif
+	std::list<RsGxsGroupId> grps_to_request;
+    GxsMsgReq msgs_to_request;
+
 	for( auto it = changes.begin(); it != changes.end(); ++it )
 	{
 		RsGxsGroupChange* grpChange = dynamic_cast<RsGxsGroupChange *>(*it);
@@ -666,18 +669,15 @@ void p3GxsTrans::notifyChanges(std::vector<RsGxsNotify*>& changes)
 #ifdef DEBUG_GXSTRANS
 			std::cout << "p3GxsTrans::notifyChanges(...) grpChange" << std::endl;
 #endif
-			requestGroupsData(&(grpChange->mGrpIdList));
+            grps_to_request.push_back(grpChange->mGroupId);
 		}
 		else if(msgChange)
 		{
 #ifdef DEBUG_GXSTRANS
 			std::cout << "p3GxsTrans::notifyChanges(...) msgChange" << std::endl;
 #endif
-			uint32_t token;
-			RsTokReqOptions opts; opts.mReqType = GXS_REQUEST_TYPE_MSG_DATA;
-			RsGenExchange::getTokenService()->requestMsgInfo( token, 0xcaca,
-			                                   opts, msgChange->msgChangeMap );
-			GxsTokenQueue::queueRequest(token, MAILS_UPDATE);
+
+            msgs_to_request[msgChange->mGroupId].insert(msgChange->mMsgId);
 
 #ifdef DEBUG_GXSTRANS
 			for( GxsMsgReq::const_iterator it = msgChange->msgChangeMap.begin();
@@ -698,6 +698,20 @@ void p3GxsTrans::notifyChanges(std::vector<RsGxsNotify*>& changes)
 		}
         delete *it;
 	}
+
+    if(!msgs_to_request.empty())
+	{
+		uint32_t token;
+		RsTokReqOptions opts;
+        opts.mReqType = GXS_REQUEST_TYPE_MSG_DATA;
+		RsGenExchange::getTokenService()->requestMsgInfo( token, 0xcaca, opts, msgs_to_request);
+
+		GxsTokenQueue::queueRequest(token, MAILS_UPDATE);
+	}
+
+
+    if(!grps_to_request.empty())
+		requestGroupsData(&grps_to_request);
 }
 
 uint32_t p3GxsTrans::AuthenPolicy()
@@ -1335,4 +1349,53 @@ bool p3GxsTrans::acceptNewMessage(const RsGxsMsgMetaData *msgMeta,uint32_t msg_s
 }
 
 
+bool p3GxsTrans::getGroupStatistics(std::map<RsGxsGroupId,RsGxsTransGroupStatistics>& stats)
+{
+    uint32_t token1;
+
+	RsTokReqOptions opts1;
+	opts1.mReqType = GXS_REQUEST_TYPE_GROUP_META;
+	if( !requestGroupInfo(token1, opts1) || waitToken(token1) != RsTokenService::COMPLETE )
+        return false;
+
+    std::list<RsGroupMetaData> group_metas;
+    getGroupSummary(token1,group_metas);
+
+	for(auto& group_meta:group_metas)
+	{
+		RsGxsTransGroupStatistics& stat(stats[group_meta.mGroupId]);
+
+		uint32_t token2;
+		if(!RsGxsIfaceHelper::requestGroupStatistic(token2,group_meta.mGroupId) || waitToken(token2) != RsTokenService::COMPLETE)
+			continue;
+
+		RsGenExchange::getGroupStatistic(token2,stat);
+
+		stat.popularity = group_meta.mPop ;
+		stat.subscribed = IS_GROUP_SUBSCRIBED(group_meta.mSubscribeFlags) ;
+		stat.mGrpId = group_meta.mGroupId ;
+
+		std::vector<RsMsgMetaData> metas;
+
+        uint32_t token3;
+		RsTokReqOptions opts;
+		opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
+
+		std::list<RsGxsGroupId> groupIds;
+		groupIds.push_back(group_meta.mGroupId);
+
+		if( !requestMsgInfo(token3, opts, groupIds) || waitToken(token3, std::chrono::seconds(5)) != RsTokenService::COMPLETE )
+			continue;
+
+		GxsMsgMetaMap metaMap;
+		if(!RsGenExchange::getMsgMeta(token3, metaMap) || metaMap.size() != 1)
+            continue;
+
+		for(auto& meta: metaMap.begin()->second)
+			stat.addMessageMeta(group_meta.mGroupId,meta) ;
+	}
+
+	return true;
+}
+
 

libretroshare/src/gxstrans/p3gxstrans.h

@@ -113,13 +113,21 @@ public:
 
     /*!
      * \brief getStatistics
-     * 				Gathers all sorts of statistics about the internals of p3GxsTrans, in order to display info about the running status,
-     * 			message transport, etc.
+     * 				Gathers all sorts of statistics about the data transported by p3GxsTrans, in order to display info about the running status,
+     * 			message transport, etc. This is a blocking call. Use it in a thread.
      * \param stats This structure contains all statistics information.
      * \return 		true is the call succeeds.
      */
 
-	virtual bool getStatistics(GxsTransStatistics& stats);
+	virtual bool getDataStatistics(GxsTransStatistics& stats) override;
+
+    /*!
+     * \brief getGroupStatistics
+     * 				Gathers statistics about GXS groups and messages used by GxsTrans to transport data. This is a blocking call. Use it in a thread.
+     * \param stats
+     * \return true if the data collection succeeds.
+     */
+	virtual bool getGroupStatistics(std::map<RsGxsGroupId,RsGxsTransGroupStatistics>& stats) override;
 
 	/**
 	 * Send an email to recipient, in the process author of the email is

libretroshare/src/gxstrans/p3gxstransitems.h

@@ -181,8 +181,6 @@ public:
 	{}
 
 	void clear() {}
-	std::ostream &print(std::ostream &out, uint16_t /*indent = 0*/)
-	{ return out; }
 };
 
 class RsGxsTransSerializer;

libretroshare/src/gxstunnel/p3gxstunnel.cc

@@ -113,8 +113,6 @@ int p3GxsTunnelService::tick()
     
     flush() ;
 
-    rstime::rs_usleep(1000*500);
-    
     return 0 ;
 }
 
@@ -978,7 +976,7 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item)
         std::cerr << "(SS) Signature was verified and it doesn't check! This is a security issue!" << std::endl;
         return ;
     }
-    mGixs->timeStampKey(item->signature.keyId,RsIdentityUsage(RS_SERVICE_TYPE_GXS_TUNNEL,RsIdentityUsage::GXS_TUNNEL_DH_SIGNATURE_CHECK));
+    mGixs->timeStampKey(item->signature.keyId,RsIdentityUsage(RsServiceType::GXS_TUNNEL,RsIdentityUsage::GXS_TUNNEL_DH_SIGNATURE_CHECK));
 
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "  Signature checks! Sender's ID = " << senders_id << std::endl;

libretroshare/src/jsonapi/jsonapi.cpp

@@ -78,10 +78,10 @@ JsonApiServer::corsOptionsHeaders =
 #define INITIALIZE_API_CALL_JSON_CONTEXT \
 	RsGenericSerializer::SerializeContext cReq( \
 	            nullptr, 0, \
-	            RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ); \
+	            RsSerializationFlags::YIELDING ); \
 	RsJson& jReq(cReq.mJson); \
 	if(session->get_request()->get_method() == "GET") \
-    { \
+	{ \
 	    const std::string jrqp(session->get_request()->get_query_parameter("jsonData")); \
 	    jReq.Parse(jrqp.c_str(), jrqp.size()); \
 	} \
@@ -165,6 +165,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 	RsThread::setStopTimeout(10);
 #endif
 
+#if !RS_VERSION_AT_LEAST(0,6,6)
 	registerHandler("/rsLoginHelper/createLocation",
 	                [this](const std::shared_ptr<rb::Session> session)
 	{
@@ -180,6 +181,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 			std::string errorMessage;
 			bool makeHidden = false;
 			bool makeAutoTor = false;
+			std::string createToken;
 
 			// deserialize input parameters from JSON
 			{
@@ -189,6 +191,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 				RS_SERIAL_PROCESS(password);
 				RS_SERIAL_PROCESS(makeHidden);
 				RS_SERIAL_PROCESS(makeAutoTor);
+				RS_SERIAL_PROCESS(createToken);
 			}
 
 			// call retroshare C++ API
@@ -196,8 +199,9 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 			            location, password, errorMessage, makeHidden,
 			            makeAutoTor );
 
-			if(retval)
-				authorizeUser(location.mLocationId.toStdString(),password);
+			std::string tokenUser, tokenPw;
+			if(retval && parseToken(createToken, tokenUser, tokenPw))
+				authorizeUser(tokenUser,tokenPw);
 
 			// serialize out parameters and return value to JSON
 			{
@@ -212,8 +216,9 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 			DEFAULT_API_CALL_JSON_RETURN(rb::OK);
 		} );
 	}, false);
+#endif // !RS_VERSION_AT_LEAST(0,6,6)
 
-	registerHandler("/rsLoginHelper/attemptLogin",
+	registerHandler("/rsLoginHelper/createLocationV2",
 	                [this](const std::shared_ptr<rb::Session> session)
 	{
 		auto reqSize = session->get_request()->get_header("Content-Length", 0);
@@ -223,28 +228,51 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 		{
 			INITIALIZE_API_CALL_JSON_CONTEXT;
 
-			RsPeerId account;
+			RsPeerId locationId;
+			RsPgpId pgpId;
+			std::string locationName;
+			std::string pgpName;
 			std::string password;
 
+			// JSON API only
+			std::string apiUser;
+			std::string apiPass;
+
 			// deserialize input parameters from JSON
 			{
 				RsGenericSerializer::SerializeContext& ctx(cReq);
 				RsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON);
-				RS_SERIAL_PROCESS(account);
+				RS_SERIAL_PROCESS(locationId);
+				RS_SERIAL_PROCESS(pgpId);
+				RS_SERIAL_PROCESS(locationName);
+				RS_SERIAL_PROCESS(pgpName);
 				RS_SERIAL_PROCESS(password);
+
+				// JSON API only
+				RS_SERIAL_PROCESS(apiUser);
+				RS_SERIAL_PROCESS(apiPass);
 			}
 
-			// call retroshare C++ API
-			RsInit::LoadCertificateStatus retval =
-			        rsLoginHelper->attemptLogin(account, password);
+			std::error_condition retval;
 
-			if( retval == RsInit::OK )
-				authorizeUser(account.toStdString(), password);
+			if(apiUser.empty())
+				retval = RsJsonApiErrorNum::TOKEN_FORMAT_INVALID;
+
+			if(!retval)
+				retval = badApiCredientalsFormat(apiUser, apiPass);
+
+			if(!retval) // call retroshare C++ API
+				retval = rsLoginHelper->createLocationV2(
+				            locationId, pgpId, locationName, pgpName, password );
+
+			if(!retval) retval = authorizeUser(apiUser, apiPass);
 
 			// serialize out parameters and return value to JSON
 			{
 				RsGenericSerializer::SerializeContext& ctx(cAns);
 				RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);
+				RS_SERIAL_PROCESS(locationId);
+				RS_SERIAL_PROCESS(pgpId);
 				RS_SERIAL_PROCESS(retval);
 			}
 
@@ -352,7 +380,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 						rsEvents, "rsEvents", cAns, session ) )
 				return;
 
-			RsEventType eventType = RsEventType::NONE;
+			RsEventType eventType = RsEventType::__NONE;
 
 			// deserialize input parameters from JSON
 			{
@@ -395,7 +423,8 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
 				} );
 			};
 
-			bool retval = rsEvents->registerEventsHandler(eventType,multiCallback, hId);
+			std::error_condition retval = rsEvents->registerEventsHandler(
+			            multiCallback, hId, eventType );
 
 			{
 				RsGenericSerializer::SerializeContext& ctx(cAns);
@@ -465,16 +494,19 @@ void JsonApiServer::registerHandler(
 		                const std::shared_ptr<rb::Session> session,
 		                const std::function<void (const std::shared_ptr<rb::Session>)>& callback )
 		{
+			/* Declare outside the lambda to avoid returning a dangling
+			 * reference */
+			RsWarn tWarn;
 			const auto authFail =
-			        [&path, &session](int status) -> RsWarn::stream_type&
+			        [&](int status) -> std::ostream&
 			{
 				/* Capture session by reference as it is cheaper then copying
 				 * shared_ptr by value which is not needed in this case */
 
 				session->close(status, corsOptionsHeaders);
-				return RsWarn() << "JsonApiServer authentication handler "
-				                   "blocked an attempt to call JSON API "
-				                   "authenticated method: " << path;
+				return tWarn << "JsonApiServer authentication handler "
+				                "blocked an attempt to call JSON API "
+				                "authenticated method: " << path;
 			};
 
 			if(session->get_request()->get_method() == "OPTIONS")

libretroshare/src/jsonapi/jsonapi.h

@@ -151,8 +151,6 @@ protected:
 	/// @see RsThread
 	void onStopRequested() override;
 
-	static const RsJsonApiErrorCategory sErrorCategory;
-
 	static std::error_condition badApiCredientalsFormat(
 	        const std::string& user, const std::string& passwd );
 

libretroshare/src/libretroshare.pro

@@ -12,14 +12,15 @@ DESTDIR = lib
 
 !include("use_libretroshare.pri"):error("Including")
 
-# the dht stunner is used to obtain RS external ip addr. when it is natted
-# this system is unreliable and rs supports a newer and better one (asking connected peers)
-# CONFIG += useDhtStunner
-
 # treat warnings as error for better removing
 #QMAKE_CFLAGS += -Werror
 #QMAKE_CXXFLAGS += -Werror
 
+## Uncomment to enable Unfinished Services.
+#CONFIG += wikipoos
+#CONFIG += gxsthewire
+#CONFIG += gxsphotoshare
+
 debug {
 #	DEFINES *= DEBUG
 #	DEFINES *= OPENDHT_DEBUG DHT_DEBUG CONN_DEBUG DEBUG_UDP_SORTER P3DISC_DEBUG DEBUG_UDP_LAYER FT_DEBUG EXTADDRSEARCH_DEBUG
@@ -44,7 +45,6 @@ file_lists {
 			file_sharing/directory_updater.h \
 			file_sharing/rsfilelistitems.h \
 			file_sharing/dir_hierarchy.h \
-			file_sharing/file_tree.h \
 			file_sharing/file_sharing_defaults.h
 
 	SOURCES *= file_sharing/p3filelists.cc \
@@ -154,6 +154,7 @@ rs_webui {
 HEADERS += plugins/pluginmanager.h \
 		plugins/dlfcn_win32.h \
 		rsitems/rspluginitems.h \
+    util/i2pcommon.h \
     util/rsinitedptr.h
 
 HEADERS += $$PUBLIC_HEADERS
@@ -237,7 +238,7 @@ win32-x-g++ {
 }
 ################################# Windows ##########################################
 
-win32-g++ {
+win32-g++|win32-clang-g++ {
 	QMAKE_CC = $${QMAKE_CXX}
 	OBJECTS_DIR = temp/obj
 	MOC_DIR = temp/moc
@@ -327,6 +328,8 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR}
 PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a
 LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2
 
+################################### HEADERS & SOURCES #############################
+
 HEADERS +=	ft/ftchunkmap.h \
 			ft/ftcontroller.h \
 			ft/ftdata.h \
@@ -468,7 +471,12 @@ HEADERS +=	turtle/p3turtle.h \
 			turtle/turtleclientservice.h
 
 HEADERS +=	util/folderiterator.h \
-			util/rsdebug.h \
+    util/rsdebug.h \
+    util/rsdebuglevel0.h \
+    util/rsdebuglevel1.h \
+    util/rsdebuglevel2.h \
+    util/rsdebuglevel3.h \
+    util/rsdebuglevel4.h \
 			util/rskbdinput.h \
 			util/rsmemory.h \
 			util/smallobject.h \
@@ -480,6 +488,8 @@ HEADERS +=	util/folderiterator.h \
 			util/dnsresolver.h \
                         util/radix32.h \
                         util/radix64.h \
+                        util/rsbase64.h \
+                        util/rsendian.h \
                         util/rsinitedptr.h \
 			util/rsprint.h \
 			util/rsstring.h \
@@ -493,7 +503,8 @@ HEADERS +=	util/folderiterator.h \
 			util/rstime.h \
             util/stacktrace.h \
             util/rsdeprecate.h \
-            util/cxx11retrocompat.h \
+    util/cxx11retrocompat.h \
+    util/cxx14retrocompat.h \
     util/cxx17retrocompat.h \
             util/rsurl.h \
     util/rserrno.h
@@ -507,7 +518,8 @@ SOURCES +=	ft/ftchunkmap.cc \
 			ft/ftfilesearch.cc \
 			ft/ftserver.cc \
 			ft/fttransfermodule.cc \
-            ft/ftturtlefiletransferitem.cc
+            ft/ftturtlefiletransferitem.cc \
+    util/i2pcommon.cpp
 
 SOURCES += crypto/chacha20.cpp \
            crypto/hashstream.cc\
@@ -636,6 +648,7 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsrecogn.cc \
             util/rstime.cc \
             util/rsurl.cc \
+            util/rsbase64.cc \
     util/rserrno.cc
 
 equals(RS_UPNP_LIB, miniupnpc) {
@@ -668,6 +681,7 @@ HEADERS += rsitems/rsnxsitems.h \
 	util/rsdbbind.h \
 	util/contentvalue.h \
 	gxs/rsgxsutil.h \
+	gxs/rsgxsnotify.h \
 	gxs/gxssecurity.h \
 	gxs/rsgds.h \
 	gxs/rsgxs.h \
@@ -763,6 +777,8 @@ SOURCES += services/p3gxschannels.cc \
 	rsitems/rsgxschannelitems.cc \
 
 wikipoos {
+	DEFINES *= RS_USE_WIKI
+
 	# Wiki Service
 	HEADERS += retroshare/rswiki.h \
 		services/p3wiki.h \
@@ -773,6 +789,8 @@ wikipoos {
 }
 
 gxsthewire {
+	DEFINES *= RS_USE_WIRE
+
 	# Wire Service
 	HEADERS += retroshare/rswire.h \
 		services/p3wire.h \
@@ -793,6 +811,8 @@ SOURCES +=  services/p3postbase.cc \
 	rsitems/rsposteditems.cc
 
 gxsphotoshare {
+	DEFINES *= RS_USE_PHOTO
+
 	#Photo Service
 	HEADERS += services/p3photoservice.h \
 		retroshare/rsphoto.h \
@@ -832,23 +852,41 @@ rs_jsonapi {
     no_rs_cross_compiling {
         DUMMYRESTBEDINPUT = FORCE
         CMAKE_GENERATOR_OVERRIDE=""
-        win32-g++:CMAKE_GENERATOR_OVERRIDE="-G \"MSYS Makefiles\""
+        win32-g++|win32-clang-g++ {
+            isEmpty(QMAKE_SH) {
+                CMAKE_GENERATOR_OVERRIDE="-G \"MinGW Makefiles\""
+            } else {
+                CMAKE_GENERATOR_OVERRIDE="-G \"MSYS Makefiles\""
+            }
+        }
         genrestbedlib.name = Generating librestbed.
         genrestbedlib.input = DUMMYRESTBEDINPUT
         genrestbedlib.output = $$clean_path($${RESTBED_BUILD_PATH}/librestbed.a)
         genrestbedlib.CONFIG += target_predeps combine
         genrestbedlib.variable_out = PRE_TARGETDEPS
-        genrestbedlib.commands = \
-            cd $${RS_SRC_PATH} && ( \
-            git submodule update --init supportlibs/restbed ; \
-            cd $${RESTBED_SRC_PATH} ; \
-            git submodule update --init dependency/asio ; \
-            git submodule update --init dependency/catch ; \
-            git submodule update --init dependency/kashmir ; \
-            true ) && \
-            mkdir -p $${RESTBED_BUILD_PATH} && cd $${RESTBED_BUILD_PATH} && \
+        win32-g++:isEmpty(QMAKE_SH) {
+            genrestbedlib.commands = \
+                cd $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/restbed || cd . $$escape_expand(\\n\\t) \
+                cd $$shell_path($${RESTBED_SRC_PATH}) && git submodule update --init dependency/asio || cd . $$escape_expand(\\n\\t) \
+                cd $$shell_path($${RESTBED_SRC_PATH}) && git submodule update --init dependency/catch || cd . $$escape_expand(\\n\\t )\
+                cd $$shell_path($${RESTBED_SRC_PATH}) && git submodule update --init dependency/kashmir || cd . $$escape_expand(\\n\\t) \
+                $(CHK_DIR_EXISTS) $$shell_path($$UDP_DISCOVERY_BUILD_PATH) $(MKDIR) $$shell_path($${UDP_DISCOVERY_BUILD_PATH}) $$escape_expand(\\n\\t)
+        } else {
+            genrestbedlib.commands = \
+                cd $${RS_SRC_PATH} && ( \
+                git submodule update --init supportlibs/restbed ; \
+                cd $${RESTBED_SRC_PATH} ; \
+                git submodule update --init dependency/asio ; \
+                git submodule update --init dependency/catch ; \
+                git submodule update --init dependency/kashmir ; \
+                true ) && \
+                mkdir -p $${RESTBED_BUILD_PATH} &&
+        }
+        genrestbedlib.commands += \
+            cd $$shell_path($${RESTBED_BUILD_PATH}) && \
             cmake \
                 -DCMAKE_CXX_COMPILER=$$QMAKE_CXX \
+                \"-DCMAKE_CXX_FLAGS=$${QMAKE_CXXFLAGS}\" \
                 $${CMAKE_GENERATOR_OVERRIDE} -DBUILD_SSL=OFF \
                 -DCMAKE_INSTALL_PREFIX=. -B. \
                 -H$$shell_path($${RESTBED_SRC_PATH}) && \
@@ -859,9 +897,9 @@ rs_jsonapi {
         genrestbedheader.name = Generating restbed header.
         genrestbedheader.input = genrestbedlib.output
         genrestbedheader.output = $${RESTBED_HEADER_FILE}
-        genrestbedheader.CONFIG += target_predeps combine no_link
+        genrestbedheader.CONFIG += target_predeps no_link
         genrestbedheader.variable_out = HEADERS
-        genrestbedheader.commands = cd $${RESTBED_BUILD_PATH} && $(MAKE) install
+        genrestbedheader.commands = cd $$shell_path($${RESTBED_BUILD_PATH}) && $(MAKE) install
         QMAKE_EXTRA_COMPILERS += genrestbedheader
     }
 
@@ -877,13 +915,19 @@ rs_jsonapi {
     genjsonapi.clean = $${WRAPPERS_INCL_FILE} $${WRAPPERS_REG_FILE}
     genjsonapi.CONFIG += target_predeps combine no_link
     genjsonapi.variable_out = HEADERS
-    genjsonapi.commands = \
-        mkdir -p $${JSONAPI_GENERATOR_OUT} && \
-        cp $${DOXIGEN_CONFIG_SRC} $${DOXIGEN_CONFIG_OUT} && \
-        echo OUTPUT_DIRECTORY=$${JSONAPI_GENERATOR_OUT} >> $${DOXIGEN_CONFIG_OUT} && \
-        echo INPUT=$${DOXIGEN_INPUT_DIRECTORY} >> $${DOXIGEN_CONFIG_OUT} && \
-        doxygen $${DOXIGEN_CONFIG_OUT} && \
-        $${JSONAPI_GENERATOR_EXE} $${JSONAPI_GENERATOR_SRC} $${JSONAPI_GENERATOR_OUT};
+    win32-g++:isEmpty(QMAKE_SH) {
+        genjsonapi.commands = \
+            $(CHK_DIR_EXISTS) $$shell_path($$JSONAPI_GENERATOR_OUT) $(MKDIR) $$shell_path($${JSONAPI_GENERATOR_OUT}) $$escape_expand(\\n\\t)
+    } else {
+        genjsonapi.commands = \
+            mkdir -p $${JSONAPI_GENERATOR_OUT} && \
+            cp $${DOXIGEN_CONFIG_SRC} $${DOXIGEN_CONFIG_OUT} && \
+            echo OUTPUT_DIRECTORY=$${JSONAPI_GENERATOR_OUT} >> $${DOXIGEN_CONFIG_OUT} && \
+            echo INPUT=$${DOXIGEN_INPUT_DIRECTORY} >> $${DOXIGEN_CONFIG_OUT} && \
+            doxygen $${DOXIGEN_CONFIG_OUT} &&
+    }
+    genjsonapi.commands += \
+        $${JSONAPI_GENERATOR_EXE} $${JSONAPI_GENERATOR_SRC} $${JSONAPI_GENERATOR_OUT}
     QMAKE_EXTRA_COMPILERS += genjsonapi
 
     # Force recalculation of libretroshare dependencies see https://stackoverflow.com/a/47884045
@@ -929,20 +973,34 @@ rs_broadcast_discovery {
     no_rs_cross_compiling {
         DUMMYQMAKECOMPILERINPUT = FORCE
         CMAKE_GENERATOR_OVERRIDE=""
-        win32-g++:CMAKE_GENERATOR_OVERRIDE="-G \"MSYS Makefiles\""
+        win32-g++|win32-clang-g++ {
+            isEmpty(QMAKE_SH) {
+                CMAKE_GENERATOR_OVERRIDE="-G \"MinGW Makefiles\""
+            } else {
+                CMAKE_GENERATOR_OVERRIDE="-G \"MSYS Makefiles\""
+            }
+        }
         udpdiscoverycpplib.name = Generating libudp-discovery.a.
         udpdiscoverycpplib.input = DUMMYQMAKECOMPILERINPUT
         udpdiscoverycpplib.output = $$clean_path($${UDP_DISCOVERY_BUILD_PATH}/libudp-discovery.a)
         udpdiscoverycpplib.CONFIG += target_predeps combine
         udpdiscoverycpplib.variable_out = PRE_TARGETDEPS
-        udpdiscoverycpplib.commands = \
-            cd $${RS_SRC_PATH} && ( \
-            git submodule update --init supportlibs/udp-discovery-cpp || \
-            true ) && \
-            mkdir -p $${UDP_DISCOVERY_BUILD_PATH} && \
-            cd $${UDP_DISCOVERY_BUILD_PATH} && \
+        win32-g++:isEmpty(QMAKE_SH) {
+            udpdiscoverycpplib.commands = \
+                cd $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/udp-discovery-cpp || cd . $$escape_expand(\\n\\t) \
+                $(CHK_DIR_EXISTS) $$shell_path($$UDP_DISCOVERY_BUILD_PATH) $(MKDIR) $$shell_path($${UDP_DISCOVERY_BUILD_PATH}) $$escape_expand(\\n\\t)
+        } else {
+            udpdiscoverycpplib.commands = \
+                cd $${RS_SRC_PATH} && ( \
+                git submodule update --init supportlibs/udp-discovery-cpp || \
+                true ) && \
+                mkdir -p $${UDP_DISCOVERY_BUILD_PATH} &&
+        }
+        udpdiscoverycpplib.commands += \
+            cd $$shell_path($${UDP_DISCOVERY_BUILD_PATH}) && \
             cmake -DCMAKE_C_COMPILER=$$fixQmakeCC($$QMAKE_CC) \
                 -DCMAKE_CXX_COMPILER=$$QMAKE_CXX \
+                \"-DCMAKE_CXX_FLAGS=$${QMAKE_CXXFLAGS}\" \
                 $${CMAKE_GENERATOR_OVERRIDE} \
                 -DBUILD_EXAMPLE=OFF -DBUILD_TOOL=OFF \
                 -DCMAKE_INSTALL_PREFIX=. -B. \

libretroshare/src/pgp/pgpauxutils.cc

@@ -67,14 +67,6 @@ bool PgpAuxUtilsImpl::parseSignature(unsigned char *sign, unsigned int signlen,
 	return AuthGPG::getAuthGPG()->parseSignature(sign,signlen,issuer);
 }
 
-bool PgpAuxUtilsImpl::askForDeferredSelfSignature(const void *data,
-		const uint32_t len,
-		unsigned char *sign,
-		unsigned int *signlen,
-		int& signature_result , std::string reason)
-{
-	return RsServer::notify()->askForDeferredSelfSignature(data, len, sign, signlen, signature_result, reason);
-}
 
 
 

libretroshare/src/pgp/pgpauxutils.h

@@ -41,10 +41,6 @@ class PgpAuxUtils
 
 	virtual bool parseSignature(unsigned char *sign, unsigned int signlen, RsPgpId& issuer) const =0;
 	virtual bool VerifySignBin(const void *data, uint32_t len, unsigned char *sign, unsigned int signlen, const PGPFingerprintType& withfingerprint) = 0;
-
-	/** @deprecated this method depends on retroshare-gui to work */
-	RS_DEPRECATED_FOR("AuthGPG::SignDataBin")
-	virtual bool askForDeferredSelfSignature(const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen,int& signature_result, std::string reason) = 0;
 };
 
 class PgpAuxUtilsImpl: public PgpAuxUtils
@@ -59,8 +55,6 @@ public:
 	virtual	bool getKeyFingerprint(const RsPgpId& id,PGPFingerprintType& fp) const;
 	virtual bool VerifySignBin(const void *data, uint32_t len, unsigned char *sign, unsigned int signlen, const PGPFingerprintType& withfingerprint);
 	virtual bool getGPGAllList(std::list<RsPgpId> &ids);
-	virtual bool askForDeferredSelfSignature(const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen,int& signature_result, std::string reason);
-
 };
 
 

libretroshare/src/pgp/pgphandler.cc

@@ -630,50 +630,61 @@ std::string PGPHandler::SaveCertificateToString(const RsPgpId& id,bool include_s
 	return makeRadixEncodedPGPKey(key,include_signatures) ;
 }
 
-bool PGPHandler::exportPublicKey(const RsPgpId& id,unsigned char *& mem_block,size_t& mem_size,bool armoured,bool include_signatures) const
+bool PGPHandler::exportPublicKey(
+        const RsPgpId& id,
+        unsigned char*& mem_block, size_t& mem_size,
+        bool armoured, bool include_signatures ) const
 {
-	RsStackMutex mtx(pgphandlerMtx) ;				// lock access to PGP memory structures.
-	const ops_keydata_t *key = locked_getPublicKey(id,false) ;
-	mem_block = NULL ;
+	mem_block = nullptr; mem_size = 0; // clear just in case
 
 	if(armoured)
 	{
-		std::cerr << __PRETTY_FUNCTION__ << ": should not be used with armoured=true, because there's a bug in the armoured export of OPS" << std::endl;
-		return false ;
+		RsErr() << __PRETTY_FUNCTION__ << " should not be used with "
+		        << "armoured=true, because there's a bug in the armoured export"
+		        << " of OPS" << std::endl;
+		print_stacktrace();
+		return false;
 	}
 
-	if(key == NULL)
+	RS_STACK_MUTEX(pgphandlerMtx);
+	const ops_keydata_t* key = locked_getPublicKey(id,false);
+
+	if(!key)
 	{
-		std::cerr << "Cannot output key " << id.toStdString() << ": not found in keyring." << std::endl;
-		return false ;
+		RsErr() << __PRETTY_FUNCTION__ << " key id: " << id
+		        << " not found in keyring." << std::endl;
+		return false;
 	}
 
-   ops_create_info_t* cinfo;
-	ops_memory_t *buf = NULL ;
-   ops_setup_memory_write(&cinfo, &buf, 0);
+	ops_create_info_t* cinfo;
+	ops_memory_t *buf = nullptr;
+	ops_setup_memory_write(&cinfo, &buf, 0);
 
-	if(ops_write_transferable_public_key_from_packet_data(key,armoured,cinfo) != ops_true)
+	if(ops_write_transferable_public_key_from_packet_data(
+	            key, armoured, cinfo ) != ops_true)
 	{
-		std::cerr << "ERROR: This key cannot be processed by RetroShare because\nDSA certificates are not yet handled." << std::endl;
-		return false ;
+		RsErr() << __PRETTY_FUNCTION__ << "  This key id " << id
+		        << " cannot be processed by RetroShare because DSA certificates"
+		        << " support is not implemented yet." << std::endl;
+		return false;
 	}
 
-	ops_writer_close(cinfo) ;
+	ops_writer_close(cinfo);
 
-	mem_block = new unsigned char[ops_memory_get_length(buf)] ;
-	mem_size = ops_memory_get_length(buf) ;
-	memcpy(mem_block,ops_memory_get_data(buf),mem_size) ;
+	mem_size = ops_memory_get_length(buf);
+	mem_block = reinterpret_cast<unsigned char*>(malloc(mem_size));
+	memcpy(mem_block,ops_memory_get_data(buf),mem_size);
 
-   ops_teardown_memory_write(cinfo,buf);
+	ops_teardown_memory_write(cinfo,buf);
 
 	if(!include_signatures)
 	{
-		size_t new_size ;
-		PGPKeyManagement::findLengthOfMinimalKey(mem_block,mem_size,new_size) ;
-		mem_size = new_size ;
+		size_t new_size;
+		PGPKeyManagement::findLengthOfMinimalKey(mem_block, mem_size, new_size);
+		mem_size = new_size;
 	}
 
-	return true ;
+	return true;
 }
 
 bool PGPHandler::exportGPGKeyPair(const std::string& filename,const RsPgpId& exported_key_id) const

libretroshare/src/pgp/pgphandler.h

@@ -107,7 +107,11 @@ public:
 		bool LoadCertificateFromBinaryData(const unsigned char *bin_data,uint32_t bin_data_len, RsPgpId& gpg_id, std::string& error_string);
 
 		std::string SaveCertificateToString(const RsPgpId& id,bool include_signatures) const ;
-		bool exportPublicKey(const RsPgpId& id,unsigned char *& mem,size_t& mem_size,bool armoured,bool include_signatures) const ;
+
+		/** The caller is in charge of freeing `mem` once finished */
+		bool exportPublicKey( const RsPgpId& id,
+		                      unsigned char*& mem, size_t& mem_size,
+		                      bool armoured, bool include_signatures) const;
 
 		bool parseSignature(unsigned char *sign, unsigned int signlen,RsPgpId& issuer_id) ;
 		bool SignDataBin(const RsPgpId& id, const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen, bool make_raw_signature=false, std::string reason = "") ;

libretroshare/src/plugins/pluginmanager.cc

@@ -362,6 +362,7 @@ bool RsPluginManager::loadPlugin(const std::string& plugin_name,bool first_time)
 		dlclose(handle);
 		return false ;
 	} 
+#ifdef TO_REMOVE
 	if(pinfo.svn_revision == 0)
 	{
 		std::cerr  << "    -> No svn revision number." << std::endl;
@@ -370,6 +371,7 @@ bool RsPluginManager::loadPlugin(const std::string& plugin_name,bool first_time)
 		dlclose(handle);
 		return false ;
 	} 
+#endif
 
 	// Now look for the plugin class symbol.
 	//

libretroshare/src/pqi/authssl.cc

@@ -30,6 +30,7 @@
 #include "pqinetwork.h"
 #include "authgpg.h"
 #include "rsitems/rsconfigitems.h"
+#include "util/rsdebug.h"
 #include "util/rsdir.h"
 #include "util/rsstring.h"
 #include "pgp/pgpkeyutil.h"
@@ -404,10 +405,16 @@ int AuthSSLimpl::InitAuth(
 
 		std::cout.flush() ;
 
+#ifndef RS_DISABLE_DIFFIE_HELLMAN_INIT_CHECK
 		if(DH_check(dh, &codes) && codes == 0)
-			SSL_CTX_set_tmp_dh(sslctx, dh);	
+			SSL_CTX_set_tmp_dh(sslctx, dh);
 		else
-			pfs_enabled = false ;
+			pfs_enabled = false;
+#else // ndef RS_DISABLE_DIFFIE_HELLMAN_INIT_CHECK
+		/* DH_check(...) is not strictly necessary and on Android devices it
+		 * takes at least one minute which is untolerable there */
+		SSL_CTX_set_tmp_dh(sslctx, dh);
+#endif // ndef RS_DISABLE_DIFFIE_HELLMAN_INIT_CHECK
 	}
 	else
 		pfs_enabled = false ;
@@ -798,10 +805,24 @@ X509 *AuthSSLimpl::SignX509ReqWithGPG(X509_REQ *req, long /*days*/)
         }
         X509_NAME_free(issuer_name);
 
-        // NEW code, set validity time between null and null
-        // (does not leak the key creation date to the outside anymore. for more privacy)
-        ASN1_TIME_set(X509_get_notBefore(x509), 0);
-        ASN1_TIME_set(X509_get_notAfter(x509), 0);
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+		// (does not leak the key creation date to the outside anymore. for more privacy)
+		ASN1_TIME_set(X509_get_notBefore(x509), 0);
+		ASN1_TIME_set(X509_get_notAfter(x509), 0);
+#else
+		// NEW code, set validity time between 2010 and 2110 (remember to change it when, if OpenSSL check it by default. ;) )
+		// (does not leak the key creation date to the outside anymore. for more privacy)
+		if (!ASN1_TIME_set_string(X509_getm_notBefore(x509), "20100101000000Z"))
+		{
+			RsErr() << __PRETTY_FUNCTION__ << " Set notBefore FAIL" << std::endl;
+			return NULL;
+		}
+		if (!ASN1_TIME_set_string(X509_getm_notAfter(x509), "21100101000000Z"))
+		{
+			RsErr() << __PRETTY_FUNCTION__ << " Set notAfter FAIL" << std::endl;
+			return NULL;
+		}
+#endif
 
         if (!X509_set_subject_name(x509, X509_REQ_get_subject_name(req)))
         {

libretroshare/src/pqi/p3netmgr.cc

@@ -711,47 +711,6 @@ void p3NetMgrIMPL::netExtCheck()
 			}
 		}
 
-#ifdef ALLOW_DHT_STUNNER
-        // (cyril) I disabled this because it's pretty dangerous. The DHT can report a wrong address quite easily
-        // if the other DHT peers are not collaborating.
-        
-		/* Next ask the DhtStunner */
-		{
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-			std::cerr << "p3NetMgrIMPL::netExtCheck() Ext Not Ok, Checking DhtStunner" << std::endl;
-#endif
-			uint8_t isstable = 0;
-			struct sockaddr_storage tmpaddr;
-			sockaddr_storage_clear(tmpaddr);
-
-			if (mDhtStunner)
-			{
-				/* input network bits */
-				if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
-				{
-					if((rsBanList == NULL) || rsBanList->isAddressAccepted(tmpaddr,RSBANLIST_CHECKING_FLAGS_BLACKLIST))
-					{
-						// must be stable???
-						isStable = (isstable == 1);
-						//mNetFlags.mExtAddr = tmpaddr;
-						mNetFlags.mExtAddrOk = true;
-						mNetFlags.mExtAddrStableOk = isStable;
-
-						address_votes[tmpaddr].n++ ;
-#ifdef	NETMGR_DEBUG_STATEBOX
-						std::cerr << "p3NetMgrIMPL::netExtCheck() From DhtStunner: ";
-						std::cerr << sockaddr_storage_tostring(tmpaddr);
-						std::cerr << " Stable: " << (uint32_t) isstable;
-						std::cerr << std::endl;
-#endif
-					}
-					else
-						std::cerr << "(SS) DHTStunner returned wrong own IP " << sockaddr_storage_iptostring(tmpaddr) << " (banned). Rejecting." << std::endl;
-				}
-			}
-		}
-#endif
-
 		/* ask ExtAddrFinder */
 		{
 			/* ExtAddrFinder */
@@ -829,6 +788,49 @@ void p3NetMgrIMPL::netExtCheck()
 #endif
 		}
 
+#ifdef ALLOW_DHT_STUNNER
+		// (cyril) I disabled this because it's pretty dangerous. The DHT can report a wrong address quite easily
+		// if the other DHT peers are not collaborating.
+		// (sehraf) For the record: The udp stunner uses multiple (as for now: two) peers to ensure that the IP recieved is the correct one, see UdpStunner::locked_checkExternalAddress()
+		// Nevertheless this stays a more risky method to determine the external ip address.
+
+		/* lastly ask the DhtStunner as fallback */
+		if (address_votes.empty()) {
+#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
+			std::cerr << "p3NetMgrIMPL::netExtCheck() Ext Not Ok, Checking DhtStunner" << std::endl;
+#endif
+			uint8_t isstable = 0;
+			struct sockaddr_storage tmpaddr;
+			sockaddr_storage_clear(tmpaddr);
+
+			if (mDhtStunner)
+			{
+				/* input network bits */
+				if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
+				{
+					if((rsBanList == NULL) || rsBanList->isAddressAccepted(tmpaddr,RSBANLIST_CHECKING_FLAGS_BLACKLIST))
+					{
+						// must be stable???
+						isStable = (isstable == 1);
+						//mNetFlags.mExtAddr = tmpaddr;
+						mNetFlags.mExtAddrOk = true;
+						mNetFlags.mExtAddrStableOk = isStable;
+
+						address_votes[tmpaddr].n++ ;
+#ifdef	NETMGR_DEBUG_STATEBOX
+						std::cerr << "p3NetMgrIMPL::netExtCheck() From DhtStunner: ";
+						std::cerr << sockaddr_storage_tostring(tmpaddr);
+						std::cerr << " Stable: " << (uint32_t) isstable;
+						std::cerr << std::endl;
+#endif
+					}
+					else
+						std::cerr << "(SS) DHTStunner returned wrong own IP " << sockaddr_storage_iptostring(tmpaddr) << " (banned). Rejecting." << std::endl;
+				}
+			}
+		}
+#endif
+
 		/* any other sources ??? */
 
 		/* finalise address */
@@ -1801,15 +1803,16 @@ void p3NetMgrIMPL::updateNatSetting()
 #endif
 		
 #ifdef RS_USE_DHT_STUNNER
-		switch(natType)
-		{
-		    case RsNatTypeMode::RESTRICTED_CONE:
+		if (mProxyStunner) {
+			switch(natType)
+			{
+			case RsNatTypeMode::RESTRICTED_CONE:
 			{
 			    if ((natHole == RsNatHoleMode::NONE) || (natHole == RsNatHoleMode::UNKNOWN))
 				{
 					mProxyStunner->setRefreshPeriod(NET_STUNNER_PERIOD_FAST);
 				}
-				else 
+				else
 				{
 					mProxyStunner->setRefreshPeriod(NET_STUNNER_PERIOD_SLOW);
 				}
@@ -1824,6 +1827,7 @@ void p3NetMgrIMPL::updateNatSetting()
 
 				mProxyStunner->setRefreshPeriod(NET_STUNNER_PERIOD_SLOW);
 				break;
+			}
 		}
 #endif // RS_USE_DHT_STUNNER
 

libretroshare/src/pqi/p3netmgr.h

@@ -291,8 +291,8 @@ private:
 
 	//p3BitDht   *mBitDht;
 #ifdef RS_USE_DHT_STUNNER
-	pqiAddrAssist *mDhtStunner;
-	pqiAddrAssist *mProxyStunner;
+	pqiAddrAssist *mDhtStunner = nullptr;
+	pqiAddrAssist *mProxyStunner = nullptr;
 #endif // RS_USE_DHT_STUNNER
 
 	RsMutex mNetMtx; /* protects below */

libretroshare/src/pqi/p3notify.cc

@@ -293,14 +293,6 @@ bool p3Notify::askForPluginConfirmation         (const std::string& plugin_filen
 
 	return false ;
 }
-bool p3Notify::askForDeferredSelfSignature      (const void *       data     , const uint32_t     len  , unsigned char *sign, unsigned int *signlen,int& signature_result, std::string reason /*=""*/)
-{
-	FOR_ALL_NOTIFY_CLIENTS
-		if( (*it)->askForDeferredSelfSignature(data,len,sign,signlen,signature_result, reason))
-			return true ;
-
-	return false ;
-}
 
 void p3Notify::registerNotifyClient(NotifyClient *cl)
 {

libretroshare/src/pqi/p3notify.h

@@ -114,7 +114,6 @@ class p3Notify: public RsNotify
 		void notifyPeerStatusChangedSummary   () ;
 		void notifyDiscInfoChanged            () ;
 
-		bool askForDeferredSelfSignature      (const void *       /* data      */, const uint32_t     /* len   */, unsigned char * /* sign */, unsigned int * /* signlen */, int& signature_result , std::string reason = "") ;
 		void notifyDownloadComplete           (const std::string& /* fileHash  */) ;
 		void notifyDownloadCompleteCount      (uint32_t           /* count     */) ;
 		void notifyHistoryChanged             (uint32_t           /* msgId     */, int /* type */) ;

libretroshare/src/pqi/pqihandler.cc

@@ -42,39 +42,19 @@ using std::dec;
 #include <sys/timeb.h>
 #endif
 
-//#define PQI_HDL_DEBUG_UR 1
-
-#ifdef PQI_HDL_DEBUG_UR
-static double getCurrentTS()
-{
-
-#ifndef WINDOWS_SYS
-        struct timeval cts_tmp;
-        gettimeofday(&cts_tmp, NULL);
-        double cts =  (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0;
-#else
-        struct _timeb timebuf;
-        _ftime( &timebuf);
-        double cts =  (timebuf.time) + ((double) timebuf.millitm) / 1000.0;
-#endif
-        return cts;
-}
-#endif
-
 struct RsLog::logInfo pqihandlerzoneInfo = {RsLog::Default, "pqihandler"};
 #define pqihandlerzone &pqihandlerzoneInfo
 
 //static const int PQI_HANDLER_NB_PRIORITY_LEVELS = 10 ;
 //static const float PQI_HANDLER_NB_PRIORITY_RATIO = 2 ;
 
-/****
-#define DEBUG_TICK 1
-#define RSITEM_DEBUG 1
-****/
+//#define UPDATE_RATES_DEBUG 1
+// #define DEBUG_TICK 1
+// #define RSITEM_DEBUG 1
 
 pqihandler::pqihandler() : coreMtx("pqihandler")
 {
-    RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+    RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
     // setup minimal total+individual rates.
     rateIndiv_out = 0.01;
@@ -97,7 +77,7 @@ int	pqihandler::tick()
 	int moreToTick = 0;
 
 	{
-		RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+		RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 		// tick all interfaces...
 		std::map<RsPeerId, SearchModule *>::iterator it;
@@ -127,9 +107,13 @@ int	pqihandler::tick()
 
 	if(now > mLastRateCapUpdate + 5)
 	{
+        std::map<RsPeerId, RsConfigDataRates> rateMap;
+        std::map<RsPeerId, RsConfigDataRates>::iterator it;
+
+
 		// every 5 secs, update the max rates for all modules
 
-		RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+		RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 		for(std::map<RsPeerId, SearchModule *>::iterator it = mods.begin(); it != mods.end(); ++it)
 		{
 			// This is rather inelegant, but pqihandler has searchModules that are dynamically allocated, so the max rates
@@ -149,7 +133,7 @@ int	pqihandler::tick()
 
 bool pqihandler::queueOutRsItem(RsItem *item)
 {
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 	uint32_t size ;
     locked_HandleRsItem(item, size);
@@ -166,7 +150,7 @@ bool pqihandler::queueOutRsItem(RsItem *item)
 int	pqihandler::status()
 {
 	std::map<RsPeerId, SearchModule *>::iterator it;
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 	{ // for output
 		std::string out = "pqihandler::status() Active Modules:\n";
@@ -192,7 +176,7 @@ int	pqihandler::status()
 
 bool	pqihandler::AddSearchModule(SearchModule *mod)
 {
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 	// if peerid used -> error.
 	//std::map<RsPeerId, SearchModule *>::iterator it;
 	if (mod->peerid != mod->pqi->PeerId())
@@ -223,7 +207,7 @@ bool	pqihandler::AddSearchModule(SearchModule *mod)
 
 bool	pqihandler::RemoveSearchModule(SearchModule *mod)
 {
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 	std::map<RsPeerId, SearchModule *>::iterator it;
 	for(it = mods.begin(); it != mods.end(); ++it)
 	{
@@ -313,7 +297,7 @@ int     pqihandler::ExtractRates(std::map<RsPeerId, RsBwRates> &ratemap, RsBwRat
 	total.mQueueOut = 0;
 
 	/* Lock once rates have been retrieved */
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 	std::map<RsPeerId, SearchModule *>::iterator it;
 	for(it = mods.begin(); it != mods.end(); ++it)
@@ -340,10 +324,6 @@ int     pqihandler::ExtractRates(std::map<RsPeerId, RsBwRates> &ratemap, RsBwRat
 // internal fn to send updates
 int     pqihandler::UpdateRates()
 {
-#ifdef PQI_HDL_DEBUG_UR
-	uint64_t t_now;
-#endif
-
 	std::map<RsPeerId, SearchModule *>::iterator it;
 
 	float avail_in = getMaxRate(true);
@@ -353,18 +333,15 @@ int     pqihandler::UpdateRates()
 	float used_bw_out = 0;
 
 	/* Lock once rates have been retrieved */
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 	int num_sm = mods.size();
 	float used_bw_in_table[num_sm];         /* table of in bandwidth currently used by each module */
 	float used_bw_out_table[num_sm];        /* table of out bandwidth currently used by each module */
 
-	int effectiveUploadsSm = 0;
-	int effectiveDownloadsSm = 0;
-
-	// loop through modules to get the used bandwith and the number of modules that are affectively transfering
-#ifdef PQI_HDL_DEBUG_UR
-	std::cerr << "Looping through modules" << std::endl;
+	// loop through modules to get the used bandwidth
+#ifdef UPDATE_RATES_DEBUG
+		RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Looping through modules" << std::endl;
 #endif
 
 	int index = 0;
@@ -372,49 +349,33 @@ int     pqihandler::UpdateRates()
 	for(it = mods.begin(); it != mods.end(); ++it)
 	{
 		SearchModule *mod = (it -> second);
-		float crate_in = mod -> pqi -> getRate(true);
 
 		traffInSum += mod -> pqi -> getTraffic(true);
 		traffOutSum += mod -> pqi -> getTraffic(false);
         
-#ifdef PQI_HDL_DEBUG_UR
-        if(crate_in > 0.0)
-        std::cerr << "  got in rate for peer " << it->first << " : " << crate_in << std::endl;
-#endif
-        
-		if ((crate_in > 0.01 * avail_in) || (crate_in > 0.1))
-		{
-			++effectiveDownloadsSm;
-		}
-
+                float crate_in = mod -> pqi -> getRate(true);
 		float crate_out = mod -> pqi -> getRate(false);
-		if ((crate_out > 0.01 * avail_out) || (crate_out > 0.1))
-		{
-			++effectiveUploadsSm;
-		}
 
 		used_bw_in += crate_in;
 		used_bw_out += crate_out;
 
-		/* fill the table of bandwidth */
+		/* fill the table of used bandwidths */
 		used_bw_in_table[index] = crate_in;
 		used_bw_out_table[index] = crate_out;
+
 		++index;
 	}
 
-#ifdef PQI_HDL_DEBUG_UR
-	t_now = 1000 * getCurrentTS();
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): Sorting used_bw_out_table: " << num_sm << " entries" << std::endl;
+#ifdef UPDATE_RATES_DEBUG
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Sorting used_bw_out_table: " << num_sm << " entries" << std::endl;
 #endif
 
 	/* Sort the used bw in/out table in ascending order */
 	std::sort(used_bw_in_table, used_bw_in_table + num_sm);
 	std::sort(used_bw_out_table, used_bw_out_table + num_sm);
 
-#ifdef PQI_HDL_DEBUG_UR
-	t_now = 1000 * getCurrentTS();
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): Done." << std::endl;
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): used_bw_out " << used_bw_out << std::endl;
+#ifdef UPDATE_RATES_DEBUG
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates used_bw_out " << used_bw_out << std::endl;
 #endif
 
 	/* Calculate the optimal out_max value, taking into account avail_out and the out bw requested by modules */
@@ -441,9 +402,8 @@ int     pqihandler::UpdateRates()
 		}
 	}
 
-#ifdef PQI_HDL_DEBUG_UR
-	t_now = 1000 * getCurrentTS();
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl;
+#ifdef UPDATE_RATES_DEBUG
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl;
 #endif
 
 	/* Allocate only half the remaining out bw, if any, to make it smoother */
@@ -473,67 +433,70 @@ int     pqihandler::UpdateRates()
 		}
 	}
 
-#ifdef PQI_HDL_DEBUG_UR
-	t_now = 1000 * getCurrentTS();
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl;
+#ifdef UPDATE_RATES_DEBUG
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl;
 #endif
 
 	/* Allocate only half the remaining in bw, if any, to make it smoother */
 	in_max_bw = in_max_bw + in_remaining_bw / 2;
 
-
-#ifdef DEBUG_QOS
-//	std::cerr << "Totals (In) Used B/W " << used_bw_in;
-//	std::cerr << " Available B/W " << avail_in;
-//	std::cerr << " Effective transfers " << effectiveDownloadsSm << std::endl;
-//	std::cerr << "Totals (Out) Used B/W " << used_bw_out;
-//	std::cerr << " Available B/W " << avail_out;
-//	std::cerr << " Effective transfers " << effectiveUploadsSm << std::endl;
-#endif
-
+	// store current total in and ou used bw 
 	locked_StoreCurrentRates(used_bw_in, used_bw_out);
 
-	//computing average rates for effective transfers
-	float max_in_effective = avail_in / num_sm;
-	if (effectiveDownloadsSm != 0) {
-	    max_in_effective = avail_in / effectiveDownloadsSm;
-	}
-	float max_out_effective = avail_out / num_sm;
-	if (effectiveUploadsSm != 0) {
-	    max_out_effective = avail_out / effectiveUploadsSm;
-	}
-
-	//modify the in and out limit
-#ifdef PQI_HDL_DEBUG_UR
-	t_now = 1000 * getCurrentTS();
-	std::cerr << dec << t_now << " pqihandler::UpdateRates(): setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl;
+#ifdef UPDATE_RATES_DEBUG
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl;
 #endif
 
+	// retrieve down (from peer point of view) bandwidth limits set by peers in their own settings
+        std::map<RsPeerId, RsConfigDataRates> rateMap;
+        rsConfig->getAllBandwidthRates(rateMap);
+        std::map<RsPeerId, RsConfigDataRates>::iterator rateMap_it;
+
+#ifdef UPDATE_RATES_DEBUG
+	// Dump RsConfigurationDataRates
+	RsDbg() << "UPDATE_RATES pqihandler::UpdateRates RsConfigDataRates dump" << std::endl;
+	for (rateMap_it = rateMap.begin(); rateMap_it != rateMap.end(); rateMap_it++)
+		RsDbg () << "UPDATE_RATES pqihandler::UpdateRates PeerId " << rateMap_it->first.toStdString() << " mAllowedOut " << rateMap_it->second.mAllowedOut << std::endl;
+#endif
+
+        // update max rates taking into account the limits set by peers in their own settings
 	for(it = mods.begin(); it != mods.end(); ++it)
 	{
 		SearchModule *mod = (it -> second);
-        
-		mod -> pqi -> setMaxRate(true,   in_max_bw);
-		mod -> pqi -> setMaxRate(false, out_max_bw);
+
+		// for our down bandwidth we set the max to the calculated value without taking into account the max set by peers: they will control their up bw on their side
+		mod -> pqi -> setMaxRate(true, in_max_bw);
+
+		// for our up bandwidth we limit to the maximum down bw provided by peers via BwCtrl because we don't want to clog our outqueues, the SSL buffers, and our friends inbound queues
+		if ((rateMap_it = rateMap.find(mod->pqi->PeerId())) != rateMap.end())
+		{
+			if (rateMap_it->second.mAllowedOut > 0)
+			{	
+				if (out_max_bw > rateMap_it->second.mAllowedOut)
+        	                        mod -> pqi -> setMaxRate(false, rateMap_it->second.mAllowedOut);
+				else
+					mod -> pqi -> setMaxRate(false, out_max_bw);
+			}
+			else
+				mod -> pqi -> setMaxRate(false, out_max_bw);
+		}
 	}
 
-
-	//cap the rates
+#ifdef UPDATE_RATES_DEBUG
+	// dump maxRates
 	for(it = mods.begin(); it != mods.end(); ++it)
 	{
 		SearchModule *mod = (it -> second);
-		if (mod -> pqi -> getMaxRate(false) < max_out_effective)  mod -> pqi -> setMaxRate(false, max_out_effective);
-		if (mod -> pqi -> getMaxRate(false) > avail_out)          mod -> pqi -> setMaxRate(false, avail_out);
-		if (mod -> pqi -> getMaxRate(true)  < max_in_effective)   mod -> pqi -> setMaxRate(true,  max_in_effective);
-		if (mod -> pqi -> getMaxRate(true)  > avail_in)           mod -> pqi -> setMaxRate(true,  avail_in);
+		RsDbg() << "UPDATE_RATES pqihandler::UpdateRates PeerID " << (mod ->pqi -> PeerId()).toStdString() << " new bandwidth limits up " << mod -> pqi -> getMaxRate(false) << " down " << mod -> pqi -> getMaxRate(true) << std::endl;
 	}
+#endif
 
 	return 1;
 }
 
 void    pqihandler::getCurrentRates(float &in, float &out)
 {
-	RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/
+	RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/
 
 	in = rateTotal_in;
 	out = rateTotal_out;

libretroshare/src/pqi/pqinetwork.cc

@@ -413,9 +413,10 @@ int unix_fcntl_nonblock(int fd)
 {
         int ret;
 
-/******************* WINDOWS SPECIFIC PART ******************/
+/******************* OS SPECIFIC PART ******************/
 #ifndef WINDOWS_SYS // ie UNIX
-	ret = fcntl(fd, F_SETFL, O_NONBLOCK);
+	int flags = fcntl(fd, F_GETFL);
+	ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
 
 #ifdef NET_DEBUG
 	std::cerr << "unix_fcntl_nonblock():" << ret << " errno:" << errno << std::endl;

libretroshare/src/pqi/pqiservice.cc

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * libretroshare/src/pqi: pqiservice.h                                         *
+ * libretroshare/src/pqi: pqiservice.cc                                         *
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
@@ -23,6 +23,22 @@
 #include "util/rsdebug.h"
 #include "util/rsstring.h"
 
+#include <sstream>
+#include <sys/time.h>
+static double getCurrentTS()
+{
+#ifndef WINDOWS_SYS
+        struct timeval cts_tmp;
+        gettimeofday(&cts_tmp, NULL);
+        double cts =  (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0;
+#else
+        struct _timeb timebuf;
+        _ftime( &timebuf);
+        double cts =  (timebuf.time) + ((double) timebuf.millitm) / 1000.0;
+#endif
+        return cts;
+}
+
 #ifdef  SERVICE_DEBUG
 const int pqiservicezone = 60478;
 #endif
@@ -44,7 +60,7 @@ bool pqiService::send(RsRawItem *item)
 
 p3ServiceServer::p3ServiceServer(pqiPublisher *pub, p3ServiceControl *ctrl) : mPublisher(pub), mServiceControl(ctrl), srvMtx("p3ServiceServer") 
 {
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
+	RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/
 
 #ifdef  SERVICE_DEBUG
 	pqioutput(PQL_DEBUG_BASIC, pqiservicezone, 
@@ -56,7 +72,7 @@ p3ServiceServer::p3ServiceServer(pqiPublisher *pub, p3ServiceControl *ctrl) : mP
 
 int	p3ServiceServer::addService(pqiService *ts, bool defaultOn)
 {
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
+	RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/
 
 #ifdef  SERVICE_DEBUG
 	pqioutput(PQL_DEBUG_BASIC, pqiservicezone, 
@@ -84,7 +100,7 @@ int	p3ServiceServer::addService(pqiService *ts, bool defaultOn)
 
 bool p3ServiceServer::getServiceItemNames(uint32_t service_type,std::map<uint8_t,std::string>& names)
 {
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
+	RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/
 
  	std::map<uint32_t, pqiService *>::iterator it=services.find(service_type) ;
 
@@ -99,7 +115,7 @@ bool p3ServiceServer::getServiceItemNames(uint32_t service_type,std::map<uint8_t
 
 int p3ServiceServer::removeService(pqiService *ts)
 {
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
+	RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/
 
 #ifdef SERVICE_DEBUG
 	pqioutput(PQL_DEBUG_BASIC, pqiservicezone, "p3ServiceServer::removeService()");
@@ -124,60 +140,33 @@ int p3ServiceServer::removeService(pqiService *ts)
 
 bool	p3ServiceServer::recvItem(RsRawItem *item)
 {
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
-
-#ifdef  SERVICE_DEBUG
-	std::cerr << "p3ServiceServer::incoming()";
-	std::cerr << std::endl;
-
-	{
-		std::string out;
-		rs_sprintf(out, "p3ServiceServer::incoming() PacketId: %x\nLooking for Service: %x\nItem:\n", item -> PacketId(), (item -> PacketId() & 0xffffff00));
-		item -> print_string(out);
-		std::cerr << out;
-		std::cerr << std::endl;
-	}
-#endif
-
 	// Packet Filtering.
 	// This doesn't need to be in Mutex.
 	if (!mServiceControl->checkFilter(item->PacketId() & 0xffffff00, item->PeerId()))
 	{
-#ifdef  SERVICE_DEBUG
-        std::cerr << "p3ServiceServer::recvItem() Fails Filtering " << std::endl;
-#endif
 		delete item;
 		return false;
 	}
 
+	pqiService *s = NULL;
 
-	std::map<uint32_t, pqiService *>::iterator it;
-	it = services.find(item -> PacketId() & 0xffffff00);
-	if (it == services.end())
+	// access the service map under mutex lock
 	{
-#ifdef  SERVICE_DEBUG
-		std::cerr << "p3ServiceServer::incoming() Service: No Service - deleting";
-		std::cerr << std::endl;
-#endif
-		delete item;
-		return false;
+		RS_STACK_MUTEX(srvMtx);
+		auto it = services.find(item -> PacketId() & 0xffffff00);
+		if (it == services.end())
+		{
+			delete item;
+			return false;
+		}
+		s = it->second;
 	}
 
-	{
-#ifdef  SERVICE_DEBUG
-		std::cerr << "p3ServiceServer::incoming() Sending to : " << (void *) it -> second;
-		std::cerr << std::endl;
-#endif
-
-		return (it->second) -> recv(item);
-	}
-
-	delete item;
-	return false;
+	// then call recv off mutex
+	bool result = s->recv(item);
+	return result;
 }
 
-
-
 bool p3ServiceServer::sendItem(RsRawItem *item)
 {
 #ifdef  SERVICE_DEBUG
@@ -196,46 +185,35 @@ bool p3ServiceServer::sendItem(RsRawItem *item)
 	// Packet Filtering.
 	if (!mServiceControl->checkFilter(item->PacketId() & 0xffffff00, item->PeerId()))
 	{
+#ifdef  SERVICE_DEBUG
 		std::cerr << "p3ServiceServer::sendItem() Fails Filtering for packet id=" << std::hex << item->PacketId() << std::dec << ", and peer " << item->PeerId() << std::endl;
+#endif
 		delete item;
 		return false;
 	}
 
 	mPublisher->sendItem(item);
+
 	return true;
 }
 
-
-
 int	p3ServiceServer::tick()
 {
-
 	mServiceControl->tick();
 
-	RsStackMutex stack(srvMtx); /********* LOCKED *********/
-
-#ifdef  SERVICE_DEBUG
-	pqioutput(PQL_DEBUG_ALL, pqiservicezone, 
-		"p3ServiceServer::tick()");
-#endif
-
-	std::map<uint32_t, pqiService *>::iterator it;
-
-	// from the beginning to where we started.
-	for(it = services.begin();it != services.end(); ++it)
-	{
-
-#ifdef  SERVICE_DEBUG
-		std::string out;
-		rs_sprintf(out, "p3ServiceServer::service id: %u -> Service: %p", it -> first, it -> second);
-		pqioutput(PQL_DEBUG_ALL, pqiservicezone, out);
-#endif
-
-		// now we should actually tick the service.
-		(it -> second) -> tick();
+	// make a copy of the service map
+	std::map<uint32_t,pqiService *> local_map;
+	{	
+		RS_STACK_MUTEX(srvMtx);
+		local_map=services;
 	}
+
+	// tick all services off mutex
+	for(auto it(local_map.begin());it!=local_map.end();++it)
+	{
+		(it->second)->tick();
+	}
+
 	return 1;
+
 }
-
-
-

libretroshare/src/pqi/pqissl.cc

@@ -372,9 +372,11 @@ int 	pqissl::status()
 	// tick......
 int	pqissl::tick()
 {
-	RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/
+	// there is no reason to lock pqissl mutex now 
+	// we will lock the mutex later if we actually need to call to ConnectAttempt
+	// RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/
 
-	//pqistreamer::tick();
+	// pqistreamer::tick();
 
 	// continue existing connection attempt.
 	if (!active)
@@ -385,7 +387,8 @@ int	pqissl::tick()
 #ifdef PQISSL_LOG_DEBUG 
 			rslog(RSL_DEBUG_BASIC, pqisslzone, "pqissl::tick() Continuing Connection Attempt!");
 #endif
-
+			// now lock pqissl mutex, that will take up to 10 ms
+			RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/
 			ConnectAttempt();
 			return 1;
 		}
@@ -1110,10 +1113,16 @@ int pqissl::SSL_Connection_Complete()
 		if(rsEvents)
 		{
 			X509 *x509 = SSL_get_peer_certificate(ssl_connection);
-			auto ev = std::make_shared<RsAuthSslConnectionAutenticationEvent>();
-			ev->mSslId = RsX509Cert::getCertSslId(*x509);
-			ev->mErrorCode = RsAuthSslError::PEER_REFUSED_CONNECTION;
-			rsEvents->postEvent(ev);
+
+            if(x509)
+			{
+				auto ev = std::make_shared<RsAuthSslConnectionAutenticationEvent>();
+				ev->mSslId = RsX509Cert::getCertSslId(*x509);
+				ev->mErrorCode = RsAuthSslError::PEER_REFUSED_CONNECTION;
+
+				if(!ev->mSslId.isNull())
+					rsEvents->postEvent(ev);
+			}
 		}
 
 		std::string out;

libretroshare/src/pqi/pqissl.h

@@ -56,6 +56,69 @@ const int PQISSL_UDP_FLAG = 0x02;
 /* TCP buffer size for Windows systems */
 const int WINDOWS_TCP_BUFFER_SIZE = 512 * 1024; // 512 KB
 
+
+//	This is a (very) simple overview of the different state machnines. The tree includes high level funtions only.
+//
+//	connect_parameter() is used to pass down settings, like address or timeout values
+//
+//	tick() or connect()
+//     |
+//     +----- ConnectAttempt()
+//                 |
+//                 +--WAITING_NOT or WAITING_DELAY
+//                 |           |
+//                 |           +----- Delay_Connection()
+//                 |                       |
+//                 |                       +--WAITING_NOT
+//                 |                       |        - set 'waiting' to WAITING_DELAY and set delay for next connection attempt
+//                 |                       |
+//                 |                       +--WAITING_DELAY
+//                 |                                   |
+//                 |                                   +----- Initiate_Connection()
+//                 |                                               |
+//                 |                                               +----- setup socket
+//                 |                                               +----- connect
+//                 |                                                        - on success: set "waiting" to WAITING_SOCK_CONNECT and "sockfd" to newly created socket
+//                 |                                                        - on failure: set "waiting" to WAITING_FAIL_INTERFACE
+//                 |
+//                 +--WAITING_SOCK_CONNECT
+//                 |           |
+//                 |           +----- Initiate_SSL_Connection()
+//                 |                       |
+//                 |                       +----- Basic_Connection_Complete()
+//                 |                       |           |
+//                 |                       |           +----- CheckConnectionTimeout()
+//                 |                       |           |
+//                 |                       |           +----- ready up socket.
+//                 |                       |                    - SOCKS, udp tou, i2p BOB intercept here
+//                 |                       |                    - on failure: set "waiting" to WAITING_FAIL_INTERFACE and "sockfd" to -1
+//                 |                       |
+//                 |                       +----- create SSL context and attach file descriptors
+//                 |                              - on success:_set "waiting" to WAITING_SSL_CONNECTION
+//                 |
+//                 +--WAITING_SSL_CONNECTION or WAITING_SSL_AUTHORISE
+//                 |           |
+//                 |           +----- Authorise_SSL_Connection()
+//                 |                       |
+//                 |                       +----- SSL_Connection_Complete()
+//                 |                       |           |
+//                 |                       |           +----- performes TSL handshake
+//                 |                       |                    - on success: set "waiting" to WAITING_SSL_AUTHORISE
+//                 |                       |                    - on failure: set "waiting" to WAITING_FAIL_INTERFACE
+//                 |                       |
+//                 |                       +----- set "waiting" to WAITING_NOT
+//                 |                       |
+//                 |                       +----- accept_locked()
+//                 |                                - add peer to the rest of RS and start pqi thread
+//                 |
+//                 |
+//                 +--WAITING_FAIL_INTERFACE
+//	                 	       |
+//                             +----- Failed_Connection()
+//                                      - set "waiting" to WAITING_NOT
+//
+
+
 /***************************** pqi Net SSL Interface *********************************
  * This provides the base SSL interface class, 
  * and handles most of the required functionality.
@@ -203,9 +266,9 @@ bool  	CheckConnectionTimeout();
 	uint32_t mConnectTimeout;
 	rstime_t   mTimeoutTS;
 
+	RS_SET_CONTEXT_DEBUG_LEVEL(1)
+
 private:
 	// ssl only fns.
 	int connectInterface(const struct sockaddr_storage &addr);
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/pqi/pqissllistener.h

@@ -34,6 +34,73 @@
 
 #define RS_PQISSL_AUTH_DOUBLE_CHECK 1
 
+
+//  This is a simple overview of how the listener is setup, ticked (calling accept) and peers added to it.
+//  On the highest level (RsServer) the listener lives inside the pqisslpersongrp class (variable: "pqih").
+//  Inside pqisslpersongrp the listener is stored in "pqil".
+//
+//  The listener has an internal list with incoming connections that are handled in a similar fashion to pqissl.
+//  (Mainly setting up the socket (non-blocking) and establisching the ssl handshake.)
+//  When everything went fine the connection is passed to pqissl in finaliseConnection()
+//
+//  This is how the listener is initialized during start up:
+//
+//   RsServer::StartupRetroShare()
+//      |
+//      +----- pqih = new pqisslpersongrp(serviceCtrl, flags, mPeerMgr);
+//      +----- pqih->init_listener();
+//                  |
+//                  +----- pqil = locked_createListener(laddr);
+//                              |
+//                              +----- return new pqissllistener(laddr, mPeerMgr);
+//
+//
+//  This is how the listener is ticked to call accept:
+//
+//   RsServer::StartupRetroShare()
+//      |
+//      +----- pqih->tick();
+//                  |
+//                  +----- pqil->tick();
+//                              |
+//                              +----- acceptconnection();
+//                              |           |
+//                              |           +----- accecpt()
+//                              |
+//                              +----- continueaccepts();
+//                              +----- finaliseAccepts();
+//                                          |
+//                                          +----- finaliseConnection()
+//                                                      |
+//                                                      +----- pqis->accept()
+//
+//
+//  This is how peers (their id) are registered to the listener:
+//  (This is only used to tell if a connection peer is known or a new one (which is then added))
+//
+//   pqipersongrp::addPeer()
+//      |
+//      +----- pqiperson *pqip = locked_createPerson(id, pqil);
+//      |           |
+//      |           +----- pqiperson *pqip = new pqiperson(id, this);
+//      |           +----- pqissl *pqis = new pqissl((pqissllistener *) listener, pqip, mLinkMgr);
+//      |           +----- pqiconnect *pqisc = new pqiconnect(pqip, rss, pqis);
+//      |           +----- pqip->addChildInterface(PQI_CONNECT_TCP, pqisc);
+//      |                       |
+//      |                       +-- sets kids[type] = pqisc;
+//      |
+//      +----- pqip->reset();
+//      +----- pqip->listen();
+//                  |
+//                  +-- for all kids[]
+//                              |
+//                              +----- listen() ( of class pqiconnect )
+//                                          |
+//                                          +----- listen() ( of class pqissl )
+//                                                      |
+//                                                      +----- pqil->addlistenaddr(PeerId(), this);
+
+
 /***************************** pqi Net SSL Interface *********************************
  */
 

libretroshare/src/pqi/pqistreamer.cc

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * libretroshare/src/pqi: pqistreamer.h                                        *
+ * libretroshare/src/pqi: pqistreamer.cc                                       *
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
@@ -102,38 +102,39 @@ pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bi
 	mAvgDtOut(0), mAvgDtIn(0)
 {
 
-    // 100 B/s (minimal)
-    setMaxRate(true, 0.1);
-    setMaxRate(false, 0.1);
-    setRate(true, 0);		// needs to be off-mutex
-    setRate(false, 0);
+	// 100 B/s (minimal)
+	setMaxRate(true, 0.1);
+	setMaxRate(false, 0.1);
+	setRate(true, 0);		// needs to be off-mutex
+	setRate(false, 0);
 
-    RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
 
-    mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready.
-    mLastSentPacketSlicingProbe = 0 ;
+	mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready.
+	mLastSentPacketSlicingProbe = 0 ;
 
-    mAvgLastUpdate = mCurrSentTS = mCurrReadTS = getCurrentTS();
+	mAvgLastUpdate = mCurrSentTS = mCurrReadTS = getCurrentTS();
 
-    mIncomingSize = 0 ;
+	mIncomingSize = 0 ;
+	mIncomingSize_bytes = 0;
 
-    mStatisticsTimeStamp = 0 ;
-    /* allocated once */
-    mPkt_rpend_size = 0;
-    mPkt_rpending = 0;
-    mReading_state = reading_state_initial ;
+	mStatisticsTimeStamp = 0 ;
+	/* allocated once */
+	mPkt_rpend_size = 0;
+	mPkt_rpending = 0;
+	mReading_state = reading_state_initial ;
 
-    pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::pqistreamer() Initialisation!");
+	pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::pqistreamer() Initialisation!");
 
-    if (!bio_in)
-    {
-	    pqioutput(PQL_ALERT, pqistreamerzone, "pqistreamer::pqistreamer() NULL bio, FATAL ERROR!");
-	    exit(1);
-    }
+	if (!bio_in)
+	{
+		pqioutput(PQL_ALERT, pqistreamerzone, "pqistreamer::pqistreamer() NULL bio, FATAL ERROR!");
+		exit(1);
+	}
 
-    mFailed_read_attempts = 0;  // reset failed read, as no packet is still read.
+	mFailed_read_attempts = 0;  // reset failed read, as no packet is still read.
 
-    return;
+	return;
 }
 
 pqistreamer::~pqistreamer()
@@ -159,7 +160,7 @@ pqistreamer::~pqistreamer()
 	if (mRsSerialiser)
 		delete mRsSerialiser;
 
-	free_pend_locked() ;
+	free_pend() ;
 
 	// clean up incoming.
 	while (!mIncoming.empty())
@@ -177,6 +178,7 @@ pqistreamer::~pqistreamer()
 
 
 // Get/Send Items.
+// This is the entry poing for methods willing to send items through our out queue
 int	pqistreamer::SendItem(RsItem *si,uint32_t& out_size)
 {
 #ifdef RSITEM_DEBUG 
@@ -199,18 +201,30 @@ RsItem *pqistreamer::GetItem()
 	pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::GetItem()");
 #endif
 
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-
 	if(mIncoming.empty())
 		return NULL; 
 
 	RsItem *osr = mIncoming.front() ;
-    mIncoming.pop_front() ;
-    --mIncomingSize;
+	mIncoming.pop_front() ;
+	--mIncomingSize;
+// for future use
+//	mIncomingSize_bytes -= 
 
 	return osr;
 }
 
+
+float pqistreamer::getMaxRate(bool b)
+{
+        RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+        return getMaxRate_locked(b);
+}
+
+float pqistreamer::getMaxRate_locked(bool b)
+{
+        return RateInterface::getMaxRate(b) ;
+}
+
 float pqistreamer::getRate(bool b)
 {
 	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
@@ -219,26 +233,28 @@ float pqistreamer::getRate(bool b)
 
 void pqistreamer::setMaxRate(bool b,float f)
 {
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-    	RateInterface::setMaxRate(b,f) ;
+        RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+        setMaxRate_locked(b,f);
 }
+
+void pqistreamer::setMaxRate_locked(bool b,float f)
+{
+        RateInterface::setMaxRate(b,f) ;
+}
+
 void pqistreamer::setRate(bool b,float f)
 {
 	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
     	RateInterface::setRate(b,f) ;
 }
 
+
 void pqistreamer::updateRates()
 {
-	// update rates both ways.
+	// update actual rates both ways.
 
 	double t = getCurrentTS(); // get current timestamp.
-	double diff ;
-
-	{
-		RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-		diff = t - mAvgLastUpdate ;
-	}
+	double diff = t - mAvgLastUpdate;
 
 	if (diff > PQISTREAM_AVG_PERIOD)
 	{
@@ -263,10 +279,11 @@ void pqistreamer::updateRates()
 			setRate(false, 0);
 		}
 
+		mAvgLastUpdate = t;
+		mAvgReadCount = 0;
+
 		{
 			RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-			mAvgLastUpdate = t;
-			mAvgReadCount = 0;
 			mAvgSentCount = 0;
 		}
 	}
@@ -277,7 +294,7 @@ int 	pqistreamer::tick_bio()
 	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
 	mBio->tick();
 	
-	/* short circuit everything is bio isn't active */
+	/* short circuit everything if bio isn't active */
 	if (!(mBio->isactive()))
 	{
 		return 0;
@@ -285,36 +302,36 @@ int 	pqistreamer::tick_bio()
 	return 1;
 }
 
-
 int 	pqistreamer::tick_recv(uint32_t timeout)
 {
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+//      Apart from a few exceptions that are atomic (mLastIncomingTs, mIncomingSize), only this pqi thread reads/writes mIncoming queue and related counters.
+//      The lock of pqistreamer mutex is thus not needed here.
+//      The mutex lock is still needed before calling locked_addTrafficClue because this method is also used by the thread pushing packets in mOutPkts.
+//	Locks around rates are provided internally.
 
 	if (mBio->moretoread(timeout))
 	{
-		handleincoming_locked();
+		handleincoming();
+	}
+	if(!(mBio->isactive()))
+	{
+		free_pend();
 	}
-    if(!(mBio->isactive()))
-    {
-        free_pend_locked();
-    }
 	return 1;
 }
 
-
 int 	pqistreamer::tick_send(uint32_t timeout)
 {
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-
-	/* short circuit everything is bio isn't active */
+	/* short circuit everything if bio isn't active */
 	if (!(mBio->isactive()))
 	{
-        		free_pend_locked();
+		free_pend();
 		return 0;
 	}
 
 	if (mBio->cansend(timeout))
 	{
+		RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
 		handleoutgoing_locked();
 	}
     
@@ -340,12 +357,11 @@ int	pqistreamer::status()
 	return 0;
 }
 
+// this method is overloaded by pqiqosstreamer
 void pqistreamer::locked_storeInOutputQueue(void *ptr,int,int)
 {
 	mOutPkts.push_back(ptr);
 }
-//
-/**************** HANDLE OUTGOING TRANSLATION + TRANSMISSION ******/
 
 int	pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize)
 {
@@ -354,7 +370,6 @@ int	pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize)
         std::cerr << "pqistreamer::queue_outpqi() called." << std::endl;
 #endif
 
-
 	/* decide which type of packet it is */
 
 	pktsize = mRsSerialiser->size(pqi);
@@ -362,7 +377,6 @@ int	pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize)
     
     	if(ptr == NULL)
             return 0 ;
-            
 
 #ifdef DEBUG_PQISTREAMER
 	std::cerr << "pqistreamer::queue_outpqi() serializing packet with packet size : " << pktsize << std::endl;
@@ -403,27 +417,31 @@ int	pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize)
 	return 1; // keep error internal.
 }
 
-int 	pqistreamer::handleincomingitem_locked(RsItem *pqi,int len)
+int 	pqistreamer::handleincomingitem(RsItem *pqi,int len)
 {
 
 #ifdef DEBUG_PQISTREAMER
-	pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincomingitem_locked()");
+	pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincomingitem()");
 #endif
 	// timestamp last received packet.
 	mLastIncomingTs = time(NULL);
 
 	// Use overloaded Contact function 
 	pqi -> PeerId(PeerId());
-    mIncoming.push_back(pqi);
-    ++mIncomingSize ;
 
-            /*******************************************************************************************/
-    	// keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1
-    	// is a full statistics chunk that can be used in the GUI
+	mIncoming.push_back(pqi);
+	++mIncomingSize;
+	// for future use
+	//	mIncomingSize_bytes += len;
 
-    	locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ;
-
-        /*******************************************************************************************/
+	/*******************************************************************************************/
+	// keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1
+	// is a full statistics chunk that can be used in the GUI
+	{
+		RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+		locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ;
+	}
+	/*******************************************************************************************/
 
 	return 1;
 }
@@ -456,8 +474,8 @@ void pqistreamer::locked_addTrafficClue(const RsItem *pqi,uint32_t pktsize,std::
 
 rstime_t	pqistreamer::getLastIncomingTS()
 {
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-
+	// This is the only case where another thread (rs main for pqiperson) will access our data
+	// Still a mutex lock is not needed because the operation is atomic
 	return mLastIncomingTs;
 }
 
@@ -693,23 +711,23 @@ int	pqistreamer::handleoutgoing_locked()
 
 /* Handles reading from input stream.
  */
-int pqistreamer::handleincoming_locked()
+int pqistreamer::handleincoming()
 {
     int readbytes = 0;
     static const int max_failed_read_attempts = 2000 ;
 
 #ifdef DEBUG_PQISTREAMER
-    pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming_locked()");
+    pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming()");
 #endif
 
     if(!(mBio->isactive()))
     {
 	    mReading_state = reading_state_initial ;
-	    free_pend_locked();
+	    free_pend();
 	    return 0;
     }
     else
-	    allocate_rpend_locked();
+	    allocate_rpend();
 
     // enough space to read any packet.
     uint32_t maxlen = mPkt_rpend_size; 
@@ -718,7 +736,7 @@ int pqistreamer::handleincoming_locked()
     // initial read size: basic packet.
     int blen = getRsPktBaseSize();	// this is valid for both packet slices and normal un-sliced packets (same header size)
 
-    int maxin = inAllowedBytes_locked();
+    int maxin = inAllowedBytes();
 
 #ifdef DEBUG_PQISTREAMER
     std::cerr << "[" << (void*)pthread_self() << "] " << "reading state = " << mReading_state << std::endl ;
@@ -967,19 +985,19 @@ continue_packet:
 		    std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl;
 #endif
             		uint32_t packet_length = 0 ;
-		    pkt = addPartialPacket_locked(block,pktlen,slice_packet_id,is_packet_starting,is_packet_ending,packet_length) ;
+		    pkt = addPartialPacket(block,pktlen,slice_packet_id,is_packet_starting,is_packet_ending,packet_length) ;
             
             		pktlen = packet_length ;
 	    }
 	    else
 		    pkt = mRsSerialiser->deserialise(block, &pktlen);
 
-	    if ((pkt != NULL) && (0  < handleincomingitem_locked(pkt,pktlen)))
+	    if ((pkt != NULL) && (0  < handleincomingitem(pkt,pktlen)))
 	    {
 #ifdef DEBUG_PQISTREAMER
 		    pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "Successfully Read a Packet!");
 #endif
-		    inReadBytes_locked(pktlen);	// only count deserialised packets, because that's what is actually been transfered.
+		    inReadBytes(pktlen);	// only count deserialised packets, because that's what is actually been transfered.
 	    }
 	    else if (!is_partial_packet)
 	    {
@@ -1012,7 +1030,7 @@ continue_packet:
     return 0;
 }
 
-RsItem *pqistreamer::addPartialPacket_locked(const void *block, uint32_t len, uint32_t slice_packet_id, bool is_packet_starting, bool is_packet_ending, uint32_t &total_len) 
+RsItem *pqistreamer::addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id, bool is_packet_starting, bool is_packet_ending, uint32_t &total_len) 
 {
 #ifdef DEBUG_PACKET_SLICING
     std::cerr << "Receiving partial packet. size=" << len << ", ID=" << std::hex << slice_packet_id << std::dec << ", starting:" << is_packet_starting << ", ending:" << is_packet_ending ;
@@ -1134,7 +1152,7 @@ int     pqistreamer::outAllowedBytes_locked()
 	// low pass filter on mAvgDtOut
 	mAvgDtOut = PQISTREAM_AVG_DT_FRAC * mAvgDtOut + (1 - PQISTREAM_AVG_DT_FRAC) * dt;
 	
-	double maxout = getMaxRate(false) * 1024.0;
+	double maxout = getMaxRate_locked(false) * 1024.0;
 
 	// this is used to take into account a possible excess of data sent during the previous round
 	mCurrSent -= int(dt * maxout);
@@ -1156,7 +1174,7 @@ int     pqistreamer::outAllowedBytes_locked()
 	return quota;
 }
 
-int     pqistreamer::inAllowedBytes_locked()
+int     pqistreamer::inAllowedBytes()
 {
 	double t = getCurrentTS(); // in sec, with high accuracy
 
@@ -1194,7 +1212,7 @@ int     pqistreamer::inAllowedBytes_locked()
 
 #ifdef DEBUG_PQISTREAMER
 	uint64_t t_now = 1000 * getCurrentTS();
-	std::cerr << std::dec << t_now << " DEBUG_PQISTREAMER pqistreamer::inAllowedBytes_locked PeerId " << this->PeerId().toStdString() << " dt " << (int)(1000 * dt) << "ms, mAvgDtIn " << (int)(1000 * mAvgDtIn) << "ms, maxin " << (int)(maxin) << " bytes/s, mCurrRead " << mCurrRead << " bytes, quota " << (int)(quota) << " bytes" << std::endl;
+	std::cerr << std::dec << t_now << " DEBUG_PQISTREAMER pqistreamer::inAllowedBytes PeerId " << this->PeerId().toStdString() << " dt " << (int)(1000 * dt) << "ms, mAvgDtIn " << (int)(1000 * mAvgDtIn) << "ms, maxin " << (int)(maxin) << " bytes/s, mCurrRead " << mCurrRead << " bytes, quota " << (int)(quota) << " bytes" << std::endl;
 #endif
 
 	return quota;
@@ -1231,7 +1249,7 @@ void    pqistreamer::outSentBytes_locked(uint32_t outb)
 	return;
 }
 
-void    pqistreamer::inReadBytes_locked(uint32_t inb)
+void    pqistreamer::inReadBytes(uint32_t inb)
 {
 #ifdef DEBUG_PQISTREAMER
 	{
@@ -1248,7 +1266,7 @@ void    pqistreamer::inReadBytes_locked(uint32_t inb)
 	return;
 }
 
-void pqistreamer::allocate_rpend_locked()
+void pqistreamer::allocate_rpend()
 {
     if(mPkt_rpending)
         return;
@@ -1271,17 +1289,17 @@ int pqistreamer::reset()
 #ifdef DEBUG_PQISTREAMER
 	std::cerr << "pqistreamer::reset()" << std::endl;
 #endif
-	free_pend_locked();
+	free_pend();
     
     return 1 ;
 }
 
-void pqistreamer::free_pend_locked()
+void pqistreamer::free_pend()
 {
 	if(mPkt_rpending)
 	{
 #ifdef DEBUG_PQISTREAMER
-        		std::cerr << "pqistreamer::free_pend_locked(): pending input packet buffer" << std::endl;
+        		std::cerr << "pqistreamer::free_pend(): pending input packet buffer" << std::endl;
 #endif
 		free(mPkt_rpending);
 		mPkt_rpending = 0;
@@ -1291,7 +1309,7 @@ void pqistreamer::free_pend_locked()
 	if (mPkt_wpending)
 	{
 #ifdef DEBUG_PQISTREAMER
-        		std::cerr << "pqistreamer::free_pend_locked(): pending output packet buffer" << std::endl;
+        		std::cerr << "pqistreamer::free_pend(): pending output packet buffer" << std::endl;
 #endif
 		free(mPkt_wpending);
 		mPkt_wpending = NULL;
@@ -1300,7 +1318,7 @@ void pqistreamer::free_pend_locked()
 
 #ifdef DEBUG_PQISTREAMER
     if(!mPartialPackets.empty())
-        		std::cerr << "pqistreamer::free_pend_locked(): " << mPartialPackets.size() << " pending input partial packets" << std::endl;
+        		std::cerr << "pqistreamer::free_pend(): " << mPartialPackets.size() << " pending input partial packets" << std::endl;
 #endif
 	// also delete any incoming partial packet
 	for(std::map<uint32_t,PartialPacketRecord>::iterator it(mPartialPackets.begin());it!=mPartialPackets.end();++it)
@@ -1318,26 +1336,47 @@ int     pqistreamer::gatherStatistics(std::list<RSTrafficClue>& outqueue_lst,std
 
     return locked_gatherStatistics(outqueue_lst,inqueue_lst);
 }
+
+// this method is overloaded by pqiqosstreamer
 int     pqistreamer::getQueueSize(bool in)
 {
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
-
 	if (in)
-        return mIncomingSize;
-    else
-        return locked_out_queue_size();
+// no mutex is needed here because this is atomic
+		return mIncomingSize;
+	else
+	{
+		RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+		return locked_out_queue_size();
+	}
+}
+
+int     pqistreamer::getQueueSize_bytes(bool in)
+{
+        if (in)
+// no mutex is needed here because this is atomic
+// for future use, mIncomingSize_bytes is not updated yet
+                return mIncomingSize_bytes;
+        else
+        {
+                RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+                return locked_compute_out_pkt_size();
+        }
 }
 
 void    pqistreamer::getRates(RsBwRates &rates)
 {
 	RateInterface::getRates(rates);
 
-	RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+// no mutex is needed here because this is atomic
+	rates.mQueueIn = mIncomingSize;
 
-    rates.mQueueIn = mIncomingSize;
-	rates.mQueueOut = locked_out_queue_size();
+	{
+		RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/
+		rates.mQueueOut = locked_out_queue_size();
+	}
 }
 
+// this method is overloaded by pqiqosstreamer
 int pqistreamer::locked_out_queue_size() const
 {
 	// Warning: because out_pkt is a list, calling size
@@ -1347,6 +1386,7 @@ int pqistreamer::locked_out_queue_size() const
 	return mOutPkts.size() ; 
 }
 
+// this method is overloaded by pqiqosstreamer
 void pqistreamer::locked_clear_out_queue()
 {
 	for(std::list<void*>::iterator it = mOutPkts.begin(); it != mOutPkts.end(); )
@@ -1361,6 +1401,7 @@ void pqistreamer::locked_clear_out_queue()
 	}
 }
 
+// this method is overloaded by pqiqosstreamer
 int pqistreamer::locked_compute_out_pkt_size() const
 {
 	int total = 0 ;
@@ -1379,6 +1420,7 @@ int pqistreamer::locked_gatherStatistics(std::list<RSTrafficClue>& out_lst,std::
     return 1 ;
 }
 
+// this method is overloaded by pqiqosstreamer
 void *pqistreamer::locked_pop_out_data(uint32_t /*max_slice_size*/, uint32_t &size, bool &starts, bool &ends, uint32_t &packet_id)
 {
     size = 0 ;
@@ -1400,4 +1442,3 @@ void *pqistreamer::locked_pop_out_data(uint32_t /*max_slice_size*/, uint32_t &si
 	return res ;
 }
 
-    

libretroshare/src/pqi/pqistreamer.h

@@ -38,8 +38,8 @@ class RsSerialiser;
 
 struct PartialPacketRecord
 {
-    void *mem ;
-    uint32_t size ;
+	void *mem ;
+	uint32_t size ;
 };
 
 /**
@@ -65,18 +65,23 @@ class pqistreamer: public PQInterface
 		virtual RsItem *GetItem();
 		virtual int     status();
 
-		rstime_t  getLastIncomingTS(); 	// Time of last data packet, for checking a connection is alive.
+		rstime_t	getLastIncomingTS(); 	// Time of last data packet, for checking a connection is alive.
 		virtual void    getRates(RsBwRates &rates);
 		virtual int     getQueueSize(bool in); // extracting data.
+		virtual int     getQueueSize_bytes(bool in); // size of incoming queue in bytes
 		virtual int     gatherStatistics(std::list<RSTrafficClue>& outqueue_stats,std::list<RSTrafficClue>& inqueue_stats); // extracting data.
         
             	// mutex protected versions of RateInterface calls.
             	virtual void setRate(bool b,float f) ;
             	virtual void setMaxRate(bool b,float f) ;
-            	virtual float getRate(bool b) ;
+		virtual void setMaxRate_locked(bool b,float f) ;
 
-    protected:
-        		virtual int reset() ;
+            	virtual float getRate(bool b) ;
+		virtual float getMaxRate(bool b) ;
+		virtual float getMaxRate_locked(bool b);
+
+	protected:
+       		virtual int reset() ;
 
 		int tick_bio();
 		int tick_send(uint32_t timeout);
@@ -104,12 +109,12 @@ class pqistreamer: public PQInterface
 
 	private:
 		int queue_outpqi_locked(RsItem *i,uint32_t& serialized_size);
-		int handleincomingitem_locked(RsItem *i, int len);
+		int handleincomingitem(RsItem *i, int len);
 
 		// ticked regularly (manages out queues and sending
 		// via above interfaces.
 		virtual int	handleoutgoing_locked();
-		virtual int	handleincoming_locked();
+		virtual int	handleincoming();
 
 		// Bandwidth/Streaming Management.
 		float	outTimeSlice_locked();
@@ -117,11 +122,11 @@ class pqistreamer: public PQInterface
 		int	outAllowedBytes_locked();
 		void	outSentBytes_locked(uint32_t );
 
-		int	inAllowedBytes_locked();
-		void	inReadBytes_locked(uint32_t );
+		int	inAllowedBytes();
+		void	inReadBytes(uint32_t );
 
         		// cleans up everything that's pending / half finished.
-		void free_pend_locked();
+		void free_pend();
 
 		// RsSerialiser - determines which packets can be serialised.
 		RsSerialiser *mRsSerialiser;
@@ -129,13 +134,12 @@ class pqistreamer: public PQInterface
 		void *mPkt_wpending; // storage for pending packet to write.
         	uint32_t mPkt_wpending_size; // ... and its size.
 
-        void allocate_rpend_locked(); // use these two functions to allocate/free the buffer below
+		void allocate_rpend(); // use these two functions to allocate/free the buffer below
         
 		int   mPkt_rpend_size; // size of pkt_rpending.
 		void *mPkt_rpending; // storage for read in pending packets.
 
-		enum {reading_state_packet_started=1,
-			reading_state_initial=0 } ;
+		enum {reading_state_packet_started=1, reading_state_initial=0 } ;
 
 		int   mReading_state ;
 		int   mFailed_read_attempts ;
@@ -144,7 +148,8 @@ class pqistreamer: public PQInterface
 		std::list<void *> mOutPkts; // Cntrl / Search / Results queue
 		std::list<RsItem *> mIncoming;
 
-        uint32_t mIncomingSize; // size of mIncoming. To avoid calling linear cost std::list::size()
+		uint32_t mIncomingSize; // size of mIncoming. To avoid calling linear cost std::list::size()
+		uint32_t mIncomingSize_bytes; // size of Incoming in btyes
 
 		// data for network stats.
 		int mTotalRead;
@@ -154,8 +159,8 @@ class pqistreamer: public PQInterface
 		int mCurrRead;
 		int mCurrSent;
         
-        double mCurrReadTS; // TS from which these are measured.
-        double mCurrSentTS;
+		double mCurrReadTS; // TS from which these are measured.
+		double mCurrSentTS;
 
 		double mAvgLastUpdate; // TS from which these are measured.
 		uint32_t mAvgReadCount;
@@ -174,12 +179,12 @@ class pqistreamer: public PQInterface
         	std::list<RSTrafficClue> mCurrentStatsChunk_Out ;
 		rstime_t mStatisticsTimeStamp ;
 
-        bool mAcceptsPacketSlicing ;
-        rstime_t mLastSentPacketSlicingProbe ;
-        void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list<RSTrafficClue> &lst);
-        RsItem *addPartialPacket_locked(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len);
+		bool mAcceptsPacketSlicing ;
+		rstime_t mLastSentPacketSlicingProbe ;
+		void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list<RSTrafficClue> &lst);
+		RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len);
         
-        std::map<uint32_t,PartialPacketRecord> mPartialPackets ;
+		std::map<uint32_t,PartialPacketRecord> mPartialPackets ;
 };
 
 #endif //MRK_PQI_STREAMER_HEADER

libretroshare/src/pqi/pqithreadstreamer.cc

@@ -23,17 +23,17 @@
 #include "pqi/pqithreadstreamer.h"
 #include <unistd.h>
 
-#define DEFAULT_STREAMER_TIMEOUT	  10000 // 10 ms.
-#define DEFAULT_STREAMER_SLEEP		   1000 // 1 ms.
+#define DEFAULT_STREAMER_TIMEOUT	  10000 // 10 ms
+#define DEFAULT_STREAMER_SLEEP		  30000 // 30 ms
 #define DEFAULT_STREAMER_IDLE_SLEEP	1000000 // 1 sec
 
-//#define PQISTREAMER_DEBUG
+// #define PQISTREAMER_DEBUG
 
 pqithreadstreamer::pqithreadstreamer(PQInterface *parent, RsSerialiser *rss, const RsPeerId& id, BinInterface *bio_in, int bio_flags_in)
 :pqistreamer(rss, id, bio_in, bio_flags_in), mParent(parent), mTimeout(0), mThreadMutex("pqithreadstreamer")
 {
-    mTimeout = DEFAULT_STREAMER_TIMEOUT;
-    mSleepPeriod = DEFAULT_STREAMER_SLEEP;
+	mTimeout = DEFAULT_STREAMER_TIMEOUT;
+	mSleepPeriod = DEFAULT_STREAMER_SLEEP;
 }
 
 bool pqithreadstreamer::RecvItem(RsItem *item)
@@ -43,55 +43,59 @@ bool pqithreadstreamer::RecvItem(RsItem *item)
 
 int	pqithreadstreamer::tick()
 {
-        RsStackMutex stack(mThreadMutex);
-    tick_bio();
+	// pqithreadstreamer mutex lock is not needed here
+	// we will only check if the connection is active, and if not we will try to establish it
+	tick_bio();
 
 	return 0;
 }
 
 void	pqithreadstreamer::threadTick()
 {
-    uint32_t recv_timeout = 0;
-    uint32_t sleep_period = 0;
-    bool isactive = false;
-    {
-        RsStackMutex stack(mStreamerMtx);
-        recv_timeout = mTimeout;
-        sleep_period = mSleepPeriod;
-        isactive = mBio->isactive();
-    }
+	uint32_t recv_timeout = 0;
+	uint32_t sleep_period = 0;
+	bool isactive = false;
+
+	{
+		RsStackMutex stack(mStreamerMtx);
+		recv_timeout = mTimeout;
+		sleep_period = mSleepPeriod;
+		isactive = mBio->isactive();
+	}
     
-    updateRates() ;
+	// update the connection rates
+	updateRates() ;
 
-    if (!isactive)
-    {
-        rstime::rs_usleep(DEFAULT_STREAMER_IDLE_SLEEP);
-        return ;
-    }
+	// if the connection est not active, long sleep then return
+	if (!isactive)
+	{
+		rstime::rs_usleep(DEFAULT_STREAMER_IDLE_SLEEP);
+		return ;
+	}
 
-    {
-        RsStackMutex stack(mThreadMutex);
-        tick_recv(recv_timeout);
-    }
+	// fill incoming queue with items from SSL
+	{
+		RsStackMutex stack(mThreadMutex);
+		tick_recv(recv_timeout);
+	}
 
-    // Push Items, Outside of Mutex.
-    RsItem *incoming = NULL;
-    while((incoming = GetItem()))
-    {
-        RecvItem(incoming);
-    }
+	// move items to appropriate service queue or shortcut  to fast service
+	RsItem *incoming = NULL;
+	while((incoming = GetItem()))
+	{
+		RecvItem(incoming);
+	}
 
-    {
-        RsStackMutex stack(mThreadMutex);
-        tick_send(0);
-    }
+	// parse the outgoing queue and send items to SSL
+	{
+		RsStackMutex stack(mThreadMutex);
+		tick_send(0);
+	}
 
-    if (sleep_period)
-    {
-        rstime::rs_usleep(sleep_period);
-    }
+	// sleep 
+	if (sleep_period)
+	{
+		rstime::rs_usleep(sleep_period);
+	}
 }
 
-
-
-

libretroshare/src/retroshare/rsevents.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2020  Retroshare Team <contact@retroshare.cc>            *
+ * Copyright (C) 2019-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -31,6 +33,7 @@
 #include "serialiser/rsserializable.h"
 #include "serialiser/rstypeserializer.h"
 #include "util/rstime.h"
+#include "util/rsdebug.h"
 
 class RsEvents;
 
@@ -50,7 +53,7 @@ extern RsEvents* rsEvents;
  */
 enum class RsEventType : uint32_t
 {
-	NONE = 0, /// Used to detect uninitialized event
+	__NONE = 0, /// Used internally to detect invalid event type passed
 
 	/// @see RsBroadcastDiscovery
 	BROADCAST_DISCOVERY                                     = 1,
@@ -64,7 +67,7 @@ enum class RsEventType : uint32_t
 	/// @see pqissl
 	PEER_CONNECTION                                         = 4,
 
-	/// @see RsGxsChanges												// this one is used in RsGxsBroadcast
+	/// @see RsGxsChanges, used also in @see RsGxsBroadcast
 	GXS_CHANGES                                             = 5,
 
 	/// Emitted when a peer state changes, @see RsPeers
@@ -94,9 +97,62 @@ enum class RsEventType : uint32_t
     /// @see RsFiles
     FILE_TRANSFER                                           = 14,
 
-	 MAX       /// Used to detect invalid event type passed
+	/// @see RsMsgs
+	CHAT_MESSAGE                                            = 15,
+
+	__MAX /// Used internally, keep last
 };
 
+enum class RsEventsErrorNum : int32_t
+{
+	EVENT_TYPE_UNDEFINED             = 3004,
+	EVENT_TYPE_OUT_OF_RANGE          = 3005,
+	INVALID_HANDLER_ID               = 3006,
+	NULL_EVENT_POINTER               = 3007
+};
+
+struct RsEventsErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare Events"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsEventsErrorNum>(ev))
+		{
+		case RsEventsErrorNum::EVENT_TYPE_UNDEFINED:
+			return "Undefined event type";
+		case RsEventsErrorNum::EVENT_TYPE_OUT_OF_RANGE:
+			return "Event type out of range";
+		case RsEventsErrorNum::INVALID_HANDLER_ID:
+			return "Invalid handler id";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	std::error_condition default_error_condition(int ev) const noexcept override;
+
+	const static RsEventsErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsJsonApiErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsEventsErrorNum> : true_type {};
+}
+
+/** Provide RsEventsErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsJsonApiErrorNum */
+inline std::error_condition make_error_condition(RsEventsErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsEventsErrorCategory::instance );
+};
+
+
 /**
  * This struct is not meant to be used directly, you should create events type
  * deriving from it.
@@ -104,7 +160,7 @@ enum class RsEventType : uint32_t
 struct RsEvent : RsSerializable
 {
 protected:
-	RsEvent(RsEventType type) :
+	explicit RsEvent(RsEventType type) :
 	    mType(type), mTimePoint(std::chrono::system_clock::now()) {}
 
 	RsEvent() = delete;
@@ -139,27 +195,19 @@ public:
 	/**
 	 * @brief Post event to the event queue.
 	 * @param[in] event
-	 * @param[out] errorMessage Optional storage for error messsage, meaningful
-	 *                          only on failure.
-	 * @return False on error, true otherwise.
+	 * @return Success or error details.
 	 */
-	virtual bool postEvent(
-	        std::shared_ptr<const RsEvent> event,
-	        std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string)
-	        ) = 0;
+	virtual std::error_condition postEvent(
+	        std::shared_ptr<const RsEvent> event ) = 0;
 
 	/**
 	 * @brief Send event directly to handlers. Blocking API
 	 * The handlers get exectuded on the caller thread.
 	 * @param[in] event
-	 * @param[out] errorMessage Optional storage for error messsage, meaningful
-	 *                          only on failure.
-	 * @return False on error, true otherwise.
+	 * @return Success or error details.
 	 */
-	virtual bool sendEvent(
-	        std::shared_ptr<const RsEvent> event,
-	        std::string& errorMessage = RS_DEFAULT_STORAGE_PARAM(std::string)
-	        ) = 0;
+	virtual std::error_condition sendEvent(
+	        std::shared_ptr<const RsEvent> event ) = 0;
 
 	/**
 	 * @brief Generate unique handler identifier
@@ -171,8 +219,9 @@ public:
 	 * @brief Register events handler
 	 * Every time an event is dispatced the registered events handlers will get
 	 * their method handleEvent called with the event passed as paramether.
+	 * @attention Callbacks must not fiddle internally with methods of this
+	 * class otherwise a deadlock will happen.
 	 * @jsonapi{development,manualwrapper}
-	 * @param eventType         Type of event for which the callback is called
 	 * @param multiCallback     Function that will be called each time an event
 	 *                          is dispatched.
 	 * @param[inout] hId        Optional storage for handler id, useful to
@@ -180,21 +229,23 @@ public:
 	 *                          value may be provided to the function call but
 	 *                          must habe been generated with
 	 *                          @see generateUniqueHandlerId()
-	 * @return False on error, true otherwise.
+	 * @param[in] eventType     Optional type of event for which the callback is
+	 *                          called, if __NONE is passed multiCallback is
+	 *                          called for every events without filtering.
+	 * @return Success or error details.
 	 */
-	virtual bool registerEventsHandler(
-            RsEventType eventType,
+	virtual std::error_condition registerEventsHandler(
 	        std::function<void(std::shared_ptr<const RsEvent>)> multiCallback,
-	        RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0)
-	        ) = 0;
+	        RsEventsHandlerId_t& hId = RS_DEFAULT_STORAGE_PARAM(RsEventsHandlerId_t, 0),
+	        RsEventType eventType = RsEventType::__NONE ) = 0;
 
 	/**
 	 * @brief Unregister event handler
 	 * @param[in] hId Id of the event handler to unregister
-	 * @return True if the handler id has been found, false otherwise.
+	 * @return Success or error details.
 	 */
-	virtual bool unregisterEventsHandler(RsEventsHandlerId_t hId) = 0;
+	virtual std::error_condition unregisterEventsHandler(
+	        RsEventsHandlerId_t hId ) = 0;
 
 	virtual ~RsEvents();
 };
-

libretroshare/src/retroshare/rsfiles.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2008  Robert Fernie <retroshare@lunamutt.com>                 *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -28,12 +29,15 @@
 #include <functional>
 #include <chrono>
 #include <cstdint>
+#include <system_error>
 
 #include "rstypes.h"
 #include "serialiser/rsserializable.h"
 #include "rsturtle.h"
 #include "util/rstime.h"
 #include "retroshare/rsevents.h"
+#include "util/rsmemory.h"
+#include "util/rsdebug.h"
 
 class RsFiles;
 
@@ -43,6 +47,49 @@ class RsFiles;
  */
 extern RsFiles* rsFiles;
 
+enum class RsFilesErrorNum : int32_t
+{
+	FILES_HANDLE_NOT_FOUND                = 2004,
+};
+
+struct RsFilesErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare Files"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsFilesErrorNum>(ev))
+		{
+		case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND:
+			return "Files handle not found";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	std::error_condition default_error_condition(int ev) const noexcept override;
+
+	const static RsFilesErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsFilesErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsFilesErrorNum> : true_type {};
+}
+
+/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsJsonApiErrorNum */
+inline std::error_condition make_error_condition(RsFilesErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsFilesErrorCategory::instance );
+};
+
+
 namespace RsRegularExpression { class Expression; }
 
 /* These are used mainly by ftController at the moment */
@@ -96,8 +143,28 @@ const FileSearchFlags RS_FILE_HINTS_BROWSABLE              ( 0x00000100 );// bro
 const FileSearchFlags RS_FILE_HINTS_SEARCHABLE             ( 0x00000200 );// can be searched anonymously
 const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK        ( 0x00000380 );// OR of the last tree flags. Used to filter out.
 
-// Flags used when requesting a transfer
-//
+/** Flags used when requesting a new file transfer */
+enum class FileRequestFlags: uint32_t
+{
+	/// Enable requesting file via turtle routing.
+	ANONYMOUS_ROUTING = 0x00000040,
+
+	/// Asks (TODO: or enforce?) for end-to-end encryption of file trasfer
+	ENCRYPTED = 0x00000080,
+
+	/// Asks (TODO: or enforce?) no end-to-end encryption of file trasfer
+	UNENCRYPTED = 0x00000100,
+
+	/// Start trasfer very slow
+	SLOW = 0x00002000,
+
+	/// Disable searching for potential direct sources
+	NO_DIRECT_SEARCH = 0x02000000
+};
+RS_REGISTER_ENUM_FLAGS_TYPE(FileRequestFlags)
+
+/// @deprecated Flags used when requesting a transfer
+/// @see FileRequestFlags instead
 const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING   ( 0x00000040 ); // Use to ask turtle router to download the file.
 const TransferRequestFlags RS_FILE_REQ_ENCRYPTED           ( 0x00000080 ); // Asks for end-to-end encryption of file at the level of ftServer
 const TransferRequestFlags RS_FILE_REQ_UNENCRYPTED         ( 0x00000100 ); // Asks for no end-to-end encryption of file at the level of ftServer
@@ -111,10 +178,6 @@ const TransferRequestFlags RS_FILE_REQ_NO_SEARCH           ( 0x02000000 );	// di
 // const uint32_t RS_FILE_HINTS_SHARE_FLAGS_MASK	 = 	RS_FILE_HINTS_NETWORK_WIDE_OTHERS | RS_FILE_HINTS_BROWSABLE_OTHERS
 // 																	 | RS_FILE_HINTS_NETWORK_WIDE_GROUPS | RS_FILE_HINTS_BROWSABLE_GROUPS ;
 
-/* Callback Codes */
-
-const uint32_t RS_FILE_EXTRA_DELETE	 = 0x0010;
-
 enum class RsSharedDirectoriesEventCode: uint8_t {
     UNKNOWN                     = 0x00,
     STARTING_DIRECTORY_SWEEP    = 0x01, // (void)
@@ -202,36 +265,117 @@ struct SharedDirStats
     uint64_t total_shared_size ;
 };
 
-// This class represents a tree of directories and files, only with their names size and hash. It is used to create collection links in the GUI
-// and to transmit directory information between services. This class is independent from the existing FileHierarchy classes used in storage because
-// we need a very copact serialization and storage size since we create links with it. Besides, we cannot afford to risk the leak of other local information
-// by using the orignal classes.
-
-class FileTree
+/** This class represents a tree of directories and files, only with their names
+ * size and hash. It is used to create collection links in the GUI and to
+ * transmit directory information between services. This class is independent
+ * from the existing FileHierarchy classes used in storage because we need a
+ * very copact serialization and storage size since we create links with it.
+ * Besides, we cannot afford to risk the leak of other local information
+ *  by using the orignal classes.
+ */
+struct RsFileTree : RsSerializable
 {
 public:
-	virtual ~FileTree() {}
+	RsFileTree() : mTotalFiles(0), mTotalSize(0) {}
 
-	static FileTree *create(const DirDetails& dd, bool remote, bool remove_top_dirs = true) ;
-	static FileTree *create(const std::string& radix64_string) ;
+	/**
+	 * @brief Create a RsFileTree from directory details
+	 * @param dd directory or file details
+	 * @param remote
+	 * @param remove_top_dirs
+	 * @return pointer to the created file-tree
+	 */
+	static std::unique_ptr<RsFileTree> fromDirDetails(
+	        const DirDetails& dd, bool remote, bool remove_top_dirs = true );
 
-	virtual std::string toRadix64() const =0 ;
+	/**
+	 * @brief Create a RsFileTree from Base64 representation
+	 * @param base64 base64 or base64url string representation of the file-tree
+	 * @return pointer to the parsed file-tree on success, nullptr plus error
+	 *	details on failure
+	 */
+	static std::tuple<std::unique_ptr<RsFileTree>, std::error_condition>
+	fromBase64(const std::string& base64);
 
-	// These methods allow the user to browse the hierarchy
+	/** @brief Convert to base64 representetion */
+	std::string toBase64() const;
 
-	struct FileData {
-		std::string name ;
-		uint64_t    size ;
-		RsFileHash  hash ;
+	/// @see RsSerializable
+	virtual void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mFiles);
+		RS_SERIAL_PROCESS(mDirs);
+		RS_SERIAL_PROCESS(mTotalFiles);
+		RS_SERIAL_PROCESS(mTotalSize);
+	}
+
+	struct FileData: RsSerializable
+	{
+		std::string name;
+		uint64_t    size;
+		RsFileHash  hash;
+
+		void serial_process(
+		        RsGenericSerializer::SerializeJob j,
+		        RsGenericSerializer::SerializeContext& ctx ) override;
 	};
 
-	virtual uint32_t root() const { return 0;}
-	virtual bool getDirectoryContent(uint32_t index,std::string& name,std::vector<uint32_t>& subdirs,std::vector<FileData>& subfiles) const = 0;
+	/// Allow browsing the hierarchy
+	bool getDirectoryContent(
+	        std::string& name, std::vector<uint64_t>& subdirs,
+	        std::vector<FileData>& subfiles, uint64_t handle = 0 ) const;
 
-	virtual void print() const=0;
+	struct DirData: RsSerializable
+	{
+		std::string name;
+		std::vector<uint64_t> subdirs;
+		std::vector<uint64_t> subfiles;
 
-	uint32_t mTotalFiles ;
-	uint64_t mTotalSize ;
+		void serial_process(
+		        RsGenericSerializer::SerializeJob j,
+		        RsGenericSerializer::SerializeContext& ctx ) override;
+	};
+
+	static void recurs_buildFileTree(
+	        RsFileTree& ft, uint32_t index, const DirDetails& dd, bool remote,
+	        bool remove_top_dirs );
+
+	std::vector<FileData> mFiles;
+	std::vector<DirData> mDirs;
+
+	uint32_t mTotalFiles;
+	uint64_t mTotalSize;
+
+	virtual ~RsFileTree();
+
+	/**
+	 * @brief Create a RsFileTree from Radix64 representation
+	 * @deprecated kept for retrocompatibility with RetroShare-gui
+	 * The format is not compatible with the new methods
+	 * @param radix64_string
+	 * @return nullptr if on failure, pointer to the created FileTree on success
+	 */
+	RS_DEPRECATED_FOR(fromBase64)
+	static std::unique_ptr<RsFileTree> fromRadix64(
+	        const std::string& radix64_string );
+
+	/** @brief Convert to radix64 representetion
+	 * @deprecated kept for retrocompatibility with RetroShare-gui
+	 * The format is not compatible with the new methods
+	 */
+	RS_DEPRECATED_FOR(toBase64)
+	std::string toRadix64() const;
+
+private:
+	/** @deprecated kept for retrocompatibility with RetroShare-gui */
+	RS_DEPRECATED_FOR(serial_process)
+	bool serialise(unsigned char *& data,uint32_t& data_size) const;
+
+	/** @deprecated kept for retrocompatibility with RetroShare-gui */
+	RS_DEPRECATED_FOR(serial_process)
+	bool deserialise(unsigned char* data, uint32_t data_size);
 };
 
 struct BannedFileEntry : RsSerializable
@@ -258,12 +402,12 @@ struct TurtleFileInfoV2 : RsSerializable
 {
 	TurtleFileInfoV2() : fSize(0), fWeight(0) {}
 
-	TurtleFileInfoV2(const TurtleFileInfo& oldInfo) :
+	explicit TurtleFileInfoV2(const TurtleFileInfo& oldInfo) :
 	    fSize(oldInfo.size), fHash(oldInfo.hash), fName(oldInfo.name),
 	    fWeight(0) {}
 
 #ifdef RS_DEEP_FILES_INDEX
-	TurtleFileInfoV2(const DeepFilesSearchResult& dRes);
+	explicit TurtleFileInfoV2(const DeepFilesSearchResult& dRes);
 #endif // def RS_DEEP_FILES_INDEX
 
 	uint64_t fSize;    /// File size
@@ -301,9 +445,6 @@ struct TurtleFileInfoV2 : RsSerializable
 class RsFiles
 {
 public:
-	RsFiles() {}
-	virtual ~RsFiles() {}
-
 	/**
 	 * @brief Provides file data for the GUI, media streaming or API clients.
 	 * It may return unverified chunks. This allows streaming without having to
@@ -344,10 +485,10 @@ public:
 	/**
 	 * @brief Initiate downloading of a file
 	 * @jsonapi{development}
-	 * @param[in] fileName
-	 * @param[in] hash
-	 * @param[in] size
-	 * @param[in] destPath in not empty specify a destination path
+	 * @param[in] fileName file name
+	 * @param[in] hash file hash
+	 * @param[in] size file size
+	 * @param[in] destPath optional specify the destination directory
 	 * @param[in] flags you usually want RS_FILE_REQ_ANONYMOUS_ROUTING
 	 * @param[in] srcIds eventually specify known sources
 	 * @return false if we already have the file, true otherwhise
@@ -357,6 +498,25 @@ public:
 	        const std::string& destPath, TransferRequestFlags flags,
 	        const std::list<RsPeerId>& srcIds ) = 0;
 
+	/**
+	 * @brief Initiate download of a files collection
+	 * @jsonapi{development}
+	 * An usually useful companion method of this is @see parseFilesLink()
+	 * @param[in] collection collection of files to download
+	 * @param[in] destPath optional base path on which to download the
+	 *	collection, if left empty the default download directory will be used
+	 * @param[in] srcIds optional peers id known as direct source of the
+	 *	collection
+	 * @param[in] flags optional flags to fine tune search and download
+	 *	algorithm
+	 * @return success or error details.
+	 */
+	virtual std::error_condition requestFiles(
+	        const RsFileTree& collection,
+	        const std::string& destPath = "",
+	        const std::vector<RsPeerId>& srcIds = std::vector<RsPeerId>(),
+	        FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING ) = 0;
+
 	/**
 	 * @brief Cancel file downloading
 	 * @jsonapi{development}
@@ -498,7 +658,8 @@ public:
 	 * @brief Get file details
 	 * @jsonapi{development}
 	 * @param[in] hash file identifier
-	 * @param[in] hintflags filtering hint (RS_FILE_HINTS_EXTRA|...|RS_FILE_HINTS_LOCAL)
+	 * @param[in] hintflags filtering hint ( RS_FILE_HINTS_UPLOAD|...|
+	 *	RS_FILE_HINTS_EXTRA|RS_FILE_HINTS_LOCAL )
 	 * @param[out] info storage for file information
 	 * @return true if file found, false otherwise
 	 */
@@ -575,7 +736,7 @@ public:
 	 * @return false if error occurred, true otherwise
 	 */
 	virtual bool requestDirDetails(
-	        DirDetails &details, std::uintptr_t handle = 0,
+	        DirDetails &details, uint64_t handle = 0,
 	        FileSearchFlags flags = RS_FILE_HINTS_LOCAL ) = 0;
 
 	/***
@@ -584,8 +745,9 @@ public:
 	/**
 	 * Kept for retrocompatibility, it was originally written for easier
 	 * interaction with Qt. As soon as you can, you should prefer to use the
-	 * version of this methodn which take `std::uintptr_t handle` as paramether.
+	 * version of this method which take `uint64_t handle` as paramether.
 	 */
+	RS_DEPRECATED_FOR(requestDirDetails)
 	virtual int RequestDirDetails(
 	        void* handle, DirDetails& details, FileSearchFlags flags ) = 0;
 
@@ -726,6 +888,76 @@ public:
 	 */
 	virtual bool removeSharedDirectory(std::string dir) = 0;
 
+	/// Default base URL used for files links @see exportFilesLink
+	static const std::string DEFAULT_FILES_BASE_URL;
+
+	/** Link query field used to store collection files count
+	 * @see exportFilesLink */
+	static const std::string FILES_URL_COUNT_FIELD;
+
+	/// Link query field used to store collection data @see exportFilesLink
+	static const std::string FILES_URL_DATA_FIELD;
+
+	/** Link query FILES_URL_DATA_FIELD field value used to instruct the parser
+	 * to look for the data into the link fragment
+	 * @see exportFilesLink and parseFilesLink */
+	static const std::string FILES_URL_FAGMENT_FORWARD;
+
+	/// Link query field used to store collection name @see exportFilesLink
+	static const std::string FILES_URL_NAME_FIELD;
+
+	/// Link query field used to store collection size @see exportFilesLink
+	static const std::string FILES_URL_SIZE_FIELD;
+
+	/**
+	 * @brief Export link to a collection of files
+	 * @jsonapi{development}
+	 * @param[out] link storage for the generated link
+	 * @param[in] handle directory RetroShare handle
+	 * @param[in] fragSneak when true the file data is sneaked into fragment
+	 *	instead of FILES_URL_DATA_FIELD query field, this way if using an
+	 *	http[s] link to pass around a disguised file link a misconfigured host
+	 *	attempting to visit that link with a web browser will not send the file
+	 *	data to the server thus protecting at least some of the privacy of the
+	 *	user even in a misconfiguration scenario.
+	 * @param[in] baseUrl URL into which to sneak in the RetroShare file link
+	 *	base64, this is primarly useful to induce applications into making the
+	 *	link clickable, or to disguise the RetroShare link into a "normal"
+	 *	looking web link. If empty the collection data link will be outputted in
+	 *	plain base64 format.
+	 * @return error information if some error occurred, 0/SUCCESS otherwise
+	 */
+	virtual std::error_condition exportCollectionLink(
+	        std::string& link, uint64_t handle, bool fragSneak = false,
+	        const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0;
+
+	/**
+	 * @brief Export link to a file
+	 * @jsonapi{development}
+	 * @param[out] link @see exportCollectionLink
+	 * @param[in] fileHash hash of the file
+	 * @param[in] fileSize size of the file
+	 * @param[in] fileName name of the file
+	 * @param[in] fragSneak @see exportCollectionLink
+	 * @param[in] baseUrl @see exportCollectionLink
+	 * @return error @see exportCollectionLink
+	 */
+	virtual std::error_condition exportFileLink(
+	        std::string& link, const RsFileHash& fileHash, uint64_t fileSize,
+	        const std::string& fileName, bool fragSneak = false,
+	        const std::string& baseUrl = RsFiles::DEFAULT_FILES_BASE_URL ) = 0;
+
+	/**
+	 * @brief Parse RetroShare files link
+	 * @jsonapi{development}
+	 * Support also old RetroShare-gui file and collections links format.
+	 * @param[in] link files link either in base64 or URL format
+	 * @param[out] collection storage for parsed files link
+	 * @return error information if some error occurred, 0/SUCCESS otherwise
+	 */
+	virtual std::error_condition parseFilesLink(
+	        const std::string& link, RsFileTree& collection ) = 0;
+
 	/**
 	 * @brief Get list of ignored file name prefixes and suffixes
 	 * @param[out] ignoredPrefixes storage for ingored prefixes
@@ -757,4 +989,6 @@ public:
 
 		virtual bool	ignoreDuplicates() = 0;
 		virtual void 	setIgnoreDuplicates(bool ignore) = 0;
+
+	virtual ~RsFiles() = default;
 };

libretroshare/src/retroshare/rsflags.h

@@ -24,6 +24,7 @@
 
 #include <type_traits>
 #include <ostream>
+#include <bitset>
 
 /** Check if given type is a scoped enum */
 template<typename E>
@@ -135,13 +136,7 @@ typename std::enable_if<Rs__BitFlagsOps<EFT>::enabled, std::ostream>::type&
 operator <<(std::ostream& stream, EFT flags)
 {
 	using u_t = typename std::underlying_type<EFT>::type;
-
-	for(int i = sizeof(u_t) << 3; i>=0; --i)
-	{
-		stream << ((static_cast<u_t>(flags) & ( 1 << i )) ? "1" : "0");
-		if( i % 8 == 0 ) stream << " ";
-	}
-	return stream;
+	return stream << std::bitset<sizeof(u_t)>(static_cast<u_t>(flags));
 }
 
 #include <cstdint>
@@ -177,6 +172,7 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
 	    inline t_RsFlags32() : _bits(0) {}
 		inline explicit t_RsFlags32(uint32_t N) : _bits(N) {}					// allows initialization from a set of uint32_t
 
+
 		inline t_RsFlags32<n> operator| (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits | f._bits) ; }
 		inline t_RsFlags32<n> operator^ (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits ^ f._bits) ; }
 		inline t_RsFlags32<n> operator* (const t_RsFlags32<n>& f) const { return t_RsFlags32<n>(_bits & f._bits) ; }
@@ -194,6 +190,19 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
 		//inline explicit operator bool() 	const { return _bits>0; }
 		inline uint32_t toUInt32() const { return _bits ; }
 
+		/// Easier porting to new flag system
+		template<typename EFT> inline
+		typename std::enable_if<(Rs__BitFlagsOps<EFT>::enabled &&
+		                         sizeof(EFT) >= sizeof(uint32_t) ), EFT>::type
+		        toEFT() { return static_cast<EFT>(_bits); }
+
+		/// Easier porting to new flag system
+		template<typename EFT>
+		static inline typename std::enable_if<
+		        Rs__BitFlagsOps<EFT>::enabled &&
+		        sizeof(EFT) <= sizeof(uint32_t), t_RsFlags32>::type
+		        fromEFT(EFT e) { return t_RsFlags32(static_cast<uint32_t>(e)); }
+
 		void clear() { _bits = 0 ; }
 
 		friend std::ostream& operator<<(std::ostream& o,const t_RsFlags32<n>& f) 	// friendly print with 0 and I
@@ -206,8 +215,10 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
 			}
 			return o ;
 		}
-	private:
-		uint32_t _bits ;
+
+private:
+	friend struct RsTypeSerializer;
+	uint32_t _bits;
 };
 
 #define FLAGS_TAG_TRANSFER_REQS 0x4228af
@@ -217,9 +228,9 @@ template<int n> class RS_DEPRECATED_FOR(RS_REGISTER_ENUM_FLAGS_TYPE) t_RsFlags32
 #define FLAGS_TAG_SERVICE_CHAT 	0x839042
 #define FLAGS_TAG_SERIALIZER   	0xa0338d
 
-// Flags for requesting transfers, ask for turtle, cache, speed, etc.
-//
-typedef t_RsFlags32<FLAGS_TAG_TRANSFER_REQS> TransferRequestFlags ;
+/// @deprecated Flags for requesting transfers, ask for turtle, cache, speed, etc.
+RS_DEPRECATED_FOR(FileRequestFlags)
+typedef t_RsFlags32<FLAGS_TAG_TRANSFER_REQS> TransferRequestFlags;
 
 // Flags for file storage. Mainly permissions like BROWSABLE/NETWORK_WIDE for groups and peers.
 //
@@ -237,7 +248,3 @@ typedef t_RsFlags32<FLAGS_TAG_SERVICE_PERM > ServicePermissionFlags ;
 //
 typedef t_RsFlags32<FLAGS_TAG_SERVICE_CHAT > ChatLobbyFlags ;			
 
-// Flags for serializer
-//
-typedef t_RsFlags32<FLAGS_TAG_SERIALIZER > SerializationFlags ;
-

libretroshare/src/retroshare/rsgxschannels.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Robert Fernie <retroshare@lunamutt.com>                 *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -45,11 +46,10 @@ class RsGxsChannels;
 extern RsGxsChannels* rsGxsChannels;
 
 
-struct RsGxsChannelGroup : RsSerializable
+struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData
 {
 	RsGxsChannelGroup() : mAutoDownload(false) {}
 
-	RsGroupMetaData mMeta;
 	std::string mDescription;
 	RsGxsImage mImage;
 
@@ -69,12 +69,10 @@ struct RsGxsChannelGroup : RsSerializable
 	~RsGxsChannelGroup() override;
 };
 
-struct RsGxsChannelPost : RsSerializable
+struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData
 {
 	RsGxsChannelPost() : mCount(0), mSize(0) {}
 
-	RsMsgMetaData mMeta;
-
 	std::set<RsGxsMessageId> mOlderVersions;
 	std::string mMsg;  // UTF8 encoded.
 
@@ -114,18 +112,16 @@ enum class RsChannelEventCode: uint8_t
 	SUBSCRIBE_STATUS_CHANGED        = 0x06, // subscription for channel mChannelGroupId changed.
 	READ_STATUS_CHANGED             = 0x07, // existing message has been read or set to unread
 	RECEIVED_DISTANT_SEARCH_RESULT  = 0x08, // result for the given group id available for the given turtle request id
+	STATISTICS_CHANGED              = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed
 };
 
 struct RsGxsChannelEvent: RsEvent
 {
-	RsGxsChannelEvent():
-	    RsEvent(RsEventType::GXS_CHANNELS),
-	    mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
+	RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
 
 	RsChannelEventCode mChannelEventCode;
 	RsGxsGroupId mChannelGroupId;
 	RsGxsMessageId mChannelMsgId;
-	TurtleRequestId mDistantSearchRequestId;
 
 	///* @see RsEvent @see RsSerializable
 	void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
@@ -135,8 +131,28 @@ struct RsGxsChannelEvent: RsEvent
 		RS_SERIAL_PROCESS(mChannelEventCode);
 		RS_SERIAL_PROCESS(mChannelGroupId);
 		RS_SERIAL_PROCESS(mChannelMsgId);
-		RS_SERIAL_PROCESS(mDistantSearchRequestId);
-    }
+	}
+};
+
+// This event is used to factor multiple search results notifications in a single event.
+
+struct RsGxsChannelSearchResultEvent: RsEvent
+{
+	RsGxsChannelSearchResultEvent():
+	    RsEvent(RsEventType::GXS_CHANNELS),
+	    mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {}
+
+	RsChannelEventCode mChannelEventCode;
+	std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mChannelEventCode);
+		RS_SERIAL_PROCESS(mSearchResultsMap);
+	}
 };
 
 class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
@@ -285,26 +301,6 @@ public:
 	 */
 	virtual bool ExtraFileRemove(const RsFileHash& hash) = 0;
 
-	/**
-	 * @brief Get auto-download option value for given channel
-	 * @jsonapi{development}
-	 * @param[in] channelId channel id
-	 * @param[out] enabled storage for the auto-download option value
-	 * @return false if something failed, true otherwhise
-	 */
-	virtual bool getChannelAutoDownload(
-	        const RsGxsGroupId& channelId, bool& enabled ) = 0;
-
-	/**
-	 * @brief Get download directory for the given channel
-	 * @jsonapi{development}
-	 * @param[in] channelId id of the channel
-	 * @param[out] directory reference to string where to store the path
-	 * @return false on error, true otherwise
-	 */
-	virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId,
-	                                          std::string& directory ) = 0;
-
 	/**
 	 * @brief Get channels summaries list. Blocking API.
 	 * @jsonapi{development}
@@ -326,18 +322,50 @@ public:
 	        std::vector<RsGxsChannelGroup>& channelsInfo ) = 0;
 
 	/**
-	 * @brief Get channel contents
+	 * @brief Get all channel messages and comments in a given channel
+	 * @jsonapi{development}
+	 * @param[in] channelId id of the channel of which the content is requested
+	 * @param[out] posts storage for posts
+	 * @param[out] comments storage for the comments
+	 * @param[out] votes storage for votes
+	 * @return false if something failed, true otherwhise
+	 */
+	virtual bool getChannelAllContent( const RsGxsGroupId& channelId,
+	                                   std::vector<RsGxsChannelPost>& posts,
+	                                   std::vector<RsGxsComment>& comments,
+	                                   std::vector<RsGxsVote>& votes ) = 0;
+
+	/**
+	 * @brief Get channel messages and comments corresponding to the given IDs.
+	 * If the set is empty, nothing is returned.
+	 * @note Since comments are internally themselves messages, it is possible
+	 * to get comments only by supplying their IDs.
 	 * @jsonapi{development}
 	 * @param[in] channelId id of the channel of which the content is requested
 	 * @param[in] contentsIds ids of requested contents
 	 * @param[out] posts storage for posts
 	 * @param[out] comments storage for the comments
+	 * @param[out] votes storage for the votes
 	 * @return false if something failed, true otherwhise
 	 */
 	virtual bool getChannelContent( const RsGxsGroupId& channelId,
-	                       const std::set<RsGxsMessageId>& contentsIds,
-	                       std::vector<RsGxsChannelPost>& posts,
-	                       std::vector<RsGxsComment>& comments ) = 0;
+	                                const std::set<RsGxsMessageId>& contentsIds,
+	                                std::vector<RsGxsChannelPost>& posts,
+	                                std::vector<RsGxsComment>& comments,
+	                                std::vector<RsGxsVote>& votes ) = 0;
+
+	/**
+	 * @brief Get channel comments corresponding to the given IDs.
+	 * If the set is empty, nothing is returned.
+	 * @jsonapi{development}
+	 * @param[in] channelId id of the channel of which the content is requested
+	 * @param[in] contentIds ids of requested contents
+	 * @param[out] comments storage for the comments
+	 * @return false if something failed, true otherwhise
+	 */
+	virtual bool getChannelComments(const RsGxsGroupId &channelId,
+	                                const std::set<RsGxsMessageId> &contentIds,
+	                                std::vector<RsGxsComment> &comments) = 0;
 
 	/**
 	 * @brief Get channel content summaries
@@ -358,16 +386,6 @@ public:
 	 */
 	virtual bool markRead(const RsGxsGrpMsgIdPair& postId, bool read) = 0;
 
-	/**
-	 * @brief Enable or disable auto-download for given channel. Blocking API
-	 * @jsonapi{development}
-	 * @param[in] channelId channel id
-	 * @param[in] enable true to enable, false to disable
-	 * @return false if something failed, true otherwhise
-	 */
-	virtual bool setChannelAutoDownload(
-	        const RsGxsGroupId& channelId, bool enable ) = 0;
-
 	/**
 	 * @brief Share channel publishing key
 	 * This can be used to authorize other peers to post on the channel
@@ -379,16 +397,6 @@ public:
 	virtual bool shareChannelKeys(
 	        const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) = 0;
 
-	/**
-	 * @brief Set download directory for the given channel. Blocking API.
-	 * @jsonapi{development}
-	 * @param[in] channelId id of the channel
-	 * @param[in] directory path
-	 * @return false on error, true otherwise
-	 */
-	virtual bool setChannelDownloadDirectory(
-	        const RsGxsGroupId& channelId, const std::string& directory) = 0;
-
 	/**
 	 * @brief Subscrbe to a channel. Blocking API
 	 * @jsonapi{development}
@@ -399,46 +407,22 @@ public:
 	virtual bool subscribeToChannel( const RsGxsGroupId& channelId,
 	                                 bool subscribe ) = 0;
 
-	/**
-	 * @brief Request remote channels search
+    /**
+     * \brief Retrieve statistics about the channel service
 	 * @jsonapi{development}
-	 * @param[in] matchString string to look for in the search
-	 * @param multiCallback function that will be called each time a search
-	 * result is received
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool turtleSearchRequest(
-	        const std::string& matchString,
-	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
-	        rstime_t maxWait = 300 ) = 0;
+     * \param[out] stat       Statistics structure
+     * \return
+     */
+    virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0;
 
-	/**
-	 * @brief Request remote channel
+    /**
+     * \brief Retrieve statistics about the given channel
 	 * @jsonapi{development}
-	 * @param[in] channelId id of the channel to request to distants peers
-	 * @param multiCallback function that will be called each time a result is
-	 *	received
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool turtleChannelRequest(
-	        const RsGxsGroupId& channelId,
-	        const std::function<void (const RsGxsChannelGroup& result)>& multiCallback,
-	        rstime_t maxWait = 300 ) = 0;
-
-	/**
-	 * @brief Search local channels
-	 * @jsonapi{development}
-	 * @param[in] matchString string to look for in the search
-	 * @param multiCallback function that will be called for each result
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool localSearchRequest(
-	        const std::string& matchString,
-	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
-	        rstime_t maxWait = 30 ) = 0;
+     * \param[in]  channelId  Id of the channel group
+     * \param[out] stat       Statistics structure
+     * \return
+     */
+    virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0;
 
 	/// default base URL used for channels links @see exportChannelLink
 	static const std::string DEFAULT_CHANNEL_BASE_URL;
@@ -484,6 +468,7 @@ public:
 
 	/**
 	 * @brief Import channel from full link
+	 * @jsonapi{development}
 	 * @param[in] link channel link either in radix or link format
 	 * @param[out] chanId optional storage for parsed channel id
 	 * @param[out] errMsg optional storage for error message, meaningful only in
@@ -495,9 +480,114 @@ public:
 	        RsGxsGroupId& chanId = RS_DEFAULT_STORAGE_PARAM(RsGxsGroupId),
 	        std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
 
+	/**
+	 * @brief Search the turtle reachable network for matching channels
+	 * @jsonapi{development}
+	 * An @see RsGxsChannelSearchResultEvent is emitted when matching channels
+	 * arrives from the network
+	 * @param[in] matchString string to search into the channels
+	 * @return search id
+	 */
+	virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0;
 
-	/* Following functions are deprecated as they expose internal functioning
-	 * semantic, instead of a safe to use API */
+	/**
+	 * @brief Retrieve available search results
+	 * @jsonapi{development}
+	 * @param[in] searchId search id
+	 * @param[out] results storage for search results
+	 * @return false on error, true otherwise
+	 */
+	virtual bool retrieveDistantSearchResults(
+	        TurtleRequestId searchId,
+	        std::map<RsGxsGroupId, RsGxsGroupSearchResults>& results ) = 0;
+
+	/**
+	 * @brief Request distant channel details
+	 * @jsonapi{development}
+	 * An @see RsGxsChannelSearchResultEvent is emitted once details are
+	 * retrieved from the network
+	 * @param[in] groupId if of the group to request to the network
+	 * @return search id
+	 */
+	virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& groupId) = 0;
+
+	/**
+	 * @brief Retrieve previously requested distant group
+	 * @jsonapi{development}
+	 * @param[in] groupId if of teh group
+	 * @param[out] distantGroup storage for group data
+	 * @return false on error, true otherwise
+	 */
+	virtual bool getDistantSearchResultGroupData(
+	        const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
+
+	/**
+	 * @brief Clear accumulated search results
+	 * @jsonapi{development}
+	 * @param[in] reqId search id
+	 * @return false on error, true otherwise
+	 */
+	virtual bool clearDistantSearchResults(TurtleRequestId reqId) = 0;
+
+	~RsGxsChannels() override;
+
+	////////////////////////////////////////////////////////////////////////////
+	/* Following functions are deprecated and should not be considered a safe to
+	 * use API */
+
+	/**
+	 * @brief Get auto-download option value for given channel
+	 * @jsonapi{development}
+	 * @deprecated This feature rely on very buggy code, the returned value is
+	 *	not reliable @see setChannelAutoDownload().
+	 * @param[in] channelId channel id
+	 * @param[out] enabled storage for the auto-download option value
+	 * @return false if something failed, true otherwhise
+	 */
+	RS_DEPRECATED
+	virtual bool getChannelAutoDownload(
+	        const RsGxsGroupId& channelId, bool& enabled ) = 0;
+
+	/**
+	 * @brief Enable or disable auto-download for given channel. Blocking API
+	 * @jsonapi{development}
+	 * @deprecated This feature rely on very buggy code, when enabled the
+	 *	channel service start flooding erratically log with error messages,
+	 *	apparently without more dangerous consequences. Still those messages
+	 *	hints that something out of control is happening under the hood, use at
+	 *	your own risk. A safe alternative to this method can easly implemented
+	 *	at API client level instead.
+	 * @param[in] channelId channel id
+	 * @param[in] enable true to enable, false to disable
+	 * @return false if something failed, true otherwhise
+	 */
+	RS_DEPRECATED
+	virtual bool setChannelAutoDownload(
+	        const RsGxsGroupId& channelId, bool enable ) = 0;
+
+	/**
+	 * @brief Get download directory for the given channel
+	 * @jsonapi{development}
+	 * @deprecated @see setChannelAutoDownload()
+	 * @param[in] channelId id of the channel
+	 * @param[out] directory reference to string where to store the path
+	 * @return false on error, true otherwise
+	 */
+	RS_DEPRECATED
+	virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId,
+	                                          std::string& directory ) = 0;
+
+	/**
+	 * @brief Set download directory for the given channel. Blocking API.
+	 * @jsonapi{development}
+	 * @deprecated @see setChannelAutoDownload()
+	 * @param[in] channelId id of the channel
+	 * @param[in] directory path
+	 * @return false on error, true otherwise
+	 */
+	RS_DEPRECATED
+	virtual bool setChannelDownloadDirectory(
+	        const RsGxsGroupId& channelId, const std::string& directory) = 0;
 
 	/**
 	 * @brief Create channel. Blocking API.
@@ -512,10 +602,13 @@ public:
 	RS_DEPRECATED_FOR(getChannelsInfo)
 	virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups) = 0;
 
-	RS_DEPRECATED_FOR(getChannelsContent)
+	RS_DEPRECATED_FOR(getChannelContent)
+	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts, std::vector<RsGxsVote> &votes) = 0;
+
+	RS_DEPRECATED_FOR(getChannelContent)
 	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
 
-	RS_DEPRECATED_FOR(getChannelsContent)
+	RS_DEPRECATED_FOR(getChannelContent)
 	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
 
 	/**
@@ -624,22 +717,4 @@ public:
 	 */
 	RS_DEPRECATED_FOR(editChannel)
 	virtual bool updateGroup(uint32_t& token, RsGxsChannelGroup& group) = 0;
-
-	//////////////////////////////////////////////////////////////////////////////
-    ///                     Distant synchronisation methods                    ///
-    //////////////////////////////////////////////////////////////////////////////
-    ///
-	RS_DEPRECATED_FOR(turtleChannelRequest)
-	virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
-	RS_DEPRECATED
-	virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
-	RS_DEPRECATED_FOR(turtleSearchRequest)
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) =0;
-	RS_DEPRECATED
-	virtual bool clearDistantSearchResults(TurtleRequestId req)=0;
-	RS_DEPRECATED_FOR(turtleChannelRequest)
-	virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0;
-	//////////////////////////////////////////////////////////////////////////////
-
-	~RsGxsChannels() override;
 };

libretroshare/src/retroshare/rsgxscircles.h

@@ -45,25 +45,23 @@ extern RsGxsCircles* rsGxsCircles;
 
 enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
 {
-	UNKNOWN           = 0, /// Used to detect uninizialized values.
-	PUBLIC            = 1, /// Public distribution
-	EXTERNAL          = 2, /// Restricted to an external circle
+	UNKNOWN            = 0,	/// Used to detect uninizialized values.
+	PUBLIC             = 1,	/// Public distribution, based on GxsIds
+	EXTERNAL           = 2,	/// Restricted to an external circle, based on GxsIds
 
-	/** Restricted to a group of friend nodes, the administrator of the circle
-	 * behave as a hub for them */
-	NODES_GROUP        = 3,
+	NODES_GROUP        = 3,	/// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them
+    						/// Based on PGP nodes ids.
 
 	LOCAL              = 4, /// not distributed at all
 
 	/** Self-restricted. Used only at creation time of self-restricted circles
 	 *  when the circle id isn't known yet. Once the circle id is known the type
 	 *  is set to EXTERNAL, and the external circle id is set to the id of the
-	 *  circle itself.
+	 *  circle itself. Based on GxsIds.
 	 */
 	EXT_SELF           = 5,
 
-	/// distributed to nodes signed by your own PGP key only.
-	YOUR_EYES_ONLY     = 6
+	YOUR_EYES_ONLY     = 6 	/// distributed to nodes signed by your own PGP key only.
 };
 
 // TODO: convert to enum class
@@ -98,22 +96,32 @@ struct RsGxsCircleGroup : RsSerializable
 	~RsGxsCircleGroup() override;
 };
 
+enum class RsGxsCircleSubscriptionType:uint8_t {
+	UNKNOWN     = 0x00,
+	SUBSCRIBE   = 0x01,
+	UNSUBSCRIBE = 0x02
+};
+
 struct RsGxsCircleMsg : RsSerializable
 {
 	RsMsgMetaData mMeta;
 
+#ifdef TO_REMOVE
+    // This item is actually totally unused, so we can change it no problem
 #ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_UNNAMED
 	/* This is horrible and should be changed into yet to be defined something
 	 * reasonable in next non-retrocompatible version */
 	std::string stuff;
 #endif
+#endif
+    RsGxsCircleSubscriptionType mSubscriptionType;
 
 	/// @see RsSerializable
 	void serial_process( RsGenericSerializer::SerializeJob j,
 	                     RsGenericSerializer::SerializeContext& ctx) override
 	{
 		RS_SERIAL_PROCESS(mMeta);
-		RS_SERIAL_PROCESS(stuff);
+		RS_SERIAL_PROCESS(mSubscriptionType);
 	}
 
 	~RsGxsCircleMsg() override;
@@ -121,15 +129,29 @@ struct RsGxsCircleMsg : RsSerializable
 
 struct RsGxsCircleDetails : RsSerializable
 {
-	RsGxsCircleDetails() :
-	    mCircleType(static_cast<uint32_t>(RsGxsCircleType::EXTERNAL)),
-	    mAmIAllowed(false),mAmIAdmin(false) {}
+	RsGxsCircleDetails() : mCircleType(RsGxsCircleType::EXTERNAL), mAmIAllowed(false),mAmIAdmin(false) {}
 	~RsGxsCircleDetails() override;
 
+    // helper functions.
+    bool isIdInCircle(const RsGxsId& id) const { return mAllowedGxsIds.find(id) != mAllowedGxsIds.end(); }
+    bool isIdInInviteeList(const RsGxsId& id) const
+    {
+        auto it = mSubscriptionFlags.find(id);
+        return (it != mSubscriptionFlags.end()) && (it->second & GXS_EXTERNAL_CIRCLE_FLAGS_IN_ADMIN_LIST );
+    }
+    bool isIdRequestingMembership(const RsGxsId& id) const
+    {
+        auto it = mSubscriptionFlags.find(id);
+        return it != mSubscriptionFlags.end() && (it->second & GXS_EXTERNAL_CIRCLE_FLAGS_SUBSCRIBED );
+    }
+    bool isGxsIdBased() const { return mCircleType==RsGxsCircleType::PUBLIC || mCircleType==RsGxsCircleType::EXTERNAL || mCircleType==RsGxsCircleType::EXT_SELF; }
+
+    // Members
+
 	RsGxsCircleId mCircleId;
 	std::string mCircleName;
 
-	uint32_t mCircleType;
+	RsGxsCircleType mCircleType;
 	RsGxsCircleId mRestrictedCircleId;
 
 	/** true when one of load GXS ids belong to the circle allowed list (admin
@@ -165,28 +187,54 @@ struct RsGxsCircleDetails : RsSerializable
 
 enum class RsGxsCircleEventCode: uint8_t
 {
+    // Notifications be only have 4 different possibilities:
+    //
+    //           invitee list join/leave and
+    //           membership request / leave request
+    //
+    // From there, depending on what the client displays, it is possible to interpret these
+    // as "some user joined the circle", or "membership pending for that Id", etc, depending
+    // on whether the current node owns the circle, or the admin is or is not yours.
+    //
+    // These should be decided in the UI based on what the circle cache is displaying.
+    //
 	UNKNOWN                   = 0x00,
 
-	/** mCircleId contains the circle id and mGxsId is the id requesting
-	 * membership */
-	CIRCLE_MEMBERSHIP_REQUEST = 0x01,
+	/**
+	 * Sent when we receive a membership request msg for a particular circle.
+	 *
+	 * mCircleId contains the circle id and mGxsId is the id requesting membership */
+	CIRCLE_MEMBERSHIP_REQUEST                      = 0x01,
 
-	/** mCircleId is the circle that invites me, and mGxsId is my own Id that is
-	 * invited */
-	CIRCLE_MEMBERSHIP_INVITE  = 0x02,
+	/**
+	 * Sent when the ID has been added to the circle invitee list.
+	 *
+	 * mCircleId is the circle that invites me, and mGxsId is my own Id that is invited */
+	CIRCLE_MEMBERSHIP_ID_ADDED_TO_INVITEE_LIST     = 0x02,
 
-	/** mCircleId contains the circle id and mGxsId is the id dropping
-	 * membership */
-	CIRCLE_MEMBERSHIP_LEAVE   = 0x03,
+	/**
+	 * Sent when a GxsId annouces its will to not be in the circle.
+	 *
+	 * mCircleId contains the circle id and mGxsId is the id dropping membership */
+	CIRCLE_MEMBERSHIP_LEAVE                        = 0x03,
 
-	/// mCircleId contains the circle id and mGxsId is the id of the new member
-	CIRCLE_MEMBERSHIP_JOIN    = 0x04,
+	/**
+	 * Sent when the Id has been removed from the invitee list.
+	 *
+	 * mCircleId contains the circle id and mGxsId is the id that was revoqued * by admin */
+	CIRCLE_MEMBERSHIP_ID_REMOVED_FROM_INVITEE_LIST = 0x04,
 
-	/** mCircleId contains the circle id and mGxsId is the id that was revoqued * by admin */
-	CIRCLE_MEMBERSHIP_REVOQUED= 0x05,
+	/**
+	 * Means a new circle has been received.
+	 *
+	 * mCircleId contains the circle id */
+	NEW_CIRCLE                                     = 0x05,
 
-	/** mCircleId contains the circle id */
-	NEW_CIRCLE                = 0x06,
+	/**
+	 * Means that the circle cache has updated, and membership status that is displayed should probably be updated to.
+	 *
+	 * no additional information. Simply means that the info previously from the cache has changed. */
+	CACHE_DATA_UPDATED                             = 0x06,
 };
 
 struct RsGxsCircleEvent: RsEvent
@@ -312,7 +360,7 @@ public:
 	                              RsGxsCircleMsg& msg) =0;
 
 	/**
-	 * @brief Invite identities to circle
+	 * @brief Invite identities to circle (admin key is required)
 	 * @jsonapi{development}
 	 * @param[in] identities ids of the identities to invite
 	 * @param[in] circleId Id of the circle you own and want to invite ids in
@@ -321,6 +369,16 @@ public:
 	virtual bool inviteIdsToCircle( const std::set<RsGxsId>& identities,
 	                                const RsGxsCircleId& circleId ) = 0;
 
+	/**
+	 * @brief Remove identities from circle (admin key is required)
+	 * @jsonapi{development}
+	 * @param[in] identities ids of the identities to remove from the invite list
+	 * @param[in] circleId Id of the circle you own and want to invite ids in
+	 * @return false if something failed, true otherwhise
+	 */
+	virtual bool revokeIdsFromCircle( const std::set<RsGxsId>& identities,
+	                                const RsGxsCircleId& circleId ) = 0;
+
 	/**
 	 * @brief Request circle membership, or accept circle invitation
 	 * @jsonapi{development}

libretroshare/src/retroshare/rsgxscommon.h

@@ -43,6 +43,13 @@ struct RsGxsFile : RsSerializable
 		RS_SERIAL_PROCESS(mHash);
 		RS_SERIAL_PROCESS(mSize);
 	}
+
+	void clear()
+	{
+		mName.clear();
+		mHash.clear();
+		mSize = 0;
+	}
 };
 
 struct RsGxsImage  : RsSerializable
@@ -64,6 +71,7 @@ struct RsGxsImage  : RsSerializable
 	void copy(uint8_t *data, uint32_t size); // Allocates and Copies.
 	void clear(); 				// Frees.
 	void shallowClear(); 			// Clears Pointer.
+	bool empty() const;
 
 	uint32_t mSize;
 	uint8_t* mData;
@@ -156,7 +164,9 @@ struct RsGxsCommentService
 	virtual bool getRelatedComments( uint32_t token,
 	                                 std::vector<RsGxsComment> &comments ) = 0;
 
-	virtual bool createNewComment(uint32_t &token, RsGxsComment &comment) = 0;
+	virtual bool createNewComment(uint32_t &token, const RsGxsComment &comment) = 0;  // async API
+	virtual bool createComment(RsGxsComment& comment) = 0;				// blocking API. Updates comment with new metadata.
+
 	virtual bool createNewVote(uint32_t &token, RsGxsVote &vote) = 0;
 
 	virtual bool acknowledgeComment(

libretroshare/src/retroshare/rsgxsflags.h

@@ -107,7 +107,8 @@ namespace GXS_SERV {
     static const uint32_t GXS_MSG_STATUS_UNPROCESSED = 0x00000001;	// Flags to store the read/process status of group messages.
     static const uint32_t GXS_MSG_STATUS_GUI_UNREAD  = 0x00000002;	// The actual meaning may depend on the type of service.
     static const uint32_t GXS_MSG_STATUS_GUI_NEW     = 0x00000004;	//
-    static const uint32_t GXS_MSG_STATUS_KEEP        = 0x00000008;	//
+	/** Do not delete message even if older then group maximum storage time */
+	static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008;
     static const uint32_t GXS_MSG_STATUS_DELETE      = 0x00000020;	//
 
     /** END GXS Msg status flags **/

libretroshare/src/retroshare/rsgxsforums.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -25,6 +26,7 @@
 #include <cstdint>
 #include <string>
 #include <list>
+#include <system_error>
 
 #include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
@@ -54,11 +56,8 @@ static const uint32_t RS_GXS_FORUM_MSG_FLAGS_MODERATED = 0x00000001;
 #define IS_FORUM_MSG_MODERATION(flags) (flags & RS_GXS_FORUM_MSG_FLAGS_MODERATED)
 
 
-struct RsGxsForumGroup : RsSerializable
+struct RsGxsForumGroup : RsSerializable, RsGxsGenericGroupData
 {
-	/** Forum GXS metadata */
-	RsGroupMetaData mMeta;
-
 	/** @brief Forum desciption */
 	std::string mDescription;
 
@@ -114,6 +113,8 @@ enum class RsForumEventCode: uint8_t
 	UPDATED_MESSAGE          = 0x04, /// existing message has been updated in a particular forum
 	SUBSCRIBE_STATUS_CHANGED = 0x05, /// forum was subscribed or unsubscribed
 	READ_STATUS_CHANGED      = 0x06, /// msg was read or marked unread
+	STATISTICS_CHANGED       = 0x07, /// suppliers and how many messages they have changed
+	MODERATOR_LIST_CHANGED   = 0x08, /// forum moderation list has changed.
 };
 
 struct RsGxsForumEvent: RsEvent
@@ -125,6 +126,8 @@ struct RsGxsForumEvent: RsEvent
 	RsForumEventCode mForumEventCode;
 	RsGxsGroupId mForumGroupId;
 	RsGxsMessageId mForumMsgId;
+    std::list<RsGxsId> mModeratorsAdded;
+    std::list<RsGxsId> mModeratorsRemoved;
 
 	///* @see RsEvent @see RsSerializable
 	void serial_process(
@@ -135,6 +138,9 @@ struct RsGxsForumEvent: RsEvent
 		RS_SERIAL_PROCESS(mForumEventCode);
 		RS_SERIAL_PROCESS(mForumGroupId);
 		RS_SERIAL_PROCESS(mForumMsgId);
+		RS_SERIAL_PROCESS(mForumMsgId);
+		RS_SERIAL_PROCESS(mModeratorsAdded);
+		RS_SERIAL_PROCESS(mModeratorsRemoved);
 	}
 
 	~RsGxsForumEvent() override;
@@ -222,6 +228,24 @@ public:
 	 */
 	virtual bool getForumsSummaries(std::list<RsGroupMetaData>& forums) = 0;
 
+    /**
+     * @brief returns statistics for the forum service
+	 * @jsonapi{development}
+     * @param[out] stat     statistics struct
+     * @return              false if the call fails
+     */
+	virtual bool getForumServiceStatistics(GxsServiceStatistic& stat) =0;
+
+    /**
+     * @brief returns statistics about a particular forum
+	 * @jsonapi{development}
+     * @param[in]  forumId  Id of the forum
+     * @param[out] stat     statistics struct
+     * @return              false when the object doesn't exist or when the timeout is reached requesting the data
+     */
+	virtual bool getForumStatistics(const RsGxsGroupId& forumId,GxsGroupStatistic& stat)=0;
+
+
 	/**
 	 * @brief Get forums information (description, thumbnail...).
 	 * Blocking API.
@@ -245,7 +269,7 @@ public:
 	                                  std::vector<RsMsgMetaData>& msgMetas) = 0;
 
 	/**
-	 * @brief Get specific list of messages from a single forums. Blocking API
+	 * @brief Get specific list of messages from a single forum. Blocking API
 	 * @jsonapi{development}
 	 * @param[in] forumId id of the forum of which the content is requested
 	 * @param[in] msgsIds list of message ids to request
@@ -289,10 +313,10 @@ public:
 	static const std::string FORUM_URL_DATA_FIELD;
 
 	/** Link query field used to store forum message title
-	 * @see exportChannelLink */
+	 * @see exportForumLink */
 	static const std::string FORUM_URL_MSG_TITLE_FIELD;
 
-	/// Link query field used to store forum message id @see exportChannelLink
+	/// Link query field used to store forum message id @see exportForumLink
 	static const std::string FORUM_URL_MSG_ID_FIELD;
 
 	/**
@@ -331,6 +355,32 @@ public:
 	        RsGxsGroupId& forumId = RS_DEFAULT_STORAGE_PARAM(RsGxsGroupId),
 	        std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
 
+	/**
+	 * @brief Get posts related to the given post.
+	 * If the set is empty, nothing is returned.
+	 * @jsonapi{development}
+	 * @param[in] forumId id of the forum of which the content is requested
+	 * @param[in] parentId id of the post of which child posts (aka replies)
+	 *	are requested.
+	 * @param[out] childPosts storage for the child posts
+	 * @return Success or error details
+	 */
+	virtual std::error_condition getChildPosts(
+	        const RsGxsGroupId& forumId, const RsGxsMessageId& parentId,
+	        std::vector<RsGxsForumMsg>& childPosts ) = 0;
+
+	/**
+	 * @brief Set keep forever flag on a post so it is not deleted even if older
+	 * then group maximum storage time
+	 * @jsonapi{development}
+	 * @param[in] forumId id of the forum of which the post pertain
+	 * @param[in] postId id of the post on which to set the flag
+	 * @param[in] keepForever true to set the flag, false to unset it
+	 * @return Success or error details
+	 */
+	virtual std::error_condition setPostKeepForever(
+	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
+	        bool keepForever ) = 0;
 
 	/**
 	 * @brief Create forum. Blocking API.
@@ -364,5 +414,5 @@ public:
 	RS_DEPRECATED_FOR(createMessage)
 	virtual bool createMsg(uint32_t &token, RsGxsForumMsg &msg) = 0;
 	RS_DEPRECATED_FOR(editForum)
-	virtual bool updateGroup(uint32_t &token, RsGxsForumGroup &group) = 0;
+	virtual bool updateGroup(uint32_t &token, const RsGxsForumGroup &group) = 0;
 };

libretroshare/src/retroshare/rsgxsiface.h

@@ -72,6 +72,46 @@ struct RsGxsGroupSummary : RsSerializable
 	~RsGxsGroupSummary();
 };
 
+/*!
+ * This structure is used to locally store group search results for a given service.
+ * It contains the group information as well as a context
+ * strings to tell where the information was found. It is more compact than a
+ * GroupMeta object, so as to make search responses as light as possible.
+ */
+struct RsGxsGroupSearchResults : RsSerializable
+{
+	RsGxsGroupSearchResults()
+        : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
+    {}
+
+	RsGxsGroupId mGroupId;
+	std::string  mGroupName;
+	RsGxsId      mAuthorId;
+	rstime_t     mPublishTs;
+	uint32_t     mNumberOfMessages;
+	rstime_t     mLastMessageTs;
+	uint32_t     mSignFlags;
+	uint32_t     mPopularity;
+
+    std::set<std::string> mSearchContexts;
+
+	/// @see RsSerializable::serial_process
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mGroupId);
+		RS_SERIAL_PROCESS(mGroupName);
+		RS_SERIAL_PROCESS(mAuthorId);
+		RS_SERIAL_PROCESS(mPublishTs);
+		RS_SERIAL_PROCESS(mNumberOfMessages);
+		RS_SERIAL_PROCESS(mLastMessageTs);
+		RS_SERIAL_PROCESS(mSignFlags);
+		RS_SERIAL_PROCESS(mPopularity);
+		RS_SERIAL_PROCESS(mSearchContexts);
+	}
+
+	virtual ~RsGxsGroupSearchResults() = default;
+};
 
 /*!
  * Stores ids of changed gxs groups and messages.
@@ -110,12 +150,20 @@ struct RsGxsChanges : RsEvent
  */
 struct RsGxsIface
 {
+    /*!
+     * \brief serviceType
+     * \return  The 16-bits service type. See @serialiser/rsserviceids.h
+     */
+    virtual uint16_t serviceType() const =0;
+
+#ifdef TO_REMOVE
     /*!
      * Gxs services should call this for automatic handling of
      * changes, send
      * @param changes
      */
     virtual void receiveChanges(std::vector<RsGxsNotify*>& changes) = 0;
+#endif
 
     /*!
      * @return handle to token service for this GXS service

libretroshare/src/retroshare/rsgxsifacehelper.h

@@ -3,8 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011 by Christopher Evi-Parker                                    *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2011  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -26,9 +26,11 @@
 #include <thread>
 
 #include "retroshare/rsgxsiface.h"
+#include "retroshare/rsservicecontrol.h"
 #include "retroshare/rsreputations.h"
 #include "rsgxsflags.h"
 #include "util/rsdeprecate.h"
+#include "util/rsdebug.h"
 
 /*!
  * This class only make method of internal members visible tu upper level to
@@ -38,17 +40,40 @@
  * To properly fix the API design many changes with the implied chain reactions
  * are necessary, so at this point this workaround seems acceptable.
  */
-struct RsGxsIfaceHelper
+
+//==================================
+//  #define DEBUG_GXSIFACEHELPER 1
+//==================================
+
+enum class TokenRequestType: uint8_t
 {
+	__NONE              = 0x00, /// Used to detect uninitialized
+    GROUP_DATA          = 0x01,
+    GROUP_META          = 0x02,
+    GROUP_IDS           = 0x03,
+    POSTS               = 0x04,
+    ALL_POSTS           = 0x05,
+    MSG_RELATED_INFO    = 0x06,
+    GROUP_STATISTICS    = 0x07,
+    SERVICE_STATISTICS  = 0x08,
+    NO_KILL_TYPE        = 0x09,
+	__MAX                       /// Used to detect out of range
+};
+
+class RsGxsIfaceHelper
+{
+public:
 	/*!
 	 * @param gxs handle to RsGenExchange instance of service (Usually the
 	 *   service class itself)
 	 */
-	RsGxsIfaceHelper(RsGxsIface& gxs) :
-	    mGxs(gxs), mTokenService(*gxs.getTokenService()) {}
+	explicit RsGxsIfaceHelper(RsGxsIface& gxs) :
+	    mGxs(gxs), mTokenService(*gxs.getTokenService()), mMtx("GxsIfaceHelper")
+	{}
 
-    ~RsGxsIfaceHelper(){}
+	~RsGxsIfaceHelper() = default;
 
+#ifdef TO_REMOVE
     /*!
      * Gxs services should call this for automatic handling of
      * changes, send
@@ -58,6 +83,7 @@ struct RsGxsIfaceHelper
     {
 		mGxs.receiveChanges(changes);
     }
+#endif
 
     /* Generic Lists */
 
@@ -67,8 +93,7 @@ struct RsGxsIfaceHelper
      * @param groupIds the ids return for given request token
      * @return false if request token is invalid, check token status for error report
      */
-    bool getGroupList(const uint32_t &token,
-            std::list<RsGxsGroupId> &groupIds)
+    bool getGroupList(const uint32_t &token, std::list<RsGxsGroupId> &groupIds)
 	{
 		return mGxs.getGroupList(token, groupIds);
 	}
@@ -101,8 +126,7 @@ struct RsGxsIfaceHelper
      * @param groupInfo the ids returned for given request token
      * @return false if request token is invalid, check token status for error report
      */
-    bool getGroupSummary(const uint32_t &token,
-            std::list<RsGroupMetaData> &groupInfo)
+    bool getGroupSummary(const uint32_t &token, std::list<RsGroupMetaData> &groupInfo)
 	{
 		return mGxs.getGroupMeta(token, groupInfo);
 	}
@@ -112,8 +136,7 @@ struct RsGxsIfaceHelper
      * @param msgInfo the message metadata returned for given request token
      * @return false if request token is invalid, check token status for error report
      */
-    bool getMsgSummary(const uint32_t &token,
-            GxsMsgMetaMap &msgInfo)
+    bool getMsgSummary(const uint32_t &token, GxsMsgMetaMap &msgInfo)
 	{
 		return mGxs.getMsgMeta(token, msgInfo);
 	}
@@ -233,30 +256,116 @@ struct RsGxsIfaceHelper
     }
 
 	/// @see RsTokenService::requestGroupInfo
-	bool requestGroupInfo( uint32_t& token, const RsTokReqOptions& opts,
-	                       const std::list<RsGxsGroupId> &groupIds )
-	{ return mTokenService.requestGroupInfo(token, 0, opts, groupIds); }
+	bool requestGroupInfo( uint32_t& token, const RsTokReqOptions& opts, const std::list<RsGxsGroupId> &groupIds, bool high_priority_request = false )
+	{
+        TokenRequestType token_request_type;
+
+        switch(opts.mReqType)
+        {
+        case GXS_REQUEST_TYPE_GROUP_DATA: token_request_type = TokenRequestType::GROUP_DATA; break;
+        case GXS_REQUEST_TYPE_GROUP_META: token_request_type = TokenRequestType::GROUP_META; break;
+        case GXS_REQUEST_TYPE_GROUP_IDS: token_request_type = TokenRequestType::GROUP_IDS; break;
+        default:
+            RsErr() << __PRETTY_FUNCTION__ << "(EE) Unexpected request type " << opts.mReqType << "!!" << std::endl;
+            return false;
+        }
+
+		cancelActiveRequestTokens(token_request_type);
+
+		if( mTokenService.requestGroupInfo(token, 0, opts, groupIds))
+        {
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens[token]=high_priority_request? (TokenRequestType::NO_KILL_TYPE) : token_request_type;
+#ifdef DEBUG_GXSIFACEHELPER
+            locked_dumpTokens();
+#endif
+            return true;
+        }
+        else
+            return false;
+    }
 
 	/// @see RsTokenService::requestGroupInfo
-	bool requestGroupInfo(uint32_t& token, const RsTokReqOptions& opts)
-	{ return mTokenService.requestGroupInfo(token, 0, opts); }
+	bool requestGroupInfo(uint32_t& token, const RsTokReqOptions& opts, bool high_priority_request = false)
+	{
+        TokenRequestType token_request_type;
+
+        switch(opts.mReqType)
+        {
+        case GXS_REQUEST_TYPE_GROUP_DATA: token_request_type = TokenRequestType::GROUP_DATA; break;
+        case GXS_REQUEST_TYPE_GROUP_META: token_request_type = TokenRequestType::GROUP_META; break;
+        case GXS_REQUEST_TYPE_GROUP_IDS: token_request_type = TokenRequestType::GROUP_IDS; break;
+        default:
+            RsErr() << __PRETTY_FUNCTION__ << "(EE) Unexpected request type " << opts.mReqType << "!!" << std::endl;
+            return false;
+        }
+
+		cancelActiveRequestTokens(token_request_type);
+
+
+		if(  mTokenService.requestGroupInfo(token, 0, opts))
+        {
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens[token]=high_priority_request? (TokenRequestType::NO_KILL_TYPE) : token_request_type;
+#ifdef DEBUG_GXSIFACEHELPER
+            locked_dumpTokens();
+#endif
+            return true;
+        }
+        else
+            return false;
+    }
 
 	/// @see RsTokenService::requestMsgInfo
-	bool requestMsgInfo( uint32_t& token,
-	                     const RsTokReqOptions& opts, const GxsMsgReq& msgIds )
-	{ return mTokenService.requestMsgInfo(token, 0, opts, msgIds); }
+	bool requestMsgInfo( uint32_t& token, const RsTokReqOptions& opts, const GxsMsgReq& msgIds )
+	{
+        if(mTokenService.requestMsgInfo(token, 0, opts, msgIds))
+        {
+			RS_STACK_MUTEX(mMtx);
+
+			mActiveTokens[token]=  (msgIds.size()==1 && msgIds.begin()->second.size()==0) ?(TokenRequestType::ALL_POSTS):(TokenRequestType::POSTS);
+#ifdef DEBUG_GXSIFACEHELPER
+			locked_dumpTokens();
+#endif
+			return true;
+        }
+        else
+            return false;
+    }
 
 	/// @see RsTokenService::requestMsgInfo
-	bool requestMsgInfo(
-	        uint32_t& token, const RsTokReqOptions& opts,
-	        const std::list<RsGxsGroupId>& grpIds )
-	{ return mTokenService.requestMsgInfo(token, 0, opts, grpIds); }
+	bool requestMsgInfo( uint32_t& token, const RsTokReqOptions& opts, const std::list<RsGxsGroupId>& grpIds )
+    {
+        if(mTokenService.requestMsgInfo(token, 0, opts, grpIds))
+        {
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens[token]=TokenRequestType::ALL_POSTS;
+#ifdef DEBUG_GXSIFACEHELPER
+			locked_dumpTokens();
+#endif
+            return true;
+        }
+        else
+            return false;
+    }
 
 	/// @see RsTokenService::requestMsgRelatedInfo
 	bool requestMsgRelatedInfo(
 	        uint32_t& token, const RsTokReqOptions& opts,
 	        const std::vector<RsGxsGrpMsgIdPair>& msgIds )
-	{ return mTokenService.requestMsgRelatedInfo(token, 0, opts, msgIds); }
+	{
+        if( mTokenService.requestMsgRelatedInfo(token, 0, opts, msgIds))
+        {
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens[token]=TokenRequestType::MSG_RELATED_INFO;
+#ifdef DEBUG_GXSIFACEHELPER
+            locked_dumpTokens();
+#endif
+            return true;
+        }
+        else
+            return false;
+    }
 
 	/**
 	 * @jsonapi{development}
@@ -266,16 +375,62 @@ struct RsGxsIfaceHelper
 	{ return mTokenService.requestStatus(token); }
 
 	/// @see RsTokenService::requestServiceStatistic
-	void requestServiceStatistic(uint32_t& token)
-	{ mTokenService.requestServiceStatistic(token); }
+	bool requestServiceStatistic(uint32_t& token)
+	{
+        RsTokReqOptions opts;
+        opts.mReqType = GXS_REQUEST_TYPE_SERVICE_STATS;
+
+        mTokenService.requestServiceStatistic(token,opts);
+
+		RS_STACK_MUTEX(mMtx);
+		mActiveTokens[token]=TokenRequestType::SERVICE_STATISTICS;
+
+#ifdef DEBUG_GXSIFACEHELPER
+		locked_dumpTokens();
+#endif
+        return true;
+    }
 
 	/// @see RsTokenService::requestGroupStatistic
-	void requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId)
-	{ mTokenService.requestGroupStatistic(token, grpId); }
+	bool requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId)
+	{
+        RsTokReqOptions opts;
+        opts.mReqType = GXS_REQUEST_TYPE_GROUP_STATS;
+
+		mTokenService.requestGroupStatistic(token, grpId,opts);
+
+		RS_STACK_MUTEX(mMtx);
+		mActiveTokens[token]=TokenRequestType::GROUP_STATISTICS;
+#ifdef DEBUG_GXSIFACEHELPER
+		locked_dumpTokens();
+#endif
+        return true;
+    }
+
+    bool cancelActiveRequestTokens(TokenRequestType type)
+    {
+		RS_STACK_MUTEX(mMtx);
+        for(auto it = mActiveTokens.begin();it!=mActiveTokens.end();)
+            if(it->second == type)
+			{
+                mTokenService.cancelRequest(it->first);
+                it = mActiveTokens.erase(it);
+			}
+        	else
+                ++it;
+
+        return true;
+    }
 
 	/// @see RsTokenService::cancelRequest
 	bool cancelRequest(uint32_t token)
-	{ return mTokenService.cancelRequest(token); }
+    {
+		{
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens.erase(token);
+		}
+        return mTokenService.cancelRequest(token);
+    }
 
 	/**
 	 * @deprecated
@@ -291,14 +446,91 @@ protected:
 	 * @param[in] token token associated to the request caller is waiting for
 	 * @param[in] maxWait maximum waiting time in milliseconds
 	 * @param[in] checkEvery time in millisecond between status checks
+	 * @param[in] auto_delete_if_unsuccessful delete the request when it fails. This avoid leaving useless pending requests in the queue that would slow down additional calls.
 	 */
 	RsTokenService::GxsRequestStatus waitToken(
 	        uint32_t token,
-	        std::chrono::milliseconds maxWait = std::chrono::milliseconds(500),
-	        std::chrono::milliseconds checkEvery = std::chrono::milliseconds(2))
-	{ return mTokenService.waitToken(token, maxWait, checkEvery); }
+	        std::chrono::milliseconds maxWait = std::chrono::milliseconds(20000),
+	        std::chrono::milliseconds checkEvery = std::chrono::milliseconds(100),
+	        bool auto_delete_if_unsuccessful=true)
+	{
+#if defined(__ANDROID__) && (__ANDROID_API__ < 24)
+		auto wkStartime = std::chrono::steady_clock::now();
+		int maxWorkAroundCnt = 10;
+LLwaitTokenBeginLabel:
+#endif
+		auto timeout = std::chrono::steady_clock::now() + maxWait;
+		auto st = requestStatus(token);
+
+		while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE)
+		       && std::chrono::steady_clock::now() < timeout )
+		{
+			std::this_thread::sleep_for(checkEvery);
+			st = requestStatus(token);
+		}
+		if(st != RsTokenService::COMPLETE && auto_delete_if_unsuccessful)
+			cancelRequest(token);
+
+#if defined(__ANDROID__) && (__ANDROID_API__ < 24)
+		/* Work around for very slow/old android devices, we don't expect this
+		 * to be necessary on newer devices. If it take unreasonably long
+		 * something worser is already happening elsewere and we return anyway.
+		 */
+		if( st > RsTokenService::FAILED && st < RsTokenService::COMPLETE
+		        && maxWorkAroundCnt-- > 0 )
+		{
+			maxWait *= 10;
+			checkEvery *= 3;
+			Dbg3() << __PRETTY_FUNCTION__ << " Slow Android device "
+			       << " workaround st: " << st
+			       << " maxWorkAroundCnt: " << maxWorkAroundCnt
+			       << " maxWait: " << maxWait.count()
+			       << " checkEvery: " << checkEvery.count() << std::endl;
+			goto LLwaitTokenBeginLabel;
+		}
+		Dbg3() << __PRETTY_FUNCTION__ << " lasted: "
+		       << std::chrono::duration_cast<std::chrono::milliseconds>(
+		              std::chrono::steady_clock::now() - wkStartime ).count()
+		       << "ms" << std::endl;
+
+#endif
+
+		{
+			RS_STACK_MUTEX(mMtx);
+			mActiveTokens.erase(token);
+		}
+
+		return st;
+	}
 
 private:
 	RsGxsIface& mGxs;
 	RsTokenService& mTokenService;
+	RsMutex mMtx;
+
+	std::map<uint32_t,TokenRequestType> mActiveTokens;
+
+#ifdef DEBUG_GXSIFACEHELPER
+	void locked_dumpTokens()
+	{
+		const uint16_t service_id =  mGxs.serviceType();
+		const auto countSize = static_cast<size_t>(TokenRequestType::__MAX);
+		uint32_t count[countSize] = {0};
+
+		RsDbg rsdbg;
+		rsdbg << __PRETTY_FUNCTION__ << " Service 0x" << std::hex << service_id
+		      << " (" << rsServiceControl->getServiceName(
+		             RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(service_id) )
+		      << ") this=0x" << static_cast<void*>(this)
+		      << ") Active tokens (per type): ";
+
+		// let's count how many token of each type we've got.
+		for(auto& it: mActiveTokens) ++count[static_cast<size_t>(it.second)];
+
+		for(uint32_t i=0; i < countSize; ++i)
+			rsdbg /* << i << ":" */ << count[i] << " ";
+	}
+#endif // def DEBUG_GXSIFACEHELPER
+
+	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/retroshare/rsgxsifacetypes.h

@@ -34,8 +34,6 @@
 #include "serialiser/rstypeserializer.h"
 #include "util/rstime.h"
 
-typedef Sha1CheckSum RsGxsMessageId;
-
 typedef std::map<RsGxsGroupId, std::set<RsGxsMessageId> > GxsMsgIdResult;
 typedef std::pair<RsGxsGroupId, RsGxsMessageId> RsGxsGrpMsgIdPair;
 typedef std::map<RsGxsGrpMsgIdPair, std::set<RsGxsMessageId> > MsgRelatedIdResult;
@@ -45,6 +43,13 @@ struct RsMsgMetaData;
 
 typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > MsgMetaResult;
 
+enum class GxsRequestPriority {
+    VERY_HIGH      =  0x00,
+    HIGH           =  0x01,
+    NORMAL         =  0x02,
+    LOW            =  0x03,
+    VERY_LOW       =  0x04,
+};
 
 class RsGxsGrpMetaData;
 class RsGxsMsgMetaData;
@@ -118,8 +123,15 @@ struct RsGroupMetaData : RsSerializable
 	}
 };
 
+// This is the parent class of all interface-level GXS group data. Derived classes
+// will include service-specific information, such as icon, description, etc
 
+struct RsGxsGenericGroupData
+{
+    virtual ~RsGxsGenericGroupData() = default; // making the type polymorphic
 
+	RsGroupMetaData mMeta;
+};
 
 struct RsMsgMetaData : RsSerializable
 {
@@ -177,49 +189,52 @@ struct RsMsgMetaData : RsSerializable
 	}
 };
 
-class GxsGroupStatistic
+struct RsGxsGenericMsgData
 {
-public:
-	GxsGroupStatistic()
+	virtual ~RsGxsGenericMsgData() = default; // making the type polymorphic
+
+	RsMsgMetaData mMeta;
+};
+
+
+struct GxsGroupStatistic : RsSerializable
+{
+	GxsGroupStatistic() :
+	    mNumMsgs(0), mTotalSizeOfMsgs(0), mNumThreadMsgsNew(0),
+	    mNumThreadMsgsUnread(0), mNumChildMsgsNew(0), mNumChildMsgsUnread(0) {}
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
 	{
-		mNumMsgs = 0;
-		mTotalSizeOfMsgs = 0;
-		mNumThreadMsgsNew = 0;
-		mNumThreadMsgsUnread = 0;
-		mNumChildMsgsNew = 0;
-        mNumChildMsgsUnread = 0;
+		RS_SERIAL_PROCESS(mGrpId);
+		RS_SERIAL_PROCESS(mNumMsgs);
+		RS_SERIAL_PROCESS(mTotalSizeOfMsgs);
+		RS_SERIAL_PROCESS(mNumThreadMsgsNew);
+		RS_SERIAL_PROCESS(mNumThreadMsgsUnread);
+		RS_SERIAL_PROCESS(mNumChildMsgsNew);
+		RS_SERIAL_PROCESS(mNumChildMsgsUnread);
+
 	}
 
-public:
-	/// number of message
-    RsGxsGroupId mGrpId;
-
-    uint32_t mNumMsgs;			// from the database
+	RsGxsGroupId mGrpId;
+	uint32_t mNumMsgs; /// number of message, from the database
 	uint32_t mTotalSizeOfMsgs;
 	uint32_t mNumThreadMsgsNew;
 	uint32_t mNumThreadMsgsUnread;
 	uint32_t mNumChildMsgsNew;
-    uint32_t mNumChildMsgsUnread;
+	uint32_t mNumChildMsgsUnread;
+
+	~GxsGroupStatistic() override;
 };
 
-class GxsServiceStatistic
+struct GxsServiceStatistic : RsSerializable
 {
-public:
-	GxsServiceStatistic()
-	{
-		mNumMsgs = 0;
-		mNumGrps = 0;
-		mSizeOfMsgs = 0;
-		mSizeOfGrps = 0;
-		mNumGrpsSubscribed = 0;
-		mNumThreadMsgsNew = 0;
-		mNumThreadMsgsUnread = 0;
-		mNumChildMsgsNew = 0;
-		mNumChildMsgsUnread = 0;
-		mSizeStore = 0;
-	}
+	GxsServiceStatistic() :
+	    mNumMsgs(0), mNumGrps(0), mSizeOfMsgs(0),  mSizeOfGrps(0),
+	    mNumGrpsSubscribed(0), mNumThreadMsgsNew(0), mNumThreadMsgsUnread(0),
+	    mNumChildMsgsNew(0), mNumChildMsgsUnread(0), mSizeStore(0) {}
 
-public:
 	uint32_t mNumMsgs;
 	uint32_t mNumGrps;
 	uint32_t mSizeOfMsgs;
@@ -230,32 +245,35 @@ public:
 	uint32_t mNumChildMsgsNew;
 	uint32_t mNumChildMsgsUnread;
 	uint32_t mSizeStore;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mNumMsgs);
+		RS_SERIAL_PROCESS(mNumGrps);
+		RS_SERIAL_PROCESS(mSizeOfMsgs);
+		RS_SERIAL_PROCESS(mSizeOfGrps);
+		RS_SERIAL_PROCESS(mNumGrpsSubscribed);
+		RS_SERIAL_PROCESS(mNumThreadMsgsNew);
+		RS_SERIAL_PROCESS(mNumThreadMsgsUnread);
+		RS_SERIAL_PROCESS(mNumChildMsgsNew);
+		RS_SERIAL_PROCESS(mNumChildMsgsUnread);
+		RS_SERIAL_PROCESS(mSizeStore);
+	}
+
+	~GxsServiceStatistic() override;
 };
 
-class UpdateItem
-{
-public:
-    virtual ~UpdateItem() { }
-};
-
-class StringUpdateItem : public UpdateItem
-{
-public:
-    StringUpdateItem(const std::string update) : mUpdate(update) {}
-    const std::string& getUpdate() const { return mUpdate; }
-
-private:
-    std::string mUpdate;
-};
-
-class RsGxsGroupUpdateMeta
+class RS_DEPRECATED RsGxsGroupUpdateMeta
 {
 public:
 
     // expand as support is added for other utypes
     enum UpdateType { DESCRIPTION, NAME };
 
-    RsGxsGroupUpdateMeta(const RsGxsGroupId& groupId) : mGroupId(groupId) {}
+	explicit RsGxsGroupUpdateMeta(const RsGxsGroupId& groupId):
+	    mGroupId(groupId) {}
 
     typedef std::map<UpdateType, std::string> GxsMetaUpdate;
 

libretroshare/src/retroshare/rsgxsservice.h

@@ -32,68 +32,6 @@ typedef uint32_t TurtleRequestId;
 typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > GxsMsgMetaMap;
 typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMetaMap;
 
-/*!
- * The aim of this class is to abstract how changes are represented so they can
- * be determined outside the client API without explcitly enumerating all
- * possible changes at the interface
- */
-struct RsGxsNotify
-{
-	enum NotifyType
-	{
-        TYPE_UNKNOWN                           = 0x00,
-        TYPE_PUBLISHED                         = 0x01,
-        TYPE_RECEIVED_NEW                      = 0x02,
-        TYPE_PROCESSED                         = 0x03,
-        TYPE_RECEIVED_PUBLISHKEY               = 0x04,
-        TYPE_RECEIVED_DISTANT_SEARCH_RESULTS   = 0x05
-	};
-
-	virtual ~RsGxsNotify() {}
-	virtual NotifyType getType() = 0;
-};
-
-/*!
- * Relevant to group changes
- */
-class RsGxsGroupChange : public RsGxsNotify
-{
-public:
-	RsGxsGroupChange(NotifyType type, bool metaChange) : NOTIFY_TYPE(type), mMetaChange(metaChange) {}
-    std::list<RsGxsGroupId> mGrpIdList;
-    NotifyType getType(){ return NOTIFY_TYPE;}
-    bool metaChange() { return mMetaChange; }
-private:
-    const NotifyType NOTIFY_TYPE;
-    bool mMetaChange;
-};
-
-class RsGxsDistantSearchResultChange: public RsGxsNotify
-{
-public:
-    RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& group_id) : mRequestId(id),mGroupId(group_id){}
-
-    NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; }
-
-    TurtleRequestId mRequestId ;
- 	RsGxsGroupId mGroupId;
-};
-
-/*!
- * Relevant to message changes
- */
-class RsGxsMsgChange : public RsGxsNotify
-{
-public:
-	RsGxsMsgChange(NotifyType type, bool metaChange) : NOTIFY_TYPE(type), mMetaChange(metaChange) {}
-    std::map<RsGxsGroupId, std::set<RsGxsMessageId> > msgChangeMap;
-	NotifyType getType(){ return NOTIFY_TYPE;}
-    bool metaChange() { return mMetaChange; }
-private:
-    const NotifyType NOTIFY_TYPE;
-    bool mMetaChange;
-};
-
 
 
 #endif // RSGXSSERVICE_H

libretroshare/src/retroshare/rsgxstrans.h

@@ -70,10 +70,8 @@ enum class GxsTransSendStatus : uint8_t
 
 typedef uint64_t RsGxsTransId;
 
-class RsGxsTransGroup
+class RsGxsTransGroup: public RsGxsGenericGroupData
 {
-	public:
-	RsGroupMetaData mMeta;
 };
 
 class RsGxsTransMsg
@@ -103,7 +101,34 @@ struct RsGxsTransOutgoingRecord
 	RsGxsGroupId group_id ;
 };
 
+class RsGxsTransGroupStatistics: public GxsGroupStatistic
+{
+public:
+	RsGxsTransGroupStatistics()
+	{
+		last_publish_TS = 0;
+		popularity = 0;
+		subscribed = false;
+	}
+
+	void addMessageMeta(const RsGxsGroupId& grp,const RsMsgMetaData& meta)
+	{
+		messages_metas[meta.mMsgId] = meta ;
+		last_publish_TS = std::max(last_publish_TS,meta.mPublishTs) ;
+		mGrpId = grp ;
+	}
+
+	bool subscribed ;
+	int  popularity ;
+
+	rstime_t last_publish_TS;
+
+    std::map<RsGxsMessageId,RsMsgMetaData> messages_metas ;
+};
+
+
 /// RetroShare GxsTrans asyncronous redundant small mail trasport on top of GXS
+///
 class RsGxsTrans: public RsGxsIfaceHelper
 {
 public:
@@ -120,10 +145,8 @@ public:
 
 	virtual ~RsGxsTrans() {}
 
-	virtual bool getStatistics(GxsTransStatistics& stats)=0;
-
-//	virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsTransGroup> &groups) = 0;
-//	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsTransMsg> &posts) = 0;
+	virtual bool getDataStatistics(GxsTransStatistics& stats)=0;
+	virtual bool getGroupStatistics(std::map<RsGxsGroupId,RsGxsTransGroupStatistics>& stats) =0;
 };
 
 extern RsGxsTrans *rsGxsTrans ;

libretroshare/src/retroshare/rsidentity.h

@@ -37,7 +37,7 @@
 #include "serialiser/rstypeserializer.h"
 #include "util/rsdeprecate.h"
 
-struct RsIdentity;
+class RsIdentity;
 
 /**
  * Pointer to global instance of RsIdentity service implementation
@@ -234,31 +234,30 @@ struct RsIdentityUsage : RsSerializable
 		GXS_TUNNEL_DH_SIGNATURE_CHECK        = 0x0c,
 		GXS_TUNNEL_DH_SIGNATURE_CREATION     = 0x0d,
 
+		/// Identity received through GXS sync
+		IDENTITY_NEW_FROM_GXS_SYNC           = 0x0e,
 		/// Group update on that identity data. Can be avatar, name, etc.
-		IDENTITY_DATA_UPDATE                 = 0x0e,
+		IDENTITY_NEW_FROM_DISCOVERY          = 0x0f,
+		/// Explicit request to friend
+		IDENTITY_NEW_FROM_EXPLICIT_REQUEST   = 0x10,
 
 		/// Any signature verified for that identity
-		IDENTITY_GENERIC_SIGNATURE_CHECK     = 0x0f,
+		IDENTITY_GENERIC_SIGNATURE_CHECK     = 0x11,
 
 		/// Any signature made by that identity
-		IDENTITY_GENERIC_SIGNATURE_CREATION  = 0x10,
+		IDENTITY_GENERIC_SIGNATURE_CREATION  = 0x12,
 
-		IDENTITY_GENERIC_ENCRYPTION          = 0x11,
-		IDENTITY_GENERIC_DECRYPTION          = 0x12,
-		CIRCLE_MEMBERSHIP_CHECK              = 0x13
+		IDENTITY_GENERIC_ENCRYPTION          = 0x13,
+		IDENTITY_GENERIC_DECRYPTION          = 0x14,
+		CIRCLE_MEMBERSHIP_CHECK              = 0x15
 	} ;
 
-	RS_DEPRECATED
-	RsIdentityUsage( uint16_t service, const RsIdentityUsage::UsageCode& code,
-	                 const RsGxsGroupId& gid = RsGxsGroupId(),
-	                 const RsGxsMessageId& mid = RsGxsMessageId(),
-	                 uint64_t additional_id=0,
-	                 const std::string& comment = std::string() );
-
 	RsIdentityUsage( RsServiceType service,
 	                 RsIdentityUsage::UsageCode code,
 	                 const RsGxsGroupId& gid = RsGxsGroupId(),
-	                 const RsGxsMessageId& mid = RsGxsMessageId(),
+	                 const RsGxsMessageId& message_id = RsGxsMessageId(),
+                     const RsGxsMessageId& parent_id = RsGxsMessageId(),
+                     const RsGxsMessageId& thread_id = RsGxsMessageId(),
 	                 uint64_t additional_id=0,
 	                 const std::string& comment = std::string() );
 
@@ -275,6 +274,12 @@ struct RsIdentityUsage : RsSerializable
 	/// Message ID using the identity
 	RsGxsMessageId mMsgId;
 
+	/// Reference message ID. Useful for votes/comments
+	RsGxsMessageId mParentId;
+
+	/// Reference message ID. Useful for votes/comments
+	RsGxsMessageId mThreadId;
+
 	/// Some additional ID. Can be used for e.g. chat lobbies.
 	uint64_t mAdditionalId;
 
@@ -309,6 +314,7 @@ enum class RsGxsIdentityEventCode: uint8_t
     UNKNOWN                    = 0x00,
     NEW_IDENTITY               = 0x01,
     DELETED_IDENTITY           = 0x02,
+    UPDATED_IDENTITY           = 0x03,
 };
 
 struct RsGxsIdentityEvent: public RsEvent
@@ -382,8 +388,9 @@ struct RsIdentityDetails : RsSerializable
 
 
 /** The Main Interface Class for GXS people identities */
-struct RsIdentity : RsGxsIfaceHelper
+class RsIdentity: public RsGxsIfaceHelper
 {
+public:
 	explicit RsIdentity(RsGxsIface& gxs) : RsGxsIfaceHelper(gxs) {}
 
 	/**

libretroshare/src/retroshare/rsids.h

@@ -328,6 +328,7 @@ using Sha256CheckSum    = t_RsGenericIdType<_RsIdSize::SHA256         , false, R
 using RsPgpFingerprint  = t_RsGenericIdType<_RsIdSize::PGP_FINGERPRINT, true,  RsGenericIdType::PGP_FINGERPRINT>;
 using Bias20Bytes       = t_RsGenericIdType<_RsIdSize::SHA1           , true,  RsGenericIdType::BIAS_20_BYTES  >;
 using RsGxsGroupId      = t_RsGenericIdType<_RsIdSize::CERT_SIGN      , false, RsGenericIdType::GXS_GROUP      >;
+using RsGxsMessageId    = t_RsGenericIdType<_RsIdSize::SHA1           , false, RsGenericIdType::GXS_MSG        >;
 using RsGxsId           = t_RsGenericIdType<_RsIdSize::CERT_SIGN      , false, RsGenericIdType::GXS_ID         >;
 using RsGxsCircleId     = t_RsGenericIdType<_RsIdSize::CERT_SIGN      , false, RsGenericIdType::GXS_CIRCLE     >;
 using RsGxsTunnelId     = t_RsGenericIdType<_RsIdSize::SSL_ID         , false, RsGenericIdType::GXS_TUNNEL     >;

libretroshare/src/retroshare/rsinit.h

@@ -20,8 +20,7 @@
  *******************************************************************************/
 #pragma once
 
-/// RetroShare initialization and login API
-
+/// @file RetroShare initialization and login API
 
 // Initialize ok, result >= 0
 #define RS_INIT_OK              0 // Initialize ok
@@ -32,11 +31,15 @@
 #define RS_INIT_NO_KEYRING     -3 // Keyring is empty. Need to import it.
 #define RS_INIT_NO_EXECUTABLE  -4 // executable path hasn't been set in config options
 
-#include <stdint.h>
 #include <list>
 #include <map>
 #include <vector>
-#include <retroshare/rstypes.h>
+#include <cstdint>
+#include <system_error>
+
+#include "retroshare/rstypes.h"
+#include "retroshare/rsversion.h"
+
 
 class RsLoginHelper;
 
@@ -46,6 +49,71 @@ class RsLoginHelper;
  */
 extern RsLoginHelper* rsLoginHelper;
 
+
+enum class RsInitErrorNum : int32_t
+{
+	ALREADY_LOGGED_IN         = 6000,
+	CANT_ACQUIRE_LOCK         = 6001,
+	INVALID_LOCATION_NAME     = 6002,
+	PGP_NAME_OR_ID_NEEDED     = 6003,
+	PGP_KEY_CREATION_FAILED   = 6004,
+	SSL_KEY_CREATION_FAILED   = 6005,
+	INVALID_SSL_ID            = 6006,
+	LOGIN_FAILED              = 6007
+};
+
+struct RsInitErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare init"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsInitErrorNum>(ev))
+		{
+		case RsInitErrorNum::ALREADY_LOGGED_IN:
+			return "Already logged in";
+		case RsInitErrorNum::CANT_ACQUIRE_LOCK:
+			return "Cannot aquire lock on location data. Another instance is "
+			        "already running with this profile?";
+		case RsInitErrorNum::INVALID_LOCATION_NAME:
+			return "Invalid location name";
+		case RsInitErrorNum::PGP_NAME_OR_ID_NEEDED:
+			return "Either PGP name or PGP id is needed";
+		case RsInitErrorNum::PGP_KEY_CREATION_FAILED:
+			return "Failure creating PGP key";
+		case RsInitErrorNum::SSL_KEY_CREATION_FAILED:
+			return "Failure creating SSL key";
+		case RsInitErrorNum::INVALID_SSL_ID:
+			return "Invalid SSL id";
+		case RsInitErrorNum::LOGIN_FAILED:
+			return "Generic login failure";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	const static RsInitErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsJsonApiErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsInitErrorNum> : true_type {};
+}
+
+/** Provide RsInitErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsInitErrorNum */
+inline std::error_condition make_error_condition(RsInitErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsInitErrorCategory::instance );
+};
+
+
+
 /**
  * @brief The RsInitConfig struct
  * This class contains common configuration options, that executables using libretroshare may want to
@@ -85,7 +153,7 @@ struct RsConfigOptions
 class RsInit
 {
 public:
-	enum LoadCertificateStatus : uint8_t
+	enum RS_DEPRECATED_FOR(RsInitErrorNum) LoadCertificateStatus : uint8_t
 	{
 		OK,                     /// Everything go as expected, no error occurred
 		ERR_ALREADY_RUNNING,    /// Another istance is running already
@@ -313,10 +381,11 @@ extern RsAccounts* rsAccounts;
 class RsLoginHelper
 {
 public:
-    RsLoginHelper() {}
+	RsLoginHelper() = default;
+
 	/**
 	 * @brief Normal way to attempt login
-	 * @jsonapi{development,manualwrapper}
+	 * @jsonapi{development,unauthenticated}
 	 * @param[in] account Id of the account to which attempt login
 	 * @param[in] password Password for the given account
 	 * @return RsInit::OK if login attempt success, error code otherwhise
@@ -352,6 +421,44 @@ public:
 	void getLocations(std::vector<RsLoginHelper::Location>& locations);
 
 	/**
+	 * @brief Creates a new RetroShare location, and log in once is created
+	 * @jsonapi{development,manualwrapper}
+	 * @param[out] locationId storage for generated location SSL id
+	 * @param[inout] pgpId specify PGP id to use to sign the location, if a null
+	 *	id is passed the PGP key is created too and this param is used as
+	 *	storage for its id.
+	 * @param[in] password to protect and unlock the associated PGP key
+	 * param[in] apiUser (JSON API only) string containing username for JSON API
+	 *	so it can be later used to authenticate JSON API calls. It is passed
+	 *	down to @see RsJsonApi::authorizeUser under the hood.
+	 * param[in] apiPass (JSON API only) string containing password for JSON API
+	 *	so it can be later used to authenticate JSON API calls. It is passed
+	 *	down to @see RsJsonApi::authorizeUser under the hood.
+	 *	To improve security we strongly advise to not use the same as the
+	 *	password used for the PGP key.
+	 * @return Success or error information
+	 */
+	std::error_condition createLocationV2(
+	        RsPeerId& locationId,
+	        RsPgpId& pgpId,
+	        const std::string& locationName,
+	        const std::string& pgpName,
+	        const std::string& password
+	        /* JSON API only
+	         * const std::string& apiUser
+	         * const std::string& apiPass */ );
+
+	/**
+	 * @brief Check if RetroShare is already logged in, this usually return true
+	 *	after a successfull attemptLogin() and before closeSession()
+	 * @jsonapi{development,unauthenticated}
+	 * @return true if already logged in, false otherwise
+	 */
+	bool isLoggedIn();
+
+#if !RS_VERSION_AT_LEAST(0,6,6)
+	/**
+	 * @deprecated Use @see createLocationV2 instead
 	 * @brief Creates a new RetroShare location, and log in once is created
 	 * @jsonapi{development,manualwrapper}
 	 * @param[inout] location provide input information to generate the location
@@ -364,15 +471,9 @@ public:
 	 *	Tor hidden location. UNTESTED!
 	 * @return true if success, false otherwise
 	 */
+	RS_DEPRECATED_FOR(createLocationV2)
 	bool createLocation( RsLoginHelper::Location& location,
 	                     const std::string& password, std::string& errorMessage,
 	                     bool makeHidden = false, bool makeAutoTor = false );
-
-	/**
-	 * @brief Check if RetroShare is already logged in, this usually return true
-	 *	after a successfull attemptLogin() and before closeSession()
-	 * @jsonapi{development,unauthenticated}
-	 * @return true if already logged in, false otherwise
-	 */
-	bool isLoggedIn();
+#endif // !RS_VERSION_AT_LEAST(0,6,6)
 };

libretroshare/src/retroshare/rsjsonapi.h

@@ -1,7 +1,7 @@
 /*
  * RetroShare JSON API public header
  *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio.eigenlab.org>
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>
  * Copyright (C) 2019  Cyril Soler <csoler@users.sourceforge.net>
  * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
  *
@@ -29,6 +29,7 @@
 #include <cstdint>
 #include <system_error>
 
+#include "util/rsdebug.h"
 #include "util/rsmemory.h"
 
 class RsJsonApi;
@@ -74,8 +75,7 @@ struct RsJsonApiErrorCategory: std::error_category
 		case RsJsonApiErrorNum::NOT_A_MACHINE_GUN:
 			return "Method must not be called in burst";
 		default:
-			return "Error message for error: " + std::to_string(ev) +
-			        " not available in category: " + name();
+			return rsErrorNotInCategory(ev, name());
 		}
 	}
 

libretroshare/src/retroshare/rsmsgs.h

@@ -62,6 +62,7 @@
 #define RS_MSG_SIGNED                0x004000	/* message was signed and signature didn't check */
 #define RS_MSG_LOAD_EMBEDDED_IMAGES  0x008000   /* load embedded images */
 #define RS_MSG_PUBLISH_KEY           0x020000   /* publish key */
+#define RS_MSG_SPAM                  0x040000   /* Message is marked as spam */
 
 #define RS_MSG_SYSTEM                (RS_MSG_USER_REQUEST | RS_MSG_FRIEND_RECOMMENDATION | RS_MSG_PUBLISH_KEY)
 
@@ -356,7 +357,7 @@ struct RsMailStatusEvent : RsEvent
 #define RS_DISTANT_CHAT_CONTACT_PERMISSION_FLAG_FILTER_NON_CONTACTS   0x0001 
 #define RS_DISTANT_CHAT_CONTACT_PERMISSION_FLAG_FILTER_EVERYBODY      0x0002 
 
-struct DistantChatPeerInfo
+struct DistantChatPeerInfo : RsSerializable
 {
     DistantChatPeerInfo() : status(0),pending_items(0) {}
 
@@ -365,6 +366,16 @@ struct DistantChatPeerInfo
 	DistantChatPeerId peer_id ;	// this is the tunnel id actually
 	uint32_t status ;			// see the values in rsmsgs.h
     uint32_t pending_items;		// items not sent, waiting for a tunnel
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(to_id);
+		RS_SERIAL_PROCESS(own_id);
+		RS_SERIAL_PROCESS(peer_id);
+		RS_SERIAL_PROCESS(status);
+		RS_SERIAL_PROCESS(pending_items);
+	}
 };
 
 // Identifier for an chat endpoint like
@@ -426,9 +437,8 @@ public:
 	}
 };
 
-class ChatMessage
+struct ChatMessage : RsSerializable
 {
-public:
     ChatId chat_id; // id of chat endpoint
     RsPeerId broadcast_peer_id; // only used for broadcast chat: source peer id
     RsGxsId lobby_peer_gxs_id; // only used for lobbys: nickname of message author
@@ -441,6 +451,22 @@ public:
     bool incoming;
     bool online; // for outgoing messages: was this message send?
     //bool system_message;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(chat_id);
+		RS_SERIAL_PROCESS(broadcast_peer_id);
+		RS_SERIAL_PROCESS(lobby_peer_gxs_id);
+		RS_SERIAL_PROCESS(peer_alternate_nickname);
+
+		RS_SERIAL_PROCESS(chatflags);
+		RS_SERIAL_PROCESS(sendTime);
+		RS_SERIAL_PROCESS(recvTime);
+		RS_SERIAL_PROCESS(msg);
+		RS_SERIAL_PROCESS(incoming);
+		RS_SERIAL_PROCESS(online);
+	}
 };
 
 class ChatLobbyInvite : RsSerializable
@@ -678,6 +704,15 @@ public:
 	 */
 	virtual bool MessageStar(const std::string &msgId, bool mark) = 0;
 
+	/**
+	 * @brief MessageJunk
+	 * @jsonapi{development}
+	 * @param[in] msgId
+	 * @param[in] mark
+	 * @return true on success
+	 */
+	virtual bool MessageJunk(const std::string &msgId, bool mark) = 0;
+
 	/**
 	 * @brief MessageLoadEmbeddedImages
 	 * @jsonapi{development}
@@ -962,13 +997,38 @@ virtual void getOwnAvatarData(unsigned char *& data,int& size) = 0 ;
 
     virtual uint32_t getDistantChatPermissionFlags()=0 ;
     virtual bool setDistantChatPermissionFlags(uint32_t flags)=0 ;
-    
-virtual bool initiateDistantChatConnexion(
+	
+    	/**
+	 * @brief initiateDistantChatConnexion initiate a connexion for a distant chat
+	 * @jsonapi{development}
+	 * @param[in] to_pid RsGxsId to start the connection
+	 * @param[in] from_pid owned RsGxsId who start the connection
+	 * @param[out] pid distant chat id
+	 * @param[out] error_code if the connection can't be stablished
+	 * @param[in] notify notify remote that the connection is stablished
+	 * @return true on success. If you try to initate a connection already started it will return the pid of it. 
+	 */
+	virtual bool initiateDistantChatConnexion(
 	        const RsGxsId& to_pid, const RsGxsId& from_pid,
 	        DistantChatPeerId& pid, uint32_t& error_code,
 	        bool notify = true ) = 0;
-virtual bool getDistantChatStatus(const DistantChatPeerId& pid,DistantChatPeerInfo& info)=0;
-virtual bool closeDistantChatConnexion(const DistantChatPeerId& pid)=0;
+
+	/**
+	 * @brief getDistantChatStatus receives distant chat info to a given distant chat id
+	 * @jsonapi{development}
+	 * @param[in] pid distant chat id
+	 * @param[out] info distant chat info
+	 * @return true on success
+	 */
+	virtual bool getDistantChatStatus(const DistantChatPeerId& pid, DistantChatPeerInfo& info)=0;
+	
+	/**
+	 * @brief closeDistantChatConnexion 
+	 * @jsonapi{development}
+	 * @param[in] pid distant chat id to close the connection
+	 * @return true on success
+	 */
+	virtual bool closeDistantChatConnexion(const DistantChatPeerId& pid)=0;
 
 	/**
 	 * @brief MessageSend

libretroshare/src/retroshare/rsnotify.h

@@ -115,11 +115,13 @@ const uint32_t RS_FEED_ITEM_CHAT_NEW         = RS_FEED_TYPE_CHAT  | 0x0001;
 const uint32_t RS_FEED_ITEM_MESSAGE          = RS_FEED_TYPE_MSG   | 0x0001;
 const uint32_t RS_FEED_ITEM_FILES_NEW        = RS_FEED_TYPE_FILES | 0x0001;
 
-const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_REQ      = RS_FEED_TYPE_CIRCLE  | 0x0001;
-const uint32_t RS_FEED_ITEM_CIRCLE_INVIT_REC     = RS_FEED_TYPE_CIRCLE  | 0x0002;
-const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_LEAVE    = RS_FEED_TYPE_CIRCLE  | 0x0003;
-const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_JOIN     = RS_FEED_TYPE_CIRCLE  | 0x0004;
-const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_REVOQUED = RS_FEED_TYPE_CIRCLE  | 0x0005;
+const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_REQ        = RS_FEED_TYPE_CIRCLE  | 0x0001;
+const uint32_t RS_FEED_ITEM_CIRCLE_INVITE_REC      = RS_FEED_TYPE_CIRCLE  | 0x0002;
+const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_LEAVE      = RS_FEED_TYPE_CIRCLE  | 0x0003;
+const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_JOIN       = RS_FEED_TYPE_CIRCLE  | 0x0004;
+const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_ACCEPTED   = RS_FEED_TYPE_CIRCLE  | 0x0005;
+const uint32_t RS_FEED_ITEM_CIRCLE_MEMB_REVOKED    = RS_FEED_TYPE_CIRCLE  | 0x0006;
+const uint32_t RS_FEED_ITEM_CIRCLE_INVITE_CANCELLED= RS_FEED_TYPE_CIRCLE  | 0x0007;
 
 const uint32_t RS_MESSAGE_CONNECT_ATTEMPT    = 0x0001;
 
@@ -237,7 +239,6 @@ public:
 	virtual void notifyPeerStatusChangedSummary   () {}
 	virtual void notifyDiscInfoChanged            () {}
 
-	virtual bool askForDeferredSelfSignature      (const void *       /* data      */, const uint32_t     /* len   */, unsigned char * /* sign */, unsigned int * /* signlen */,int& signature_result , std::string /*reason = ""*/) { signature_result = false ;return true; }
 	virtual void notifyDownloadComplete           (const std::string& /* fileHash  */) {}
 	virtual void notifyDownloadCompleteCount      (uint32_t           /* count     */) {}
 	virtual void notifyHistoryChanged             (uint32_t           /* msgId     */, int /* type */) {}

libretroshare/src/retroshare/rspeers.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2004-2008 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com>          *
+ * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -468,7 +470,7 @@ struct RsGroupInfo : RsSerializable
 struct RsPeerStateChangedEvent : RsEvent
 {
 	/// @param[in] sslId is of the peer which changed state
-	RsPeerStateChangedEvent(RsPeerId sslId);
+	explicit RsPeerStateChangedEvent(RsPeerId sslId);
 
 	/// Storage fot the id of the peer that changed state
 	RsPeerId mSslId;
@@ -605,7 +607,15 @@ public:
 	virtual bool getAssociatedSSLIds(const RsPgpId& gpg_id, std::list<RsPeerId>& ids) = 0;
 	virtual bool gpgSignData(const void *data, const uint32_t len, unsigned char *sign, unsigned int *signlen, std::string reason = "") = 0;
 
-    virtual RsPgpId pgpIdFromFingerprint(const RsPgpFingerprint& fpr) = 0;
+	/**
+	 * @brief Convert PGP fingerprint to PGP id
+	 * @jsonapi{development}
+	 * Helper method useful while we port the whole RetroShare codebase from
+	 * RsPgpId to RsPgpFingerprint
+	 * @param[in] fpr PGP fingerprint to convert
+	 * @return PGP id corresponding to the fingerprint
+	 */
+	virtual RsPgpId pgpIdFromFingerprint(const RsPgpFingerprint& fpr) = 0;
 
     // Note: the two methods below could be unified. The fact that one of them can take an optional RsPeerDetails struct as parameter
     // seems quite inconsistent.
@@ -803,7 +813,7 @@ public:
 	        ServicePermissionFlags flags = RS_NODE_PERM_DEFAULT ) = 0;
 
 
-	/* Auth Stuff */
+	RS_DEPRECATED /// This function doesn't provide meaningful error reporting
 	virtual	std::string getPGPKey(const RsPgpId& pgp_id,bool include_signatures) = 0;
 	virtual bool GetPGPBase64StringAndCheckSum(const RsPgpId& gpg_id,std::string& gpg_base64_string,std::string& gpg_base64_checksum) = 0;
 

libretroshare/src/retroshare/rsphoto.h

@@ -3,7 +3,7 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2008-2012 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright 2008-2020 by Robert Fernie <retroshare@lunamutt.com>              *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -25,164 +25,72 @@
 #include <inttypes.h>
 #include <string>
 #include <list>
-#include "rsgxsservice.h"
+#include "retroshare/rsgxsservice.h"
+#include "retroshare/rsgxscommon.h"
+#include "retroshare/rsgxsifacehelper.h"
+
 
 /* The Main Interface Class - for information about your Peers */
 class RsPhoto;
 extern RsPhoto *rsPhoto;
 
-/******************* NEW STUFF FOR NEW CACHE SYSTEM *********/
-
-#define RSPHOTO_MODE_NEW	1
-#define RSPHOTO_MODE_OWN	2
-#define RSPHOTO_MODE_REMOTE	3
-
-class RsPhotoThumbnail
-{
-        public:
-                RsPhotoThumbnail()
-                :data(NULL), size(0), type("N/A") { return; }
-
-        bool deleteImage();
-        bool copyFrom(const RsPhotoThumbnail &nail);
-
-        // Holds Thumbnail image.
-        uint8_t *data;
-        uint32_t size;
-        std::string type;
-};
-
-/* If these flags are no set - the Photo inherits values from the Album
- */
-
-#define RSPHOTO_FLAGS_ATTRIB_TITLE		0x0001
-#define RSPHOTO_FLAGS_ATTRIB_CAPTION		0x0002
-#define RSPHOTO_FLAGS_ATTRIB_DESC		0x0004
-#define RSPHOTO_FLAGS_ATTRIB_PHOTOGRAPHER	0x0008
-#define RSPHOTO_FLAGS_ATTRIB_WHERE		0x0010
-#define RSPHOTO_FLAGS_ATTRIB_WHEN		0x0020
-#define RSPHOTO_FLAGS_ATTRIB_OTHER		0x0040
-#define RSPHOTO_FLAGS_ATTRIB_CATEGORY		0x0080
-#define RSPHOTO_FLAGS_ATTRIB_HASHTAGS		0x0100
-#define RSPHOTO_FLAGS_ATTRIB_ORDER		0x0200
-#define RSPHOTO_FLAGS_ATTRIB_THUMBNAIL		0x0400
-#define RSPHOTO_FLAGS_ATTRIB_MODE		0x0800
-#define RSPHOTO_FLAGS_ATTRIB_AUTHOR		0x1000 // PUSH UP ORDER
-#define RSPHOTO_FLAGS_ATTRIB_PHOTO		0x2000 // PUSH UP ORDER.
-
 class RsPhotoPhoto
-{
-        public:
-
-        RsMsgMetaData mMeta;
-
-        RsPhotoPhoto();
-
-        // THESE ARE IN THE META DATA.
-        //std::string mAlbumId;
-        //std::string mId;
-        //std::string mTitle; // only used by Album.
-        std::string mCaption;
-        std::string mDescription;
-        std::string mPhotographer;
-        std::string mWhere;
-        std::string mWhen;
-        std::string mOther;
-        std::string mCategory;
-
-        std::string mHashTags;
-
-        uint32_t mSetFlags;
-
-        int mOrder;
-
-        RsPhotoThumbnail mThumbnail;
-
-        int mMode;
-
-        // These are not saved.
-        std::string path; // if in Mode NEW.
-        uint32_t mModFlags;
-};
-
-class RsPhotoAlbumShare
-{
-        public:
-
-        uint32_t mShareType;
-        std::string mShareGroupId;
-        std::string mPublishKey;
-        uint32_t mCommentMode;
-        uint32_t mResizeMode;
-};
-
-class RsPhotoAlbum
-{
-        public:
-        RsPhotoAlbum();
-
-        RsGroupMetaData mMeta;
-
-        // THESE ARE IN THE META DATA.
-        //std::string mAlbumId;
-        //std::string mTitle; // only used by Album.
-
-        std::string mCaption;
-        std::string mDescription;
-        std::string mPhotographer;
-        std::string mWhere;
-        std::string mWhen;
-        std::string mOther;
-        std::string mCategory;
-
-        std::string mHashTags;
-
-        RsPhotoThumbnail mThumbnail;
-
-        int mMode;
-
-        std::string mPhotoPath;
-        RsPhotoAlbumShare mShareOptions;
-
-        // These aren't saved.
-        uint32_t mSetFlags;
-        uint32_t mModFlags;
-};
-
-class RsGxsPhotoCommentItem;
-class RsPhotoComment
 {
 public:
-    RsPhotoComment();
 
-    explicit RsPhotoComment(const RsGxsPhotoCommentItem& comment);
+	RsMsgMetaData mMeta;
 
-    RsPhotoComment& operator=(const RsGxsPhotoCommentItem& comment);
+	RsPhotoPhoto();
 
-    RsMsgMetaData mMeta;
+	// V2 PhotoMsg - keep it simple.
+	// mMeta.mTitle used for Photo Caption.
+	// mDescription optional field for addtional notes.
+	// mLowResImage - < 50k jpg of image.
+	// mPhotoFile   - transfer details for original photo.
+	std::string mDescription;
+	uint32_t    mOrder;
+	RsGxsImage mLowResImage;
+	RsGxsFile  mPhotoFile;
 
-    std::string mComment;
-    uint32_t mCommentFlag;
+	// These are not saved.
+	std::string mPath; // if New photo
+};
+
+#define RSPHOTO_SHAREMODE_LOWRESONLY    (1)
+#define RSPHOTO_SHAREMODE_ORIGINAL      (2)
+#define RSPHOTO_SHAREMODE_DUP_ORIGINAL  (3)
+#define RSPHOTO_SHAREMODE_DUP_200K      (4)
+#define RSPHOTO_SHAREMODE_DUP_1M        (5)
+
+struct RsPhotoAlbum: RsGxsGenericGroupData
+{
+	RsPhotoAlbum();
+
+	// V2 Album - keep it simple.
+	// mMeta.mTitle.
+	uint32_t mShareMode;
+
+	std::string mCaption;
+	std::string mDescription;
+	std::string mPhotographer;
+	std::string mWhere;
+	std::string mWhen;
+
+	RsGxsImage mThumbnail;
+
+	// Below is not saved.
+	bool mAutoDownload;
 };
 
 std::ostream &operator<<(std::ostream &out, const RsPhotoPhoto &photo);
 std::ostream &operator<<(std::ostream &out, const RsPhotoAlbum &album);
 
 typedef std::map<RsGxsGroupId, std::vector<RsPhotoPhoto> > PhotoResult;
-typedef std::map<RsGxsGroupId, std::vector<RsPhotoComment> > PhotoCommentResult;
-typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsPhotoComment> > PhotoRelatedCommentResult;
 
-class RsPhoto
+class RsPhoto: public RsGxsIfaceHelper, public RsGxsCommentService
 {
-
 public:
-
-    static const uint32_t FLAG_MSG_TYPE_PHOTO_POST;
-    static const uint32_t FLAG_MSG_TYPE_PHOTO_COMMENT;
-    static const uint32_t FLAG_MSG_TYPE_MASK;
-
-
-    RsPhoto()  { return; }
+    explicit RsPhoto(RsGxsIface &gxs) : RsGxsIfaceHelper(gxs)  { return; }
 
     virtual ~RsPhoto() { return; }
 
@@ -249,7 +157,7 @@ public:
     virtual bool getMsgSummary(const uint32_t &token,
                                                        MsgMetaResult &msgInfo) = 0;
 
-                    /* Specific Service Data */
+    /* Specific Service Data */
 
     /*!
      * @param token token to be redeemed for album request
@@ -266,22 +174,6 @@ public:
     virtual bool getPhoto(const uint32_t &token,
                                               PhotoResult &photo) = 0;
 
-    /* details are updated in album - to choose Album ID, and storage path */
-
-    /*!
-     * @param token token to be redeemed for photo request
-     * @param photo the photo returned for given request token
-     * @return false if request token is invalid, check token status for error report
-     */
-    virtual bool getPhotoComment(const uint32_t &token,
-                                              PhotoCommentResult& comments) = 0;
-    /*!
-     * @param token token to be redeemed for photo request
-     * @param photo the photo returned for given request token
-     * @return false if request token is invalid, check token status for error report
-     */
-    virtual bool getPhotoRelatedComment(const uint32_t &token, PhotoRelatedCommentResult &comments) = 0;
-
     /*!
      * submits album, which returns a token that needs
      * to be acknowledge to get album grp id
@@ -298,16 +190,6 @@ public:
      */
     virtual bool submitPhoto(uint32_t& token, RsPhotoPhoto &photo) = 0;
 
-    /*!
-     * submits photo comment, which returns a token that needs
-     * to be acknowledged to get photo msg-grp id pair
-     * The mParentId needs to be set to an existing msg for which
-     * commenting is enabled
-     * @param token token to redeem for acknowledgement
-     * @param comment comment to be submitted
-     */
-    virtual bool submitComment(uint32_t& token, RsPhotoComment &photo) = 0;
-
     /*!
      * subscribes to group, and returns token which can be used
      * to be acknowledged to get group Id
@@ -336,7 +218,29 @@ public:
      */
     virtual bool acknowledgeGrp(const uint32_t& token, RsGxsGroupId& grpId) = 0;
 
+    // Blocking versions.
+    /*!
+     * request to create a new album. Blocks until process completes.
+     * @param album album to be submitted
+     * @return true if created false otherwise
+     */
+    virtual bool createAlbum(RsPhotoAlbum &album) = 0;
 
+    /*!
+     * request to update an existing album. Blocks until process completes.
+     * @param album album to be submitted
+     * @return true if created false otherwise
+     */
+    virtual bool updateAlbum(const RsPhotoAlbum &album) = 0;
+
+    /*!
+     * retrieve albums based in groupIds.
+     * @param groupIds the ids to fetch.
+     * @param albums vector to be filled by request.
+     * @return true is successful, false otherwise.
+     */
+    virtual bool getAlbums(const std::list<RsGxsGroupId> &groupIds,
+			std::vector<RsPhotoAlbum> &albums) = 0;
 };
 
 

libretroshare/src/retroshare/rsposted.h

@@ -3,7 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2008-2012 by Robert Fernie, Christopher Evi-Parker                *
+ * Copyright (C) 2008-2012  Robert Fernie, Christopher Evi-Parker              *
+ * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +20,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RETROSHARE_GXS_RSPOSTED_GUI_INTERFACE_H
-#define RETROSHARE_GXS_RSPOSTED_GUI_INTERFACE_H
+#pragma once
 
 #include <inttypes.h>
 #include <string>
@@ -32,20 +32,60 @@
 #include "retroshare/rsgxscommon.h"
 #include "serialiser/rsserializable.h"
 
-/* The Main Interface Class - for information about your Posted */
 class RsPosted;
-extern RsPosted *rsPosted;
 
-class RsPostedPost;
-class RsPostedGroup
+/**
+ * Pointer to global instance of RsGxsChannels service implementation
+ * @jsonapi{development}
+ */
+extern RsPosted* rsPosted;
+
+struct RsPostedGroup: RsGxsGenericGroupData
 {
-	public:
-	RsPostedGroup() { return; }
-
-	RsGroupMetaData mMeta;
 	std::string mDescription;
 	RsGxsImage mGroupImage;
+};
 
+struct RsPostedPost: public RsGxsGenericMsgData
+{
+	RsPostedPost(): mHaveVoted(false), mUpVotes(0), mDownVotes(0), mComments(0),
+	    mHotScore(0), mTopScore(0), mNewScore(0) {}
+
+	bool calculateScores(rstime_t ref_time);
+
+	std::string mLink;
+	std::string mNotes;
+
+	bool     mHaveVoted;
+
+	// Calculated.
+	uint32_t mUpVotes;
+	uint32_t mDownVotes;
+	uint32_t mComments;
+
+
+	// and Calculated Scores:???
+	double  mHotScore;
+	double  mTopScore;
+	double  mNewScore;
+
+	RsGxsImage mImage;
+
+	/// @see RsSerializable
+	/*virtual void serial_process( RsGenericSerializer::SerializeJob j,
+								 RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mImage);
+		RS_SERIAL_PROCESS(mMeta);
+		RS_SERIAL_PROCESS(mLink);
+		RS_SERIAL_PROCESS(mHaveVoted);
+		RS_SERIAL_PROCESS(mUpVotes);
+		RS_SERIAL_PROCESS(mDownVotes);
+		RS_SERIAL_PROCESS(mComments);
+		RS_SERIAL_PROCESS(mHotScore);
+		RS_SERIAL_PROCESS(mTopScore);
+		RS_SERIAL_PROCESS(mNewScore);
+	}*/
 };
 
 
@@ -65,9 +105,6 @@ class RsPostedGroup
 #define RSPOSTED_VIEWMODE_COMMENTS	4
 
 
-std::ostream &operator<<(std::ostream &out, const RsPostedGroup &group);
-std::ostream &operator<<(std::ostream &out, const RsPostedPost &post);
-
 enum class RsPostedEventCode: uint8_t
 {
 	UNKNOWN                  = 0x00,
@@ -77,6 +114,7 @@ enum class RsPostedEventCode: uint8_t
 	UPDATED_POSTED_GROUP     = 0x04,
 	UPDATED_MESSAGE          = 0x05,
 	READ_STATUS_CHANGED      = 0x06,
+	STATISTICS_CHANGED       = 0x07,
 };
 
 
@@ -98,25 +136,63 @@ struct RsGxsPostedEvent: RsEvent
 		RS_SERIAL_PROCESS(mPostedGroupId);
 		RS_SERIAL_PROCESS(mPostedMsgId);
 	}
+
+	~RsGxsPostedEvent() override;
 };
 
 class RsPosted : public RsGxsIfaceHelper, public RsGxsCommentService
 {
-	    public:
-
-	enum RankType {TopRankType, HotRankType, NewRankType };
-
-	//static const uint32_t FLAG_MSGTYPE_POST;
-	//static const uint32_t FLAG_MSGTYPE_MASK;
-
+public:
 	explicit RsPosted(RsGxsIface& gxs) : RsGxsIfaceHelper(gxs) {}
-	virtual ~RsPosted() {}
 
-	    /* Specific Service Data */
-virtual bool getGroupData(const uint32_t &token, std::vector<RsPostedGroup> &groups) = 0;
-virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
-virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts) = 0;
-//Not currently used
+	virtual bool getBoardsInfo(
+	        const std::list<RsGxsGroupId>& boardsIds,
+	        std::vector<RsPostedGroup>& boardsInfo ) = 0;
+
+	virtual bool getBoardsSummaries(std::list<RsGroupMetaData>& groupInfo) =0;
+
+	virtual bool getBoardAllContent(
+	        const RsGxsGroupId& boardId,
+	        std::vector<RsPostedPost>& posts,
+	        std::vector<RsGxsComment>& comments,
+	        std::vector<RsGxsVote>& votes ) = 0;
+
+	virtual bool getBoardContent(
+	        const RsGxsGroupId& boardId,
+	        const std::set<RsGxsMessageId>& contentsIds,
+	        std::vector<RsPostedPost>& posts,
+	        std::vector<RsGxsComment>& comments,
+	        std::vector<RsGxsVote>& votes ) = 0;
+
+	virtual bool editBoard(RsPostedGroup& board) =0;
+
+	virtual bool createBoard(RsPostedGroup& board) =0;
+
+	virtual bool getBoardStatistics(const RsGxsGroupId& boardId,GxsGroupStatistic& stat) =0;
+
+	virtual bool getBoardsServiceStatistics(GxsServiceStatistic& stat) =0;
+
+	enum RS_DEPRECATED RankType {TopRankType, HotRankType, NewRankType };
+
+	RS_DEPRECATED_FOR(getBoardsInfo)
+	virtual bool getGroupData( const uint32_t& token,
+	                           std::vector<RsPostedGroup> &groups ) = 0;
+
+	RS_DEPRECATED_FOR(getBoardsContent)
+	virtual bool getPostData(
+	        const uint32_t& token, std::vector<RsPostedPost>& posts,
+	        std::vector<RsGxsComment>& cmts, std::vector<RsGxsVote>& vots) = 0;
+
+	RS_DEPRECATED_FOR(getBoardsContent)
+	virtual bool getPostData(
+	        const uint32_t& token, std::vector<RsPostedPost>& posts,
+	        std::vector<RsGxsComment>& cmts) = 0;
+
+	RS_DEPRECATED_FOR(getBoardsContent)
+	virtual bool getPostData(
+	        const uint32_t& token, std::vector<RsPostedPost>& posts) = 0;
+
+	//Not currently used
 //virtual bool getRelatedPosts(const uint32_t &token, std::vector<RsPostedPost> &posts) = 0;
 
 	    /* From RsGxsCommentService */
@@ -134,62 +210,6 @@ virtual bool createPost(uint32_t &token, RsPostedPost &post) = 0;
 virtual bool updateGroup(uint32_t &token, RsPostedGroup &group) = 0;
 
     virtual bool groupShareKeys(const RsGxsGroupId& group,const std::set<RsPeerId>& peers) = 0 ;
+
+	virtual ~RsPosted();
 };
-
-
-class RsPostedPost
-{
-	public:
-	RsPostedPost()
-	{
-		//mMeta.mMsgFlags = RsPosted::FLAG_MSGTYPE_POST;
-		mUpVotes = 0;
-		mDownVotes = 0;
-		mComments = 0;
-		mHaveVoted = false;
-
-        mHotScore = 0;
-        mTopScore = 0;
-        mNewScore = 0;
-	}
-
-	bool calculateScores(rstime_t ref_time);
-
-	RsMsgMetaData mMeta;
-	std::string mLink;
-	std::string mNotes;
-
-	bool     mHaveVoted;
-
-	// Calculated.
-	uint32_t mUpVotes;
-	uint32_t mDownVotes;
-	uint32_t mComments;
-
-
-	// and Calculated Scores:???
-	double  mHotScore;
-	double  mTopScore;
-	double  mNewScore;
-	
-	RsGxsImage mImage;
-
-	/// @see RsSerializable
-	/*virtual void serial_process( RsGenericSerializer::SerializeJob j,
-	                             RsGenericSerializer::SerializeContext& ctx )
-	{
-		RS_SERIAL_PROCESS(mImage);
-		RS_SERIAL_PROCESS(mMeta);
-		RS_SERIAL_PROCESS(mLink);
-		RS_SERIAL_PROCESS(mHaveVoted);
-		RS_SERIAL_PROCESS(mUpVotes);
-		RS_SERIAL_PROCESS(mDownVotes);
-		RS_SERIAL_PROCESS(mComments);
-		RS_SERIAL_PROCESS(mHotScore);
-		RS_SERIAL_PROCESS(mTopScore);
-		RS_SERIAL_PROCESS(mNewScore);
-	}*/
-};
-
-
-#endif // RETROSHARE_GXS_RSPOSTED_GUI_INTERFACE_H

libretroshare/src/retroshare/rstokenservice.h

@@ -80,7 +80,7 @@ struct RsTokReqOptions
 {
 	RsTokReqOptions() : mOptions(0), mStatusFilter(0), mStatusMask(0),
 	    mMsgFlagMask(0), mMsgFlagFilter(0), mReqType(0), mSubscribeFilter(0),
-	    mSubscribeMask(0), mBefore(0), mAfter(0) {}
+	    mSubscribeMask(0), mBefore(0), mAfter(0),mPriority(GxsRequestPriority::NORMAL) {}
 
 	/**
 	 * Can be one or multiple RS_TOKREQOPT_*
@@ -107,6 +107,8 @@ struct RsTokReqOptions
 	// Time range... again applied after Options.
 	rstime_t   mBefore;
 	rstime_t   mAfter;
+
+    GxsRequestPriority mPriority;
 };
 
 /*!
@@ -181,6 +183,25 @@ public:
      */
     virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::vector<RsGxsGrpMsgIdPair>& msgIds) = 0;
 
+    /*!
+     * This request statistics on amount of data held
+     * number of groups
+     * number of groups subscribed
+     * number of messages
+     * size of db store
+     * total size of messages
+     * total size of groups
+     * @param token
+     */
+    virtual void requestServiceStatistic(uint32_t& token, const RsTokReqOptions &opts) = 0;
+
+	/*!
+	 * To request statistic on a group
+	 * @param token set to value to be redeemed to get statistic
+	 * @param grpId the id of the group
+	 */
+    virtual void requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId, const RsTokReqOptions &opts) = 0;
+
 
     /* Poll */
 
@@ -194,25 +215,6 @@ public:
      */
 	virtual GxsRequestStatus requestStatus(const uint32_t token) = 0;
 
-    /*!
-     * This request statistics on amount of data held
-     * number of groups
-     * number of groups subscribed
-     * number of messages
-     * size of db store
-     * total size of messages
-     * total size of groups
-     * @param token
-     */
-    virtual void requestServiceStatistic(uint32_t& token) = 0;
-
-	/*!
-	 * To request statistic on a group
-	 * @param token set to value to be redeemed to get statistic
-	 * @param grpId the id of the group
-	 */
-    virtual void requestGroupStatistic(uint32_t& token, const RsGxsGroupId& grpId) = 0;
-
 	/*!
 	 * @brief Cancel Request
 	 * If this function returns false, it may be that the request has completed
@@ -222,6 +224,7 @@ public:
 	 */
 	virtual bool cancelRequest(const uint32_t &token) = 0;
 
+#ifdef TO_REMOVE
 	/**
 	 * Block caller while request is being processed.
 	 * Useful for blocking API implementation.
@@ -231,8 +234,9 @@ public:
 	 */
 	RsTokenService::GxsRequestStatus waitToken(
 	        uint32_t token,
-	        std::chrono::milliseconds maxWait = std::chrono::milliseconds(500),
-	        std::chrono::milliseconds checkEvery = std::chrono::milliseconds(2))
+	        std::chrono::milliseconds maxWait = std::chrono::milliseconds(10000),
+	        std::chrono::milliseconds checkEvery = std::chrono::milliseconds(20),
+            bool auto_delete_if_unsuccessful=true)
 	{
 #if defined(__ANDROID__) && (__ANDROID_API__ < 24)
 		auto wkStartime = std::chrono::steady_clock::now();
@@ -241,12 +245,13 @@ LLwaitTokenBeginLabel:
 #endif
 		auto timeout = std::chrono::steady_clock::now() + maxWait;
 		auto st = requestStatus(token);
-		while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE)
-		       && std::chrono::steady_clock::now() < timeout )
+		while( !(st == RsTokenService::FAILED || st >= RsTokenService::COMPLETE) && std::chrono::steady_clock::now() < timeout )
 		{
 			std::this_thread::sleep_for(checkEvery);
 			st = requestStatus(token);
 		}
+        if(st != RsTokenService::COMPLETE && auto_delete_if_unsuccessful)
+            cancelRequest(token);
 
 #if defined(__ANDROID__) && (__ANDROID_API__ < 24)
 		/* Work around for very slow/old android devices, we don't expect this
@@ -274,6 +279,7 @@ LLwaitTokenBeginLabel:
 
 		return st;
 	}
+#endif
 
 	RS_SET_CONTEXT_DEBUG_LEVEL(2)
 };

libretroshare/src/retroshare/rstypes.h

@@ -38,7 +38,6 @@
 #define USE_NEW_CHUNK_CHECKING_CODE
 
 typedef Sha1CheckSum  RsFileHash ;
-typedef Sha1CheckSum  RsMessageId ;
 
 const uint32_t FT_STATE_FAILED			= 0x0000 ;
 const uint32_t FT_STATE_OKAY			= 0x0001 ;
@@ -201,7 +200,8 @@ struct FileInfo : RsSerializable
 	std::string path;
 	std::string fname;
 	RsFileHash hash;
-	std::string ext;
+
+	RS_DEPRECATED std::string ext; /// @deprecated unused
 
 	uint64_t size;
 	uint64_t avail; /// how much we have
@@ -303,10 +303,16 @@ struct DirDetails : RsSerializable
 	    type(DIR_TYPE_UNKNOWN), count(0), mtime(0), max_mtime(0) {}
 
 
+	/* G10h4ck do we still need to keep this as void* instead of uint64_t for
+	 * retroshare-gui sake? */
 	void* parent;
+
 	int prow; /* parent row */
 
+	/* G10h4ck do we still need to keep this as void* instead of uint64_t for
+	 * retroshare-gui sake? */
 	void* ref;
+
     uint8_t type;
     RsPeerId id;
     std::string name;
@@ -322,36 +328,18 @@ struct DirDetails : RsSerializable
 
 	/// @see RsSerializable
 	void serial_process(RsGenericSerializer::SerializeJob j,
-	                    RsGenericSerializer::SerializeContext& ctx)
+	                    RsGenericSerializer::SerializeContext& ctx) override
 	{
-#if defined(__GNUC__) && !defined(__clang__)
-#	pragma GCC diagnostic ignored "-Wstrict-aliasing"
-#endif // defined(__GNUC__) && !defined(__clang__)
+		/* Enforce serialization as uint64_t because void* changes size
+		 * depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
+		 */
+		uint64_t handle = reinterpret_cast<uint64_t>(ref);
+		RS_SERIAL_PROCESS(handle);
+		ref = reinterpret_cast<void*>(handle);
 
-        // (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
-        // preferable to force type serialization to the correct size rather than letting the compiler choose for us.
-        // /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
-
-		if(sizeof(ref) == 4)
-		{
-			std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
-			RS_SERIAL_PROCESS(handle);
-			std::uint32_t& parentHandle(reinterpret_cast<std::uint32_t&>(parent));
-			RS_SERIAL_PROCESS(parentHandle);
-		}
-		else if(sizeof(ref) == 8)
-		{
-			std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
-			RS_SERIAL_PROCESS(handle);
-			std::uint64_t& parentHandle(reinterpret_cast<std::uint64_t&>(parent));
-			RS_SERIAL_PROCESS(parentHandle);
-		}
-		else
-			std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#	pragma GCC diagnostic pop
-#endif // defined(__GNUC__) && !defined(__clang__)
+		uint64_t parentHandle = reinterpret_cast<uint64_t>(parent);
+		RS_SERIAL_PROCESS(parentHandle);
+		parent = reinterpret_cast<void*>(parentHandle);
 
 		RS_SERIAL_PROCESS(prow);
 		RS_SERIAL_PROCESS(type);
@@ -366,6 +354,8 @@ struct DirDetails : RsSerializable
 		RS_SERIAL_PROCESS(children);
 		RS_SERIAL_PROCESS(parent_groups);
 	}
+
+	~DirDetails() override = default;
 };
 
 class FileDetail

libretroshare/src/retroshare/rswiki.h

@@ -69,22 +69,17 @@ class CollectionRef
         std::string CollectionId;
 };
 
-
-class RsWikiCollection
+struct RsWikiCollection: RsGxsGenericGroupData
 {
 	public:
-
-	RsGroupMetaData mMeta;
-
 	std::string mDescription;
 	std::string mCategory;
 
 	std::string mHashTags;
 
-        //std::map<std::string, CollectionRef> linkReferences;
+	// std::map<std::string, CollectionRef> linkReferences;
 };
 
-
 class RsWikiSnapshot
 {
 	public:
@@ -129,6 +124,11 @@ virtual bool submitComment(uint32_t &token, RsWikiComment &comment) = 0;
 
 virtual bool updateCollection(uint32_t &token, RsWikiCollection &collection) = 0;
 
+	// Blocking Interfaces.
+virtual bool createCollection(RsWikiCollection &collection) = 0;
+virtual bool updateCollection(const RsWikiCollection &collection) = 0;
+virtual bool getCollections(const std::list<RsGxsGroupId> groupIds, std::vector<RsWikiCollection> &groups) = 0;
+
 };
 
 #endif

libretroshare/src/retroshare/rswire.h

@@ -3,7 +3,7 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright 2012-2020 by Robert Fernie <retroshare@lunamutt.com>              *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -28,28 +28,50 @@
 
 #include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
+#include "retroshare/rsgxscommon.h"
 
 
 /* The Main Interface Class - for information about your Peers */
 class RsWire;
 extern RsWire *rsWire;
 
-class RsWireGroup
+class RsWireGroup;
+typedef std::shared_ptr<RsWireGroup> RsWireGroupSPtr;
+typedef std::shared_ptr<const RsWireGroup> RsWireGroupConstSPtr;
+
+class RsWireGroup: public RsGxsGenericGroupData
 {
 	public:
+	RsWireGroup();
 
-	RsGroupMetaData mMeta;
-	std::string mDescription;
+	std::string mTagline;
+	std::string mLocation;
+
+	// Images max size should be enforced.
+	RsGxsImage  mHeadshot; // max size?
+	RsGxsImage  mMasthead; // max size?
+
+	// Unserialised stuff ---------------------
+
+	// These are this groups top-level msgs.
+	uint32_t mGroupPulses;
+	uint32_t mGroupRepublishes;
+	uint32_t mGroupLikes;
+	uint32_t mGroupReplies;
+	// how do we handle these. TODO
+	// uint32_t mGroupFollowing;
+	// uint32_t mGroupFollowers;
+
+	// These are this groups REF / RESPONSE msgs from others.
+	uint32_t mRefMentions; // TODO how to handle this?
+	uint32_t mRefRepublishes;
+	uint32_t mRefLikes;
+	uint32_t mRefReplies;
 };
 
 
-
 /***********************************************************************
- * So pulses operate in the following modes.
- *
- * => Standard, a post to your own group.
- * => @User, gets duplicated on each user's group.
- * => RT, duplicated as child of original post.
+ * RsWire - is intended to be a Twitter clone - but fully decentralised.
  *
  * From Twitter: 
  *  twitter can be: embedded, replied to, favourited, unfavourited, 
@@ -65,14 +87,54 @@ class RsWireGroup
  *
  ***********************************************************************/
 
-class RsWirePlace
-{
-	public:
 
-	
 
-};
+/************************************************************************
+ * Pulse comes in three flavours.
+ *
+ *
+ * Original Msg Pulse
+ * - Spontaneous msg, on your own group.
+ * - mPulseType = WIRE_PULSE_TYPE_ORIGINAL_MSG
+ * - Ref fields are empty.
+ *
+ * Reply to a Pulse (i.e Retweet), has two parts.
+ * as we want the retweet to reference the original, and the original to know about reply.
+ * This info will be duplicated in two msgs - but allow data to spread easier.
+ *
+ * Reply Msg Pulse, will be Top-Level Msg on Publisher's Group.
+ * - mPulseMode = WIRE_PULSE_TYPE_RESPONSE | WIRE_PULSE_TYPE_REPLY
+ * - Ref fields refer to Parent (InReplyTo) Msg.
+ *
+ * Reply Reference, is Child Msg of Parent Msg, on Parent Publisher's Group.
+ * - mPulseMode = WIRE_PULSE_TYPE_REFERENCE | WIRE_PULSE_TYPE_REPLY
+ * - Ref fields refer to Reply Msg.
+ * - NB: This Msg requires Parent Msg for complete info, while other two are self-contained.
+ *
+ * Additionally need to sort out additional relationships.
+ * - Mentions.
+ * - Followers.
+ * - Following.
+ ***********************************************************************/
 
+#define  WIRE_PULSE_TYPE_ORIGINAL            (0x0001)
+#define  WIRE_PULSE_TYPE_RESPONSE            (0x0002)
+#define  WIRE_PULSE_TYPE_REFERENCE           (0x0004)
+
+#define  WIRE_PULSE_RESPONSE_MASK            (0x0f00)
+#define  WIRE_PULSE_TYPE_REPLY               (0x0100)
+#define  WIRE_PULSE_TYPE_REPUBLISH           (0x0200)
+#define  WIRE_PULSE_TYPE_LIKE                (0x0400)
+
+#define  WIRE_PULSE_SENTIMENT_NO_SENTIMENT   (0x0000)
+#define  WIRE_PULSE_SENTIMENT_POSITIVE       (0x0001)
+#define  WIRE_PULSE_SENTIMENT_NEUTRAL        (0x0002)
+#define  WIRE_PULSE_SENTIMENT_NEGATIVE       (0x0003)
+
+class RsWirePulse;
+
+typedef std::shared_ptr<RsWirePulse> RsWirePulseSPtr;
+typedef std::shared_ptr<const RsWirePulse> RsWirePulseConstSPtr;
 
 class RsWirePulse
 {
@@ -80,19 +142,54 @@ class RsWirePulse
 
 	RsMsgMetaData mMeta;
 
-	std::string mPulseText; // all the text is stored here.
-	std::string mHashTags; 
+	// Store actual Pulse here.
+	std::string mPulseText;
 
-// These will be added at some point.
-//	std::string mInReplyPulse;
+	uint32_t mPulseType;
+	uint32_t mSentiment; // sentiment can be asserted at any point.
 
-//	uint32_t mPulseFlags;
+	// These Ref to the related (parent or reply) if reply (RESPONSE set)
+	// Mode                             RESPONSE          REFERENCE
+	RsGxsGroupId   mRefGroupId;    //   PARENT_GrpId      REPLY_GrpId
+	std::string    mRefGroupName;  //   PARENT_GrpName    REPLY_GrpName
+	RsGxsMessageId mRefOrigMsgId;  //   PARENT_OrigMsgId  REPLY_OrigMsgId
+	RsGxsId        mRefAuthorId;   //   PARENT_AuthorId   REPLY_AuthorId
+	rstime_t       mRefPublishTs;  //   PARENT_PublishTs  REPLY_PublishTs
+	std::string    mRefPulseText;  //   PARENT_PulseText  REPLY_PulseText
+	uint32_t       mRefImageCount; //   PARENT_#Images    REPLY_#Images
 
-//	std::list<std::string> mMentions;
-//	std::list<std::string> mHashTags;
-//	std::list<std::string> mUrls;
+	// Additional Fields for version 2.
+	// Images, need to enforce 20k limit?
+	RsGxsImage mImage1;
+	RsGxsImage mImage2;
+	RsGxsImage mImage3;
+	RsGxsImage mImage4;
 
-//	RsWirePlace mPlace;
+	// Below Here is not serialised.
+	// They are additional fields linking pulses together or parsing elements of msg.
+
+	// functions.
+	uint32_t ImageCount();
+
+	// can't have self referencial list, so need to use pointers.
+	// using SharedPointers to automatically cleanup.
+
+	// Pointer to WireGroups
+	// mRefGroupPtr is opportunistically filled in, but will often be empty.
+	RsWireGroupSPtr mRefGroupPtr; //  ORIG: N/A, RESP: Parent, REF: Reply Group
+	RsWireGroupSPtr mGroupPtr;    //  ORIG: Own, RESP: Own,    REF: Parent Group
+
+	// These are the direct children of this message
+	// split into likes, replies and retweets.
+	std::list<RsWirePulseSPtr> mReplies;
+	std::list<RsWirePulseSPtr> mLikes;
+	std::list<RsWirePulseSPtr> mRepublishes;
+
+	// parsed from msg.
+	// do we need references..?
+	std::list<std::string> mHashTags;
+	std::list<std::string> mMentions;
+	std::list<std::string> mUrls;
 };
 
 
@@ -107,6 +204,13 @@ class RsWire: public RsGxsIfaceHelper
 	explicit RsWire(RsGxsIface& gxs) : RsGxsIfaceHelper(gxs) {}
 	virtual ~RsWire() {}
 
+	/*!
+	 * To acquire a handle to token service handler
+	 * needed to make requests to the service
+	 * @return handle to token service for this gxs service
+	 */
+	virtual RsTokenService* getTokenService() = 0;
+
 	/* Specific Service Data */
 virtual bool getGroupData(const uint32_t &token, std::vector<RsWireGroup> &groups) = 0;
 virtual bool getPulseData(const uint32_t &token, std::vector<RsWirePulse> &pulses) = 0;
@@ -114,6 +218,33 @@ virtual bool getPulseData(const uint32_t &token, std::vector<RsWirePulse> &pulse
 virtual bool createGroup(uint32_t &token, RsWireGroup &group) = 0;
 virtual bool createPulse(uint32_t &token, RsWirePulse &pulse) = 0;
 
+	// Blocking Interfaces.
+virtual bool createGroup(RsWireGroup &group) = 0;
+virtual bool updateGroup(const RsWireGroup &group) = 0;
+virtual bool getGroups(const std::list<RsGxsGroupId> grpIds,
+				std::vector<RsWireGroup> &groups) = 0;
+
+	// New Blocking Interfaces.
+	// Plan to migrate all GUI calls to these, and remove old interfaces above.
+	// These are not single requests, but return data graphs for display.
+virtual bool createOriginalPulse(const RsGxsGroupId &grpId, RsWirePulseSPtr pPulse) = 0;
+virtual bool createReplyPulse(RsGxsGroupId grpId, RsGxsMessageId msgId,
+				RsGxsGroupId replyWith, uint32_t reply_type,
+				RsWirePulseSPtr pPulse) = 0;
+
+
+	// Provide Individual Group Details for display.
+virtual bool getWireGroup(const RsGxsGroupId &groupId, RsWireGroupSPtr &grp) = 0;
+virtual bool getWirePulse(const RsGxsGroupId &groupId, const RsGxsMessageId &msgId, RsWirePulseSPtr &pPulse) = 0;
+
+	// Provide list of pulses associated with groups.
+virtual bool getPulsesForGroups(const std::list<RsGxsGroupId> &groupIds,
+				std::list<RsWirePulseSPtr> &pulsePtrs) = 0;
+
+	// Provide pulse, and associated replies / like etc.
+virtual bool getPulseFocus(const RsGxsGroupId &groupId, const RsGxsMessageId &msgId,
+				int type, RsWirePulseSPtr &pPulse) = 0;
+
 };
 
 #endif

libretroshare/src/rs_upnp/upnphandler_miniupnp.cc

@@ -642,14 +642,10 @@ void    upnphandler::setExternalPort(unsigned short eport_in)
 	dataMtx.unlock(); /*** UNLOCK MUTEX ***/
 }
 
-	/* as determined by uPnP */
-bool    upnphandler::getInternalAddress(struct sockaddr_storage &addr)
+/* as determined by uPnP */
+bool upnphandler::getInternalAddress(struct sockaddr_storage &addr)
 {
-//	std::cerr << "UPnPHandler::getInternalAddress() pre Lock!" << std::endl;
-	dataMtx.lock();   /***  LOCK MUTEX  ***/
-//	std::cerr << "UPnPHandler::getInternalAddress() postLock!" << std::endl;
-
-	std::cerr << "UPnPHandler::getInternalAddress()" << std::endl;
+	RS_STACK_MUTEX(dataMtx);
 
 	// copy to universal addr.
 	sockaddr_storage_clear(addr);
@@ -657,18 +653,15 @@ bool    upnphandler::getInternalAddress(struct sockaddr_storage &addr)
 
 	bool valid = (upnpState >= RS_UPNP_S_ACTIVE);
 
-	dataMtx.unlock(); /*** UNLOCK MUTEX ***/
+	Dbg2() << __PRETTY_FUNCTION__ << " valid: " << valid
+	       << " addr: " << addr << std::endl;
 
 	return valid;
 }
 
-bool    upnphandler::getExternalAddress(struct sockaddr_storage &addr)
+bool upnphandler::getExternalAddress(sockaddr_storage &addr)
 {
-//	std::cerr << "UPnPHandler::getExternalAddress() pre Lock!" << std::endl;
-	dataMtx.lock();   /***  LOCK MUTEX  ***/
-//	std::cerr << "UPnPHandler::getExternalAddress() postLock!" << std::endl;
-
-	std::cerr << "UPnPHandler::getExternalAddress()" << std::endl;
+	RS_STACK_MUTEX(dataMtx);
 
 	// copy to universal addr.
 	sockaddr_storage_clear(addr);
@@ -676,7 +669,8 @@ bool    upnphandler::getExternalAddress(struct sockaddr_storage &addr)
 
 	bool valid = (upnpState == RS_UPNP_S_ACTIVE);
 
-	dataMtx.unlock(); /*** UNLOCK MUTEX ***/
+	Dbg2() << __PRETTY_FUNCTION__ << " valid: " << valid
+	       << " addr: " << addr << std::endl;
 
 	return valid;
 }

libretroshare/src/rs_upnp/upnphandler_miniupnp.h

@@ -28,7 +28,7 @@
 /* platform independent networking... */
 #include "pqi/pqinetwork.h"
 #include "pqi/pqiassist.h"
-
+#include "util/rsdebug.h"
 #include "util/rsthreads.h"
 
 class upnpentry
@@ -97,7 +97,7 @@ bool	shutdown_upnp();
 bool initUPnPState();
 bool printUPnPState();
 
-	private:
+private:
 
 bool background_setup_upnp(bool, bool);
 bool checkUPnPActive();
@@ -123,4 +123,5 @@ bool checkUPnPActive();
 	/* active port forwarding */
 	std::list<upnpforward> activeForwards;
 
+	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/rsitems/rsgxscircleitems.cc

@@ -35,7 +35,9 @@ RsItem *RsGxsCircleSerialiser::create_item(uint16_t service, uint8_t item_sub_id
     switch(item_sub_id)
     {
     case RS_PKT_SUBTYPE_GXSCIRCLE_GROUP_ITEM: return new RsGxsCircleGroupItem();
+#ifdef TO_REMOVE
     case RS_PKT_SUBTYPE_GXSCIRCLE_MSG_ITEM:   return new RsGxsCircleMsgItem();
+#endif
     case RS_PKT_SUBTYPE_GXSCIRCLE_SUBSCRIPTION_REQUEST_ITEM: return new RsGxsCircleSubscriptionRequestItem();
     default:
         return NULL ;
@@ -46,20 +48,27 @@ void RsGxsCircleSubscriptionRequestItem::clear()
 {
     time_stamp = 0 ;
     time_out   = 0 ;
-    subscription_type = SUBSCRIPTION_REQUEST_UNKNOWN;
+    subscription_type = RsGxsCircleSubscriptionType::UNKNOWN;
 }
 
+#ifdef TO_REMOVE
 void RsGxsCircleMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
 	//RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_MSG,mMsg.stuff,"mMsg.stuff") ;//Should be this but not retrocompatible...
 	RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_MSG,mMsg.stuff,"msg.stuff") ;
 }
 
+void RsGxsCircleMsgItem::clear()
+{
+	mMsg.stuff.clear();
+}
+#endif
+
 void RsGxsCircleSubscriptionRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,time_stamp,"time_stamp") ;
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,time_out  ,"time_out") ;
-    RsTypeSerializer::serial_process<uint8_t> (j,ctx,subscription_type  ,"subscription_type") ;
+    RsTypeSerializer::serial_process<RsGxsCircleSubscriptionType> (j,ctx,subscription_type  ,"subscription_type") ;
 }
 
 void RsGxsCircleGroupItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
@@ -69,11 +78,6 @@ void RsGxsCircleGroupItem::serial_process(RsGenericSerializer::SerializeJob j,Rs
     RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,subCircleSet,"subCircleSet") ;
 }
 
-void RsGxsCircleMsgItem::clear()
-{
-	mMsg.stuff.clear();
-}
-
 void RsGxsCircleGroupItem::clear()
 {
 	pgpIdSet.TlvClear();

libretroshare/src/rsitems/rsgxscircleitems.h

@@ -64,6 +64,7 @@ public:
 	RsTlvGxsCircleIdSet subCircleSet;
 };
 
+#ifdef TO_REMOVE
 class RsGxsCircleMsgItem : public RsGxsMsgItem
 {
 public:
@@ -76,6 +77,7 @@ public:
 
 	RsGxsCircleMsg mMsg;
 };
+#endif
 
 class RsGxsCircleSubscriptionRequestItem: public RsGxsMsgItem
 {
@@ -86,17 +88,11 @@ public:
     
     void clear();
     
-    enum {
-        SUBSCRIPTION_REQUEST_UNKNOWN     = 0x00, 
-        SUBSCRIPTION_REQUEST_SUBSCRIBE   = 0x01,
-        SUBSCRIPTION_REQUEST_UNSUBSCRIBE = 0x02
-    };
-    
 	virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 
     uint32_t           time_stamp ;
     uint32_t           time_out ;
-    uint8_t          subscription_type ;
+    RsGxsCircleSubscriptionType subscription_type ;
 };
 
 class RsGxsCircleSerialiser : public RsServiceSerializer

libretroshare/src/rsitems/rsgxsupdateitems.h

@@ -106,12 +106,22 @@ public:
     RsPeerId peerID;
 };
 
+struct RsPeerUpdateTsRecord
+{
+    RsPeerUpdateTsRecord() : mLastTsReceived(0), mTs(0) {}
+
+    rstime_t mLastTsReceived; // last TS that was sent for this group by this peer ID.
+    rstime_t mTs;             // time at which this TS was sent.
+};
+
 class RsGxsServerGrpUpdate
 {
 public:
     RsGxsServerGrpUpdate() { grpUpdateTS = 0 ; }
 
 	uint32_t grpUpdateTS;
+
+    std::map<RsPeerId,RsPeerUpdateTsRecord> grpUpdateTsRecords;
 };
 
 class RsGxsServerGrpUpdateItem : public RsGxsNetServiceItem, public RsGxsServerGrpUpdate
@@ -168,7 +178,13 @@ class RsGxsServerMsgUpdate
 public:
     RsGxsServerMsgUpdate() { msgUpdateTS = 0 ;}
 
-	uint32_t msgUpdateTS; // local time stamp this group last received a new msg
+	uint32_t msgUpdateTS; // local time stamp at which this group last received a new msg
+
+    // Now we also store for each peer the last own TS the peer sent and when it did so. This allows to detect when transactions are stuck because of
+    // outqueues clogging. If that happens, we receive multiple times the same TS from the friend, in which case we do not send the list of msgs
+    // again until a significant amount of time has passed. These values are obviously initialized to 0.
+
+    std::map<RsPeerId, RsPeerUpdateTsRecord> msgUpdateTsRecords;
 };
 
 class RsGxsServerMsgUpdateItem : public RsGxsNetServiceItem, public RsGxsServerMsgUpdate

libretroshare/src/rsitems/rsitem.h

@@ -45,11 +45,12 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
 	/// TODO: Do this make sense with the new serialization system?
 	virtual void clear() = 0;
 
+	/// @deprecated use << ostream operator instead
+	RS_DEPRECATED_FOR("<< ostream operator")
 	virtual std::ostream &print(std::ostream &out, uint16_t /* indent */ = 0)
 	{
 		RsGenericSerializer::SerializeContext ctx(
-		            NULL, 0, RsGenericSerializer::FORMAT_BINARY,
-		            RsGenericSerializer::SERIALIZATION_FLAG_NONE );
+		            nullptr, 0, RsSerializationFlags::NONE );
 		serial_process(RsGenericSerializer::PRINT,ctx);
 		return out;
 	}

libretroshare/src/rsitems/rsmsgitems.cc

@@ -83,19 +83,19 @@ void RsMsgTagType::serial_process(RsGenericSerializer::SerializeJob j,RsGenericS
 
 void RsMsgTags::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
-    RsTypeSerializer::serial_process<uint32_t>(j,ctx,msgId,"msgId") ;
+	RsTypeSerializer::serial_process<uint32_t>(j,ctx,msgId,"msgId") ;
 
 #warning this is not the correct way to serialise here. We should directly call serial_process<std::vector<uint32_t> >() but for backward compatibility, we cannot
 
-    if(j == RsGenericSerializer::DESERIALIZE)
-        while(ctx.mOffset < ctx.mSize)
-        {
-            uint32_t n ;
+	if(j == RsGenericSerializer::DESERIALIZE)
+		while(ctx.mOffset < ctx.mSize)
+		{
+			uint32_t n = 0;// No real need to initialize but otherwise the compiler complains.
 			RsTypeSerializer::serial_process<uint32_t>(j,ctx,n,"tagIds element") ;
 			tagIds.push_back(n) ;
-        }
-    else
-        for(std::list<uint32_t>::iterator it(tagIds.begin());it!=tagIds.end();++it)
+		}
+	else
+		for(std::list<uint32_t>::iterator it(tagIds.begin());it!=tagIds.end();++it)
 			RsTypeSerializer::serial_process<uint32_t>(j,ctx,*it,"tagIds element") ;
 }
 
@@ -147,8 +147,8 @@ void RsMsgItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSeri
 
     RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,attachment,"attachment");
 
-    if(ctx.mFlags & RsServiceSerializer::SERIALIZATION_FLAG_CONFIG)
-    	RsTypeSerializer::serial_process<uint32_t>(j,ctx,msgId,"msgId");
+	if(!!(ctx.mFlags & RsSerializationFlags::CONFIG))
+		RS_SERIAL_PROCESS(msgId);
 }
 
 void RsMsgTagType::clear()

libretroshare/src/rsitems/rsmsgitems.h

@@ -19,8 +19,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RS_MSG_ITEMS_H
-#define RS_MSG_ITEMS_H
+#pragma once
 
 #include <map>
 
@@ -33,10 +32,6 @@
 #include "serialiser/rstlvfileitem.h"
 #include "grouter/grouteritems.h"
 
-#if 0
-#include "serialiser/rstlvtypes.h"
-#include "serialiser/rstlvfileitem.h"
-#endif
 
 /**************************************************************************/
 
@@ -73,6 +68,7 @@ const uint32_t RS_MSG_FLAGS_LOAD_EMBEDDED_IMAGES  = 0x00040000;
 const uint32_t RS_MSG_FLAGS_DECRYPTED             = 0x00080000;
 const uint32_t RS_MSG_FLAGS_ROUTED                = 0x00100000;
 const uint32_t RS_MSG_FLAGS_PUBLISH_KEY           = 0x00200000;
+const uint32_t RS_MSG_FLAGS_SPAM                  = 0x00400000;
 
 const uint32_t RS_MSG_FLAGS_SYSTEM                = RS_MSG_FLAGS_USER_REQUEST | RS_MSG_FLAGS_FRIEND_RECOMMENDATION | RS_MSG_FLAGS_PUBLISH_KEY;
 
@@ -218,17 +214,12 @@ class RsMsgParentId : public RsMessageItem
 
 class RsMsgSerialiser: public RsServiceSerializer
 {
-	public:
-		RsMsgSerialiser(SerializationFlags flags = RsServiceSerializer::SERIALIZATION_FLAG_NONE)
-			:RsServiceSerializer(RS_SERVICE_TYPE_MSG,RsGenericSerializer::FORMAT_BINARY,flags){}
+public:
+	RsMsgSerialiser(
+	        RsSerializationFlags flags = RsSerializationFlags::NONE ):
+	    RsServiceSerializer(RS_SERVICE_TYPE_MSG, flags){}
 
-		virtual     ~RsMsgSerialiser() {}
+	RsItem* create_item(uint16_t service,uint8_t type) const override;
 
-        virtual RsItem *create_item(uint16_t service,uint8_t type) const ;
+	~RsMsgSerialiser() override = default;
 };
-
-/**************************************************************************/
-
-#endif /* RS_MSG_ITEMS_H */
-
-

libretroshare/src/rsitems/rsnxsitems.cc

@@ -173,9 +173,6 @@ void RsNxsMsg::clear()
     meta.TlvClear();
 }
 
-std::ostream&RsNxsMsg::print(std::ostream& out, uint16_t /*indent*/)
-{ return out; }
-
 void RsNxsGrp::clear()
 {
 	grpId.clear();

libretroshare/src/rsitems/rsnxsitems.h

@@ -378,7 +378,6 @@ struct RsNxsMsg : RsNxsItem
 	                             RsGenericSerializer::SerializeContext& ctx );
 
 	virtual void clear();
-	virtual std::ostream &print(std::ostream& out, uint16_t indent);
 
 	uint8_t pos; /// used for splitting up msg
 	uint8_t count; /// number of split up messages