merged with upstream/master

libretroshare/src/gxs/rsgenexchange.cc

@@ -1675,13 +1675,7 @@ void RsGenExchange::receiveNewMessages(std::vector<RsNxsMsg *>& messages)
 
 void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId)
 {
-	std::cerr << __PRETTY_FUNCTION__ << " received result for request "
+	std::cerr << __PRETTY_FUNCTION__ << " received result for request " << std::hex << id << std::dec << ": this method should be overloaded in the client service, but it is not. This is a bug!" << std::endl;
-	          << std::hex << id << std::dec << std::endl;
-
-	RS_STACK_MUTEX(mGenMtx);
-
-	RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId);
-	mNotifications.push_back(gc);
 }
 
 void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId)

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -272,9 +272,10 @@
     NXS_NET_DEBUG_6		group sync statistics (e.g. number of posts at nighbour nodes, etc)
  	NXS_NET_DEBUG_7		encryption/decryption of transactions
  	NXS_NET_DEBUG_8		gxs distant sync
+ 	NXS_NET_DEBUG_9		gxs distant search
 
  ***/
-//#define NXS_NET_DEBUG_0 	1
+#define NXS_NET_DEBUG_0 	1
 //#define NXS_NET_DEBUG_1 	1
 //#define NXS_NET_DEBUG_2 	1
 //#define NXS_NET_DEBUG_3 	1
@@ -283,6 +284,7 @@
 //#define NXS_NET_DEBUG_6 	1
 //#define NXS_NET_DEBUG_7 	1
 //#define NXS_NET_DEBUG_8 	1
+//#define NXS_NET_DEBUG_9 	1
 
 //#define NXS_FRAG
 
@@ -306,6 +308,7 @@ static const uint32_t GROUP_STATS_UPDATE_DELAY                =          240; //
 static const uint32_t GROUP_STATS_UPDATE_NB_PEERS             =            2; // number of peers to which the group stats are asked
 static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE            =       199000; // 200,000 bytes including signature and headers
 static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH          =           40; // dont search same group more than every 40 secs.
+static const uint32_t SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE    =         1800; // avoid re-sending the same msg list to a peer who asks twice for the same update in less than this time
 
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN             = 0x00 ;
 static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR            = 0x01 ;
@@ -318,7 +321,7 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING     = 0x05 ;
 
 #if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2)  || defined(NXS_NET_DEBUG_3) \
  || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6)  || defined(NXS_NET_DEBUG_7) \
- || defined(NXS_NET_DEBUG_8)
+ || defined(NXS_NET_DEBUG_8) || defined(NXS_NET_DEBUG_9)
 
 static const RsPeerId     peer_to_print     = RsPeerId();//std::string("a97fef0e2dc82ddb19200fb30f9ac575"))   ;
 static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("66052380f5d1d0c5992e2b55dc402ce6")) ;	// use this to allow to this group id only, or "" for all IDs
@@ -3993,7 +3996,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item)
 	GXSNETDEBUG_P_(item->PeerId()) << "  local modification time stamp: " << std::dec<< time(NULL) - mGrpServerUpdate.grpUpdateTS << " secs ago. Update sent: " <<
 	                                  ((item->updateTS < mGrpServerUpdate.grpUpdateTS)?"YES":"NO")  << std::endl;
 #endif
-	return item->updateTS < mGrpServerUpdate.grpUpdateTS;
+	return item->updateTS < mGrpServerUpdate.grpUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),RsGxsGroupId(),item->updateTS,mGrpServerUpdate.grpUpdateTsRecords[item->PeerId()]) ;
 }
 
 void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrpReqItem *item)
@@ -4238,10 +4241,33 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs
     return true;
 }
 
+bool RsGxsNetService::locked_checkResendingOfUpdates(const RsPeerId& pid,const RsGxsGroupId& grpId,rstime_t incoming_ts,RsPeerUpdateTsRecord& rec)
+{
+	rstime_t now = time(NULL);
+
+	// Now we check if the peer is sending the same outdated TS for the same time in a short while. This would mean the peer
+	// hasn't finished processing the updates we're sending and we shouldn't send new data anymore. Of course the peer might
+	// have disconnected or so, which means that we need to be careful about not sending. As a compromise we still send, but
+	// after waiting for a while (See
+
+	if(rec.mLastTsReceived == incoming_ts && rec.mTs + SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE > now)
+    {
+#ifdef NXS_NET_DEBUG_0
+		GXSNETDEBUG_PG(pid,grpId) << "(II) peer " << pid << " already sent the same TS " << (long int)now-(long int)rec.mTs << " secs ago for that group ID. Will not send msg list again for a while to prevent clogging..." << std::endl;
+#endif
+		return false;
+    }
+
+	rec.mLastTsReceived = incoming_ts;
+	rec.mTs = now;
+
+	return true;
+}
+
 bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& grp_is_known)
 {
     // Do we have new updates for this peer?
-    // Here we compare times in the same clock: the friend's clock, so it should be fine.
+    // Here we compare times in the same clock: our own clock, so it should be fine.
 
     grp_is_known = false ;
 
@@ -4250,7 +4276,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr
 	    // Item contains the hashed group ID in order to protect is from friends who don't know it. So we de-hash it using bruteforce over known group IDs for this peer.
 	    // We could save the de-hash result. But the cost is quite light, since the number of encrypted groups per service is usually low.
 
-	    for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
+	    for(ServerMsgMap::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
 		    if(item->grpId == hashGrpId(it->first,item->PeerId()))
 		    {
 			    item->grpId = it->first ;
@@ -4260,20 +4286,25 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr
 #endif
 			    grp_is_known = true ;
 
-			    return item->updateTS < it->second.msgUpdateTS ;
+                // The order of tests below is important because we want to only modify the map of requests records if the request actually is a valid requests instead of
+                // a simple check that nothing's changed.
+
+			    return item->updateTS < it->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,it->second.msgUpdateTsRecords[item->PeerId()]) ;
 		    }
 
 	    return false ;
     }
 
-    ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId);
+    ServerMsgMap::iterator cit = mServerMsgUpdateMap.find(item->grpId);
+
     if(cit != mServerMsgUpdateMap.end())
     {
 #ifdef NXS_NET_DEBUG_0
 	    GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "  local time stamp: " << std::dec<< time(NULL) - cit->second.msgUpdateTS << " secs ago. Update sent: " << (item->updateTS < cit->second.msgUpdateTS) << std::endl;
 #endif
 	    grp_is_known = true ;
-	    return item->updateTS < cit->second.msgUpdateTS ;
+
+		return item->updateTS < cit->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,cit->second.msgUpdateTsRecords[item->PeerId()]) ;
     }
 
 #ifdef NXS_NET_DEBUG_0
@@ -4294,6 +4325,9 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
     bool grp_is_known = false;
     bool was_circle_protected = item_was_encrypted || bool(item->flag & RsNxsSyncMsgReqItem::FLAG_USE_HASHED_GROUP_ID);
 
+    // This call determines if the peer can receive updates from us, meaning that our last TS is larger than what the peer sent.
+    // It also changes the items' group id into the un-hashed group ID if the group is a distant group.
+
     bool peer_can_receive_update = locked_CanReceiveUpdate(item, grp_is_known);
 
     if(item_was_encrypted)
@@ -4309,7 +4343,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_
 
     // We update suppliers in two cases:
     // Case 1: the grp is known because it is the hash of an existing group, but it's not yet in the server config map
-    // Case 2: the gtp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only
+    // Case 2: the grp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only
     // 		 gathered from known suppliers. So statistics never add new suppliers. These are only added here.
 
     if(grp_is_known || mServerGrpConfigMap.find(item->grpId)!=mServerGrpConfigMap.end())
@@ -5155,7 +5189,7 @@ static bool termSearch(const std::string& src, const std::string& substring)
 }
 #endif // ndef RS_DEEP_CHANNEL_INDEX
 
-bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
+bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
     RS_STACK_MUTEX(mNxsMutex) ;
 
@@ -5167,7 +5201,7 @@ bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<
     group_infos = it->second;
     return true ;
 }
-bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs)
+bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSearchResults& gs)
 {
 	RS_STACK_MUTEX(mNxsMutex) ;
     for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it)
@@ -5189,8 +5223,7 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
     return true ;
 }
 
-void RsGxsNetService::receiveTurtleSearchResults(
+void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
-        TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
 {
 	std::set<RsGxsGroupId> groupsToNotifyResults;
 
@@ -5198,20 +5231,43 @@ void RsGxsNetService::receiveTurtleSearchResults(
 		RS_STACK_MUTEX(mNxsMutex);
 
 		RsGxsGrpMetaTemporaryMap grpMeta;
-		std::map<RsGxsGroupId,RsGxsGroupSummary>&
+		std::map<RsGxsGroupId,RsGxsGroupSearchResults>& search_results_map(mDistantSearchResults[req]);
-		        search_results_map(mDistantSearchResults[req]);
+
+#ifdef NXS_NET_DEBUG_9
+        std::cerr << "Received group summary through turtle search for the following groups:" << std::endl;
+#endif
 
 		for(const RsGxsGroupSummary& gps : group_infos)
-			if(search_results_map.find(gps.mGroupId) == search_results_map.end())
+        {
-				grpMeta[gps.mGroupId] = nullptr;
+            std::cerr <<"  " << gps.mGroupId << "  \"" << gps.mGroupName << "\"" << std::endl;
+			grpMeta[gps.mGroupId] = nullptr;
+        }
+
 		mDataStore->retrieveGxsGrpMetaData(grpMeta);
 
+#ifdef NXS_NET_DEBUG_9
+        std::cerr << "Retrieved data store group data for the following groups:" <<std::endl;
+        for(auto& it:grpMeta)
+            std::cerr << "  " << it.first << " : " << it.second->mGroupName << std::endl;
+#endif
+
 		for (const RsGxsGroupSummary& gps : group_infos)
 		{
 #ifndef RS_DEEP_CHANNEL_INDEX
 			/* Only keep groups that are not locally known, and groups that are
-			 * not already in the mDistantSearchResults structure. */
+			 * not already in the mDistantSearchResults structure.
-			if(grpMeta[gps.mGroupId]) continue;
+			 * mDataStore may in some situations allocate an empty group meta data, so it's important
+			 * to test that the group meta is both non null and actually corresponds to the group id we seek. */
+
+            auto& meta(grpMeta[gps.mGroupId]);
+
+			if(meta != nullptr && meta->mGroupId == gps.mGroupId)
+                continue;
+
+#ifdef NXS_NET_DEBUG_9
+            std::cerr << "  group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
+#endif
+
 #else // ndef RS_DEEP_CHANNEL_INDEX
 			/* When deep search is enabled search results may bring more info
 			 * then we already have also about post that are indexed by xapian,
@@ -5220,22 +5276,32 @@ void RsGxsNetService::receiveTurtleSearchResults(
 			const RsGxsGroupId& grpId(gps.mGroupId);
 
 			groupsToNotifyResults.insert(grpId);
-			auto it2 = search_results_map.find(grpId);
+
-			if(it2 != search_results_map.end())
+            // Find search results place for this particular group
-			{
+
-				// update existing data
+#ifdef NXS_NET_DEBUG_9
-				RsGxsGroupSummary& eGpS(it2->second);
+            std::cerr << "  Adding gps=" << gps.mGroupId << " name=\"" << gps.mGroupName << "\" gps.mSearchContext=\"" << gps.mSearchContext << "\"" << std::endl;
-				eGpS.mPopularity++;
+#endif
-				eGpS.mNumberOfMessages = std::max(
+			RsGxsGroupSearchResults& eGpS(search_results_map[grpId]);
-				            eGpS.mNumberOfMessages,
+
-				            gps.mNumberOfMessages );
+            if(eGpS.mGroupId != grpId)	// not initialized yet. So we do it now.
-			}
+            {
-			else
+                eGpS.mGroupId   = gps.mGroupId;
-			{
+				eGpS.mGroupName = gps.mGroupName;
-				search_results_map[grpId] = gps;
+				eGpS.mAuthorId  = gps.mAuthorId;
-				// number of results so far
+				eGpS.mPublishTs = gps.mPublishTs;
-				search_results_map[grpId].mPopularity = 1;
+				eGpS.mSignFlags = gps.mSignFlags;
-			}
+            }
+            // We should check that the above values are always the same for all info that is received. In the end, we'll
+            // request the group meta and check the signature, but it may be misleading to receive a forged information
+            // that is not the real one.
+
+            ++eGpS.mPopularity;	// increase popularity. This is not a real counting, but therefore some heuristic estimate.
+            eGpS.mNumberOfMessages = std::max( eGpS.mNumberOfMessages, gps.mNumberOfMessages );
+			eGpS.mLastMessageTs    = std::max( eGpS.mLastMessageTs, gps.mLastMessageTs );
+
+            if(gps.mSearchContext != gps.mGroupName)			// this is a bit of a hack. We should have flags to tell where the search hit happens
+				eGpS.mSearchContexts.insert(gps.mSearchContext);
 		}
 	} // end RS_STACK_MUTEX(mNxsMutex);
 

libretroshare/src/gxs/rsgxsnetservice.h

@@ -140,9 +140,9 @@ public:
 	virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos);
 	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len);
 
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos);
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos);
 	virtual bool clearDistantSearchResults(const TurtleRequestId& id);
-    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&);
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&, RsGxsGroupSearchResults &);
 
     /*!
      * pauses synchronisation of subscribed groups and request for group id
@@ -439,6 +439,7 @@ private:
     bool locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item);
     bool locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item, bool &grp_is_known);
 	void locked_resetClientTS(const RsGxsGroupId& grpId);
+	bool locked_checkResendingOfUpdates(const RsPeerId& pid, const RsGxsGroupId &grpId, rstime_t incoming_ts, RsPeerUpdateTsRecord& rec);
 
     static RsGxsGroupId hashGrpId(const RsGxsGroupId& gid,const RsPeerId& pid) ;
     
@@ -609,7 +610,7 @@ private:
     std::set<RsGxsGroupId> mNewPublishKeysToNotify ;
 
     // Distant search result map
-    std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSummary> > mDistantSearchResults ;
+    std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSearchResults> > mDistantSearchResults ;
 
     void debugDump();
 

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -1090,8 +1090,10 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id
 	{
 		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
 
+#ifdef DEBUG_RSGXSNETTUNNEL
 		for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
 			std::cerr << "   group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
+#endif
 
 		auto it = mSearchableServices.find(result_gs->service) ;
 

libretroshare/src/gxs/rsgxsnotify.h

@@ -76,16 +76,6 @@ protected:
     bool mMetaChange;
 };
 
-class RsGxsDistantSearchResultChange: public RsGxsNotify
-{
-public:
-    RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& gid) : RsGxsNotify(gid), mRequestId(id){}
-
-    NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; }
-
-    TurtleRequestId mRequestId ;
-};
-
 /*!
  * Relevant to message changes
  */

libretroshare/src/gxs/rsnxs.h

@@ -128,7 +128,7 @@ public:
      * \return
      * 			false when the request is unknown.
      */
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos)=0;
+	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)=0;
     /*!
      * \brief getDistantSearchResults
      * \param id
@@ -136,7 +136,7 @@ public:
      * \return
      */
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
-    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0;
+    virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
 
     virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
 	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;

libretroshare/src/retroshare/rsgxschannels.h

@@ -117,14 +117,11 @@ enum class RsChannelEventCode: uint8_t
 
 struct RsGxsChannelEvent: RsEvent
 {
-	RsGxsChannelEvent():
+	RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
-	    RsEvent(RsEventType::GXS_CHANNELS),
-	    mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
 
 	RsChannelEventCode mChannelEventCode;
 	RsGxsGroupId mChannelGroupId;
 	RsGxsMessageId mChannelMsgId;
-	TurtleRequestId mDistantSearchRequestId;
 
 	///* @see RsEvent @see RsSerializable
 	void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
@@ -134,8 +131,28 @@ struct RsGxsChannelEvent: RsEvent
 		RS_SERIAL_PROCESS(mChannelEventCode);
 		RS_SERIAL_PROCESS(mChannelGroupId);
 		RS_SERIAL_PROCESS(mChannelMsgId);
-		RS_SERIAL_PROCESS(mDistantSearchRequestId);
+	}
-    }
+};
+
+// This event is used to factor multiple search results notifications in a single event.
+
+struct RsGxsChannelSearchResultEvent: RsEvent
+{
+	RsGxsChannelSearchResultEvent():
+	    RsEvent(RsEventType::GXS_CHANNELS),
+	    mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {}
+
+	RsChannelEventCode mChannelEventCode;
+	std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mChannelEventCode);
+		RS_SERIAL_PROCESS(mSearchResultsMap);
+	}
 };
 
 class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
@@ -407,48 +424,6 @@ public:
      */
     virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0;
 
-
-	/**
-	 * @brief Request remote channels search
-	 * @jsonapi{development}
-	 * @param[in] matchString string to look for in the search
-	 * @param multiCallback function that will be called each time a search
-	 * result is received
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool turtleSearchRequest(
-	        const std::string& matchString,
-	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
-	        rstime_t maxWait = 300 ) = 0;
-
-	/**
-	 * @brief Request remote channel
-	 * @jsonapi{development}
-	 * @param[in] channelId id of the channel to request to distants peers
-	 * @param multiCallback function that will be called each time a result is
-	 *	received
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool turtleChannelRequest(
-	        const RsGxsGroupId& channelId,
-	        const std::function<void (const RsGxsChannelGroup& result)>& multiCallback,
-	        rstime_t maxWait = 300 ) = 0;
-
-	/**
-	 * @brief Search local channels
-	 * @jsonapi{development}
-	 * @param[in] matchString string to look for in the search
-	 * @param multiCallback function that will be called for each result
-	 * @param[in] maxWait maximum wait time in seconds for search results
-	 * @return false on error, true otherwise
-	 */
-	virtual bool localSearchRequest(
-	        const std::string& matchString,
-	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
-	        rstime_t maxWait = 30 ) = 0;
-
 	/// default base URL used for channels links @see exportChannelLink
 	static const std::string DEFAULT_CHANNEL_BASE_URL;
 
@@ -493,6 +468,7 @@ public:
 
 	/**
 	 * @brief Import channel from full link
+	 * @jsonapi{development}
 	 * @param[in] link channel link either in radix or link format
 	 * @param[out] chanId optional storage for parsed channel id
 	 * @param[out] errMsg optional storage for error message, meaningful only in
@@ -504,7 +480,58 @@ public:
 	        RsGxsGroupId& chanId = RS_DEFAULT_STORAGE_PARAM(RsGxsGroupId),
 	        std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
 
+	/**
+	 * @brief Search the turtle reachable network for matching channels
+	 * @jsonapi{development}
+	 * An @see RsGxsChannelSearchResultEvent is emitted when matching channels
+	 * arrives from the network
+	 * @param[in] matchString string to search into the channels
+	 * @return search id
+	 */
+	virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0;
 
+	/**
+	 * @brief Retrieve available search results
+	 * @jsonapi{development}
+	 * @param[in] searchId search id
+	 * @param[out] results storage for search results
+	 * @return false on error, true otherwise
+	 */
+	virtual bool retrieveDistantSearchResults(
+	        TurtleRequestId searchId,
+	        std::map<RsGxsGroupId, RsGxsGroupSearchResults>& results ) = 0;
+
+	/**
+	 * @brief Request distant channel details
+	 * @jsonapi{development}
+	 * An @see RsGxsChannelSearchResultEvent is emitted once details are
+	 * retrieved from the network
+	 * @param[in] groupId if of the group to request to the network
+	 * @return search id
+	 */
+	virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& groupId) = 0;
+
+	/**
+	 * @brief Retrieve previously requested distant group
+	 * @jsonapi{development}
+	 * @param[in] groupId if of teh group
+	 * @param[out] distantGroup storage for group data
+	 * @return false on error, true otherwise
+	 */
+	virtual bool getDistantSearchResultGroupData(
+	        const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
+
+	/**
+	 * @brief Clear accumulated search results
+	 * @jsonapi{development}
+	 * @param[in] reqId search id
+	 * @return false on error, true otherwise
+	 */
+	virtual bool clearDistantSearchResults(TurtleRequestId reqId) = 0;
+
+	~RsGxsChannels() override;
+
+	////////////////////////////////////////////////////////////////////////////
 	/* Following functions are deprecated and should not be considered a safe to
 	 * use API */
 
@@ -690,22 +717,4 @@ public:
 	 */
 	RS_DEPRECATED_FOR(editChannel)
 	virtual bool updateGroup(uint32_t& token, RsGxsChannelGroup& group) = 0;
-
-	//////////////////////////////////////////////////////////////////////////////
-    ///                     Distant synchronisation methods                    ///
-    //////////////////////////////////////////////////////////////////////////////
-    ///
-	RS_DEPRECATED_FOR(turtleChannelRequest)
-	virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
-	RS_DEPRECATED
-	virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
-	RS_DEPRECATED_FOR(turtleSearchRequest)
-	virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) =0;
-	RS_DEPRECATED
-	virtual bool clearDistantSearchResults(TurtleRequestId req)=0;
-	RS_DEPRECATED_FOR(turtleChannelRequest)
-	virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0;
-	//////////////////////////////////////////////////////////////////////////////
-
-	~RsGxsChannels() override;
 };

libretroshare/src/retroshare/rsgxsiface.h

@@ -72,6 +72,46 @@ struct RsGxsGroupSummary : RsSerializable
 	~RsGxsGroupSummary();
 };
 
+/*!
+ * This structure is used to locally store group search results for a given service.
+ * It contains the group information as well as a context
+ * strings to tell where the information was found. It is more compact than a
+ * GroupMeta object, so as to make search responses as light as possible.
+ */
+struct RsGxsGroupSearchResults : RsSerializable
+{
+	RsGxsGroupSearchResults()
+        : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
+    {}
+
+	RsGxsGroupId mGroupId;
+	std::string  mGroupName;
+	RsGxsId      mAuthorId;
+	rstime_t     mPublishTs;
+	uint32_t     mNumberOfMessages;
+	rstime_t     mLastMessageTs;
+	uint32_t     mSignFlags;
+	uint32_t     mPopularity;
+
+    std::set<std::string> mSearchContexts;
+
+	/// @see RsSerializable::serial_process
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mGroupId);
+		RS_SERIAL_PROCESS(mGroupName);
+		RS_SERIAL_PROCESS(mAuthorId);
+		RS_SERIAL_PROCESS(mPublishTs);
+		RS_SERIAL_PROCESS(mNumberOfMessages);
+		RS_SERIAL_PROCESS(mLastMessageTs);
+		RS_SERIAL_PROCESS(mSignFlags);
+		RS_SERIAL_PROCESS(mPopularity);
+		RS_SERIAL_PROCESS(mSearchContexts);
+	}
+
+	virtual ~RsGxsGroupSearchResults() = default;
+};
 
 /*!
  * Stores ids of changed gxs groups and messages.

libretroshare/src/rsitems/rsgxsupdateitems.h

@@ -106,12 +106,22 @@ public:
     RsPeerId peerID;
 };
 
+struct RsPeerUpdateTsRecord
+{
+    RsPeerUpdateTsRecord() : mLastTsReceived(0), mTs(0) {}
+
+    rstime_t mLastTsReceived; // last TS that was sent for this group by this peer ID.
+    rstime_t mTs;             // time at which this TS was sent.
+};
+
 class RsGxsServerGrpUpdate
 {
 public:
     RsGxsServerGrpUpdate() { grpUpdateTS = 0 ; }
 
 	uint32_t grpUpdateTS;
+
+    std::map<RsPeerId,RsPeerUpdateTsRecord> grpUpdateTsRecords;
 };
 
 class RsGxsServerGrpUpdateItem : public RsGxsNetServiceItem, public RsGxsServerGrpUpdate
@@ -168,7 +178,13 @@ class RsGxsServerMsgUpdate
 public:
     RsGxsServerMsgUpdate() { msgUpdateTS = 0 ;}
 
-	uint32_t msgUpdateTS; // local time stamp this group last received a new msg
+	uint32_t msgUpdateTS; // local time stamp at which this group last received a new msg
+
+    // Now we also store for each peer the last own TS the peer sent and when it did so. This allows to detect when transactions are stuck because of
+    // outqueues clogging. If that happens, we receive multiple times the same TS from the friend, in which case we do not send the list of msgs
+    // again until a significant amount of time has passed. These values are obviously initialized to 0.
+
+    std::map<RsPeerId, RsPeerUpdateTsRecord> msgUpdateTsRecords;
 };
 
 class RsGxsServerMsgUpdateItem : public RsGxsNetServiceItem, public RsGxsServerMsgUpdate

libretroshare/src/services/p3gxschannels.cc

@@ -80,12 +80,15 @@ p3GxsChannels::p3GxsChannels(
                    RS_SERVICE_GXS_TYPE_CHANNELS, gixs, channelsAuthenPolicy() ),
     RsGxsChannels(static_cast<RsGxsIface&>(*this)), GxsTokenQueue(this),
     mSubscribedGroupsMutex("GXS channels subscribed groups cache"),
-    mKnownChannelsMutex("GXS channels known channels timestamp cache"),
+    mKnownChannelsMutex("GXS channels known channels timestamp cache")
+#ifdef TO_REMOVE
     mSearchCallbacksMapMutex("GXS channels search callbacks map"),
     mDistantChannelsCallbacksMapMutex("GXS channels distant channels callbacks map")
+#endif
 {
 	// For Dummy Msgs.
 	mGenActive = false;
+    mLastDistantSearchNotificationTS = 0;
 	mCommentService = new p3GxsCommentService(this,  RS_SERVICE_GXS_TYPE_CHANNELS);
 
 	RsTickEvent::schedule_in(CHANNEL_PROCESS, 0);
@@ -352,18 +355,6 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
 
 		}
 
-        RsGxsDistantSearchResultChange *dsrChange = dynamic_cast<RsGxsDistantSearchResultChange*>(*it);
-
-        if(dsrChange && rsEvents)
-        {
-			auto ev = std::make_shared<RsGxsChannelEvent>();
-			ev->mChannelGroupId = dsrChange->mGroupId;
-			ev->mChannelEventCode = RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT;
-			ev->mDistantSearchRequestId = dsrChange->mRequestId;
-
-			rsEvents->postEvent(ev);
-        }
-
 		/* shouldn't need to worry about groups - as they need to be subscribed to */
         delete *it;
 	}
@@ -383,17 +374,32 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
 void	p3GxsChannels::service_tick()
 {
 	static rstime_t last_dummy_tick = 0;
+    rstime_t now = time(NULL);
 
 	if (time(NULL) > last_dummy_tick + 5)
 	{
 		dummy_tick();
-		last_dummy_tick = time(NULL);
+		last_dummy_tick = now;
 	}
 
 	RsTickEvent::tick_events();
 	GxsTokenQueue::checkRequests();
 
 	mCommentService->comment_tick();
+
+    // Notify distant search results, not more than once per sec. Normally we should
+    // rather send one item for all, but that needs another class type
+
+    if(now > mLastDistantSearchNotificationTS+2 && !mSearchResultsToNotify.empty())
+	{
+		auto ev = std::make_shared<RsGxsChannelSearchResultEvent>();
+		ev->mSearchResultsMap = mSearchResultsToNotify;
+
+        mLastDistantSearchNotificationTS = now;
+        mSearchResultsToNotify.clear();
+
+		rsEvents->postEvent(ev);
+	}
 }
 
 bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups)
@@ -2210,7 +2216,9 @@ void p3GxsChannels::dummy_tick()
 
 	}
 
+#ifdef TO_REMOVE
 	cleanTimedOutCallbacks();
+#endif
 }
 
 
@@ -2389,18 +2397,18 @@ bool p3GxsChannels::clearDistantSearchResults(TurtleRequestId req)
 {
     return netService()->clearDistantSearchResults(req);
 }
-bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& results)
+bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& results)
 {
     return netService()->retrieveDistantSearchResults(req,results);
 }
 
-bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
+bool p3GxsChannels::getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
 {
-	RsGxsGroupSummary gs;
+	RsGxsGroupSearchResults gs;
 
     if(netService()->retrieveDistantGroupSummary(group_id,gs))
     {
-        // This is a placeholder information by the time we receive the full group meta data.
+        // This is a placeholder information by the time we receive the full group meta data and check the signature.
 		distant_group.mMeta.mGroupId         = gs.mGroupId ;
 		distant_group.mMeta.mGroupName       = gs.mGroupName;
 		distant_group.mMeta.mGroupFlags      = GXS_SERV::FLAG_PRIVACY_PUBLIC ;
@@ -2426,6 +2434,7 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann
         return false ;
 }
 
+#ifdef TO_REMOVE
 bool p3GxsChannels::turtleSearchRequest(
         const std::string& matchString,
         const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
@@ -2505,17 +2514,24 @@ bool p3GxsChannels::localSearchRequest(
 
 	return true;
 }
+#endif
 
-void p3GxsChannels::receiveDistantSearchResults(
+void p3GxsChannels::receiveDistantSearchResults( TurtleRequestId id, const RsGxsGroupId& grpId )
-        TurtleRequestId id, const RsGxsGroupId& grpId )
 {
-	std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")"
+	if(!rsEvents)
-	          << std::endl;
+		return;
+
+    // We temporise here, in order to avoid notifying clients with many events
+    // So we put some data in there and will send an event with all of them at once every 1 sec at most.
+
+    mSearchResultsToNotify[id].insert(grpId);
+
+#ifdef TO_REMOVE
+	std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")" << std::endl;
 
 	{
 		RsGenExchange::receiveDistantSearchResults(id, grpId);
-		RsGxsGroupSummary gs;
+		RsGxsGroupSearchResults gs;
-		gs.mGroupId = grpId;
 		netService()->retrieveDistantGroupSummary(grpId, gs);
 
 		{
@@ -2556,8 +2572,10 @@ void p3GxsChannels::receiveDistantSearchResults(
 			return;
 		}
 	} // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex);
+#endif
 }
 
+#ifdef TO_REMOVE
 void p3GxsChannels::cleanTimedOutCallbacks()
 {
 	auto now = std::chrono::system_clock::now();
@@ -2586,6 +2604,7 @@ void p3GxsChannels::cleanTimedOutCallbacks()
 			else ++cbpt;
 	} // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex)
 }
+#endif
 
 bool p3GxsChannels::exportChannelLink(
         std::string& link, const RsGxsGroupId& chanId, bool includeGxsData,

libretroshare/src/services/p3gxschannels.h

@@ -68,9 +68,9 @@ protected:
 
     virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id);
     virtual TurtleRequestId turtleSearchRequest(const std::string& match_string);
-    virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) ;
+    virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) ;
     virtual bool clearDistantSearchResults(TurtleRequestId req);
-    virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group);
+    virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group);
 
 	// Overloaded to cache new groups.
 virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
@@ -109,6 +109,7 @@ virtual	bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled);
 virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory);
 virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory);
 
+#ifdef TO_REMOVE
 	/// @see RsGxsChannels::turtleSearchRequest
 	virtual bool turtleSearchRequest(const std::string& matchString,
 	        const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
@@ -124,6 +125,7 @@ virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::strin
 	virtual bool localSearchRequest(const std::string& matchString,
 	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
 	        rstime_t maxWait = 30 ) override;
+#endif
 
 	/**
 	 * Receive results from turtle search @see RsGenExchange @see RsNxsObserver
@@ -374,6 +376,9 @@ bool generateGroup(uint32_t &token, std::string groupName);
 	std::map<RsGxsGroupId,rstime_t> mKnownChannels;
 	RsMutex mKnownChannelsMutex;
 
+    rstime_t mLastDistantSearchNotificationTS;
+    std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
+#ifdef TO_REMOVE
 	/** Store search callbacks with timeout*/
 	std::map<
 	    TurtleRequestId,
@@ -394,4 +399,5 @@ bool generateGroup(uint32_t &token, std::string groupName);
 
 	/// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap
 	void cleanTimedOutCallbacks();
+#endif
 };

retroshare-gui/src/gui/common/GroupTreeWidget.cpp

@@ -397,6 +397,19 @@ bool GroupTreeWidget::isSearchRequestResult(QPoint &point,QString& group_id,uint
 	return search_req_id > 0;
 }
 
+bool GroupTreeWidget::isSearchRequestResultItem(QTreeWidgetItem *item,QString& group_id,uint32_t& search_req_id)
+{
+	QTreeWidgetItem *parent = item->parent();
+
+	if(parent == NULL)
+		return false ;
+
+	search_req_id = parent->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt();
+	group_id = itemId(item) ;
+
+	return search_req_id > 0;
+}
+
 bool GroupTreeWidget::isSearchRequestItem(QPoint &point,uint32_t& search_req_id)
 {
     QTreeWidgetItem *item = ui->treeWidget->itemAt(point);
@@ -463,6 +476,17 @@ void GroupTreeWidget::fillGroupItems(QTreeWidgetItem *categoryItem, const QList<
 		item->setData(COLUMN_DATA, ROLE_NAME, itemInfo.name);
 		item->setData(COLUMN_DATA, ROLE_DESCRIPTION, itemInfo.description);
 
+        // Add children for context strings. This happens in the search.
+        while(nullptr != item->takeChild(0));
+
+        for(auto str:itemInfo.context_strings)
+            if(!str.empty())
+            {
+                QTreeWidgetItem *it = new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str())));
+                it->setData(COLUMN_DATA,ROLE_ID,itemInfo.id);
+				item->addChild(it);
+            }
+
 		/* Set last post */
 		qlonglong lastPost = itemInfo.lastpost.toTime_t();
 		item->setData(COLUMN_DATA, ROLE_LASTPOST, -lastPost); // negative for correct sorting

retroshare-gui/src/gui/common/GroupTreeWidget.h

@@ -21,6 +21,8 @@
 #ifndef GROUPTREEWIDGET_H
 #define GROUPTREEWIDGET_H
 
+#include<set>
+
 #include <QTreeWidgetItem>
 #include <QDateTime>
 
@@ -47,16 +49,17 @@ public:
 	{}
 
 public:
-	QString   id;
+	QString               id;
-	QString   name;
+	QString               name;
-	QString   description;
+	QString               description;
-	int       popularity;
+	int                   popularity;
-	QDateTime lastpost;
+	QDateTime             lastpost;
-	QIcon     icon;
+	QIcon                 icon;
-	bool      publishKey;
+	bool                  publishKey;
-	bool      adminKey;
+	bool                  adminKey;
-    quint32  subscribeFlags;
+    quint32               subscribeFlags;
-    quint32  max_visible_posts ;
+    quint32               max_visible_posts ;
+    std::set<std::string> context_strings;
 };
 
 //cppcheck-suppress noConstructor
@@ -94,6 +97,7 @@ public:
 
 	bool isSearchRequestItem(QPoint &point,uint32_t& search_req_id);
 	bool isSearchRequestResult(QPoint &point, QString &group_id, uint32_t& search_req_id);
+	bool isSearchRequestResultItem(QTreeWidgetItem *item,QString& group_id,uint32_t& search_req_id);
 
 	QTreeWidgetItem *getItemFromId(const QString &id);
 	QTreeWidgetItem *activateId(const QString &id, bool focus);

retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp

@@ -29,6 +29,7 @@
 #include "gui/settings/rsharesettings.h"
 #include "gui/RetroShareLink.h"
 #include "gui/gxs/GxsGroupShareKey.h"
+#include "gui/common/GroupTreeWidget.h"
 #include "gui/common/RSTreeWidget.h"
 #include "gui/notifyqt.h"
 #include "gui/common/UIStateHelper.h"
@@ -290,55 +291,50 @@ void GxsGroupFrameDialog::updateDisplay(bool complete)
     if(complete)    // || !getGrpIds().empty() || !getGrpIdsMeta().empty()) {
 		updateGroupSummary(); /* Update group list */
 
-    updateSearchResults() ;
+//    updateSearchResults() ;
 }
 
 void GxsGroupFrameDialog::updateSearchResults()
 {
-    const std::set<TurtleRequestId>& reqs = getSearchRequests();
+    for(auto& it:mSearchGroupsItems)
+        updateSearchResults(it.first);
+}
 
-    for(auto it(reqs.begin());it!=reqs.end();++it)
+void GxsGroupFrameDialog::updateSearchResults(const TurtleRequestId& sid)
-    {
+{
-		std::cerr << "updating search ID " << std::hex << *it << std::dec << std::endl;
+	std::cerr << "updating search ID " << std::hex << sid << std::dec << std::endl;
 
-        std::map<RsGxsGroupId,RsGxsGroupSummary> group_infos;
+	std::map<RsGxsGroupId,RsGxsGroupSearchResults> group_infos;
 
-        getDistantSearchResults(*it,group_infos) ;
+	getDistantSearchResults(sid,group_infos) ;
 
-        std::cerr << "retrieved " << std::endl;
+	std::cerr << "retrieved " << std::endl;
 
-        auto it2 = mSearchGroupsItems.find(*it);
+	auto it2 = mSearchGroupsItems.find(sid);
 
-        if(mSearchGroupsItems.end() == it2)
+	QList<GroupItemInfo> group_items ;
-        {
-            std::cerr << "GxsGroupFrameDialog::updateSearchResults(): received result notification for req " << std::hex << *it << std::dec << " but no item present!" << std::endl;
-            continue ;	// we could create the item just as well but since this situation is not supposed to happen, I prefer to make this a failure case.
-        }
 
-        QList<GroupItemInfo> group_items ;
+	for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
+	{
+		std::cerr << "  adding group " << it3->first << " " << it3->second.mGroupId << " \"" << it3->second.mGroupName << "\"" << std::endl;
+        for(auto s:it3->second.mSearchContexts)
+			std::cerr << "    Context string \"" << s << "\"" << std::endl;
 
-		for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
+		GroupItemInfo i;
-			if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end())
+		i.id             = QString(it3->second.mGroupId.toStdString().c_str());
-			{
+		i.name           = QString::fromUtf8(it3->second.mGroupName.c_str());
-				std::cerr << "  adding new group " << it3->first << " "
+		i.popularity     = 0; // could be set to the number of hits
-				          << it3->second.mGroupId << " \""
+		i.lastpost       = QDateTime::fromTime_t(it3->second.mLastMessageTs);
-				          << it3->second.mGroupName << "\"" << std::endl;
+		i.subscribeFlags = 0; // irrelevant here
+		i.publishKey     = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags);
+		i.adminKey       = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags);
+		i.max_visible_posts = it3->second.mNumberOfMessages;
+		i.context_strings = it3->second.mSearchContexts;
 
-				GroupItemInfo i;
+		group_items.push_back(i);
-				i.id             = QString(it3->second.mGroupId.toStdString().c_str());
+	}
-				i.name           = QString::fromUtf8(it3->second.mGroupName.c_str());
-				i.popularity     = 0; // could be set to the number of hits
-				i.lastpost       = QDateTime::fromTime_t(it3->second.mLastMessageTs);
-				i.subscribeFlags = 0; // irrelevant here
-				i.publishKey     = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags);
-				i.adminKey       = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags);
-				i.max_visible_posts = it3->second.mNumberOfMessages;
 
-				group_items.push_back(i);
+	ui->groupTreeWidget->fillGroupItems(it2->second, group_items);
-			}
-
-		ui->groupTreeWidget->fillGroupItems(it2->second, group_items);
-    }
 }
 
 void GxsGroupFrameDialog::todo()
@@ -364,13 +360,22 @@ void GxsGroupFrameDialog::removeCurrentSearch()
     mSearchGroupsItems.erase(it);
 
     mKnownGroups.erase(search_request_id);
+
+    clearDistantSearchResults(search_request_id);
 }
 
 void GxsGroupFrameDialog::removeAllSearches()
 {
     for(auto it(mSearchGroupsItems.begin());it!=mSearchGroupsItems.end();++it)
-		ui->groupTreeWidget->removeSearchItem(it->second) ;
+    {
+        QString group_id;
+        TurtleRequestId search_request_id;
 
+        if(ui->groupTreeWidget->isSearchRequestResultItem(it->second,group_id,search_request_id))
+			clearDistantSearchResults(search_request_id);
+
+		ui->groupTreeWidget->removeSearchItem(it->second) ;
+    }
     mSearchGroupsItems.clear();
     mKnownGroups.clear();
 }
@@ -1074,15 +1079,20 @@ void GxsGroupFrameDialog::updateGroupSummary()
 {
 	RsThread::async([this]()
 	{
-		std::list<RsGxsGenericGroupData*> groupInfo;
+		auto groupInfo = new std::list<RsGxsGenericGroupData*>() ;
 
-		if(!getGroupData(groupInfo))
+		if(!getGroupData(*groupInfo))
 		{
-			std::cerr << __PRETTY_FUNCTION__ << " failed to collect group info " << std::endl;
+			std::cerr << __PRETTY_FUNCTION__ << " failed to collect group info." << std::endl;
+            delete groupInfo;
 			return;
 		}
-        if(groupInfo.empty())
+        if(groupInfo->empty())
+        {
+			std::cerr << __PRETTY_FUNCTION__ << " no group info collected." << std::endl;
+            delete groupInfo;
             return;
+        }
 
 		RsQThreadUtils::postToObject( [this,groupInfo]()
 		{
@@ -1092,7 +1102,7 @@ void GxsGroupFrameDialog::updateGroupSummary()
 			 * Qt::QueuedConnection is important!
 			 */
 
-			insertGroupsData(groupInfo);
+			insertGroupsData(*groupInfo);
 			updateSearchResults();
 
 			mStateHelper->setLoading(TOKEN_TYPE_GROUP_SUMMARY, false);
@@ -1111,12 +1121,14 @@ void GxsGroupFrameDialog::updateGroupSummary()
 
 			// now delete the data that is not used anymore
 
-			for(auto& g:groupInfo)
+			for(auto& g:*groupInfo)
 			{
 				mCachedGroupMetas[g->mMeta.mGroupId] = g->mMeta;
 				delete g;
 			}
 
+            delete groupInfo;
+
 		}, this );
 	});
 }

retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h

@@ -161,7 +161,9 @@ private:
 	virtual void groupTreeCustomActions(RsGxsGroupId /*grpId*/, int /*subscribeFlags*/, QList<QAction*> &/*actions*/) {}
 	virtual RsGxsCommentService *getCommentService() { return NULL; }
 	virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; }
-    virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map<RsGxsGroupId,RsGxsGroupSummary>& /* group_infos */){ return false ;}
+    virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& /* group_infos */){ return false ;}
+    virtual void clearDistantSearchResults(TurtleRequestId /* id */) {}
+    virtual RsGxsGenericGroupData *getDistantSearchResultGroupData(const RsGxsGroupId& group_id){ return nullptr ;}
 
 	void initUi();
 
@@ -187,7 +189,8 @@ private:
 	GxsCommentDialog *commentWidget(const RsGxsMessageId &msgId);
 
 protected:
-	void updateSearchResults();
+	void updateSearchResults(const TurtleRequestId &sid);
+	void updateSearchResults();	// update all searches
 
 	bool mCountChildMsgs; // Count unread child messages?
 

retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp

@@ -61,9 +61,7 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> ev
 {
 	const RsGxsChannelEvent *e = dynamic_cast<const RsGxsChannelEvent*>(event.get());
 
-	if(!e)
+	if(e)
-		return;
-
         switch(e->mChannelEventCode)
         {
 		case RsChannelEventCode::NEW_MESSAGE:             // [[fallthrough]];
@@ -72,11 +70,6 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> ev
 			updateGroupStatisticsReal(e->mChannelGroupId); // update the list immediately
             break;
 
-        case RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT:
-            mSearchResults.insert(e->mDistantSearchRequestId);
-            updateSearchResults();
-            break;
-
 		case RsChannelEventCode::NEW_CHANNEL:       // [[fallthrough]];
         case RsChannelEventCode::SUBSCRIBE_STATUS_CHANGED:
             updateDisplay(true);
@@ -89,6 +82,13 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> ev
         default:
             break;
         }
+
+
+    const RsGxsChannelSearchResultEvent*f = dynamic_cast<const RsGxsChannelSearchResultEvent*>(event.get());
+
+	if(nullptr != f)
+        for(auto it:f->mSearchResultsMap)
+			updateSearchResults(it.first);
 }
 
 GxsChannelDialog::~GxsChannelDialog()
@@ -396,21 +396,36 @@ void GxsChannelDialog::groupInfoToGroupItemInfo(const RsGxsGenericGroupData *gro
 	groupItemInfo.description = QString::fromUtf8(channelGroupData->mDescription.c_str());
 }
 
+void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id)
+{
+    rsGxsChannels->clearDistantSearchResults(id);
+}
+
 TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string)
 {
     return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ;
 }
 
-bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
+bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
     return rsGxsChannels->retrieveDistantSearchResults(id,group_infos);
 }
 
+RsGxsGenericGroupData *GxsChannelDialog::getDistantSearchResultGroupData(const RsGxsGroupId& group_id)
+{
+    RsGxsChannelGroup channel_group;
+
+    if(rsGxsChannels->getDistantSearchResultGroupData(group_id,channel_group))
+        return new RsGxsGenericGroupData(channel_group);
+	else
+        return nullptr;
+}
+
 void GxsChannelDialog::checkRequestGroup(const RsGxsGroupId& grpId)
 {
     RsGxsChannelGroup distant_group;
 
-	if( rsGxsChannels->retrieveDistantGroup(grpId,distant_group)) // normally we should also check that the group meta is not already here.
+	if( rsGxsChannels->getDistantSearchResultGroupData(grpId,distant_group)) // normally we should also check that the group meta is not already here.
     {
         std::cerr << "GxsChannelDialog::checkRequestGroup() sending turtle request for group data for group " << grpId << std::endl;
         rsGxsChannels->turtleGroupRequest(grpId);

retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h

@@ -43,10 +43,12 @@ public:
 
 protected:
 	/* GxsGroupFrameDialog */
-    virtual bool getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos);
+    virtual bool getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos) override;
+	virtual RsGxsGenericGroupData *getDistantSearchResultGroupData(const RsGxsGroupId& group_id) override;
 
-	virtual TurtleRequestId distantSearch(const QString& search_string) ;
+	virtual TurtleRequestId distantSearch(const QString& search_string) override;
-    virtual void checkRequestGroup(const RsGxsGroupId& grpId) ;
+    virtual void checkRequestGroup(const RsGxsGroupId& grpId) override ;
+	virtual void clearDistantSearchResults(TurtleRequestId id) override;
 
     // Implementation of some abstract methods in GxsGroupFrameDialog
 

retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp

@@ -59,6 +59,7 @@ static const int mTokenTypeGroupData = 1;
 
 static const int CHANNEL_TABS_DETAILS= 0;
 static const int CHANNEL_TABS_POSTS  = 1;
+static const int CHANNEL_TABS_FILES  = 2;
 
 /* View mode */
 #define VIEW_MODE_FEEDS  1
@@ -538,23 +539,20 @@ void GxsChannelPostsWidgetWithModel::updateGroupData()
 
 	RsThread::async([this]()
 	{
+		RsGxsChannelGroup group;
 		std::vector<RsGxsChannelGroup> groups;
 
-		if(!rsGxsChannels->getChannelsInfo(std::list<RsGxsGroupId>{ groupId() }, groups))
+		if(rsGxsChannels->getChannelsInfo(std::list<RsGxsGroupId>{ groupId() }, groups) && groups.size()==1)
+            group = groups[0];
+        else if(!rsGxsChannels->getDistantSearchResultGroupData(groupId(),group))
 		{
-			std::cerr << __PRETTY_FUNCTION__ << " failed to get autodownload value for channel: " << groupId() << std::endl;
+			std::cerr << __PRETTY_FUNCTION__ << " failed to get group data for channel: " << groupId() << std::endl;
 			return;
 		}
 
-		if(groups.size() != 1)
+		RsQThreadUtils::postToObject( [this,group]()
-		{
-			RsErr() << __PRETTY_FUNCTION__ << " cannot retrieve channel data for group ID " << groupId() << ": ERROR." << std::endl;
-			return;
-		}
-
-		RsQThreadUtils::postToObject( [this,groups]()
         {
-            mGroup = groups[0];
+            mGroup = group;
 			mChannelPostsModel->updateChannel(groupId());
 
             insertChannelDetails(mGroup);
@@ -779,12 +777,14 @@ void GxsChannelPostsWidgetWithModel::insertChannelDetails(const RsGxsChannelGrou
 		//ui->feedToolButton->setEnabled(true);
 		//ui->fileToolButton->setEnabled(true);
         ui->channel_TW->setTabEnabled(CHANNEL_TABS_POSTS,true);
+        ui->channel_TW->setTabEnabled(CHANNEL_TABS_FILES,true);
         ui->details_TW->setEnabled(true);
 	}
     else
     {
         ui->details_TW->setEnabled(false);
         ui->channel_TW->setTabEnabled(CHANNEL_TABS_POSTS,false);
+        ui->channel_TW->setTabEnabled(CHANNEL_TABS_FILES,false);
     }
 
 

retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.ui

@@ -161,7 +161,7 @@
    <item>
     <widget class="QTabWidget" name="channel_TW">
      <property name="currentIndex">
-      <number>1</number>
+      <number>0</number>
      </property>
      <widget class="QWidget" name="tab_3">
       <attribute name="title">

retroshare-gui/src/gui/gxsforums/GxsForumModel.cpp

@@ -669,8 +669,11 @@ QVariant RsGxsForumModel::displayRole(const ForumModelPostEntry& fmpe,int col) c
 									return QVariant(DateTime::formatDateTime(qtime));
     							}
 
-		case COLUMN_THREAD_DISTRIBUTION:
+		case COLUMN_THREAD_DISTRIBUTION:	// passthrough // handled by delegate.
-		case COLUMN_THREAD_AUTHOR:{
+		case COLUMN_THREAD_MSGID:
+        								return QVariant();
+		case COLUMN_THREAD_AUTHOR:
+    	{
 			QString name;
 			RsGxsId id = RsGxsId(fmpe.mAuthorId.toStdString());
 
@@ -680,7 +683,6 @@ QVariant RsGxsForumModel::displayRole(const ForumModelPostEntry& fmpe,int col) c
 				return name;
 			return QVariant(tr("[Unknown]"));
 		}
-		case COLUMN_THREAD_MSGID: return QVariant();
 #ifdef TODO
 	if (filterColumn == COLUMN_THREAD_CONTENT) {
 		// need content for filter

retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp

@@ -134,6 +134,13 @@ public:
 		const QPoint p = QPoint((r.width() - pix.width())/2, (r.height() - pix.height())/2);
 		painter->drawPixmap(r.topLeft() + p, pix);
 	}
+
+    virtual QSize sizeHint(const QStyleOptionViewItem& option, const QModelIndex& index) const override
+	{
+        static auto img(FilesDefs::getPixmapFromQtResourcePath(IMAGE_WARNING_YELLOW));
+
+		return QSize(img.width()*1.2,option.rect.height());
+	}
 };
 
 class ReadStatusItemDelegate: public QStyledItemDelegate
@@ -183,6 +190,13 @@ public:
 		const QPoint p = QPoint((r.width() - pix.width())/2, (r.height() - pix.height())/2);
 		painter->drawPixmap(r.topLeft() + p, pix);
 	}
+
+	virtual QSize sizeHint(const QStyleOptionViewItem& option, const QModelIndex& index) const override
+	{
+        static auto img(FilesDefs::getPixmapFromQtResourcePath(":/images/message-state-unread.png"));
+
+		return QSize(img.width()*1.2,option.rect.height());
+	}
 };
 
 class ForumPostSortFilterProxyModel: public QSortFilterProxyModel
@@ -299,9 +313,10 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget
 	QHeaderView * ttheader = ui->threadTreeWidget->header () ;
 	ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_DATE,  140*f);
 	ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_TITLE, 440*f);
-	ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_DISTRIBUTION, 24*f);
 	ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_AUTHOR, 150*f);
-	ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_READ,  24*f);
+
+    ui->threadTreeWidget->resizeColumnToContents(RsGxsForumModel::COLUMN_THREAD_DISTRIBUTION);
+    ui->threadTreeWidget->resizeColumnToContents(RsGxsForumModel::COLUMN_THREAD_READ);
 
 	QHeaderView_setSectionResizeModeColumn(ttheader, RsGxsForumModel::COLUMN_THREAD_TITLE,        QHeaderView::Interactive);
 	QHeaderView_setSectionResizeModeColumn(ttheader, RsGxsForumModel::COLUMN_THREAD_DATE,         QHeaderView::Interactive);