From 61437cd0b409df19e679d068ade3900a9e862124 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 15 Jun 2020 21:11:58 +0200 Subject: [PATCH 01/11] added some debug info in rsgxsnetservice distant search result handling and fixed a bug causing so some search result to not show up --- libretroshare/src/gxs/rsgxsnetservice.cc | 35 ++++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 72a078075..139cc21c8 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5189,8 +5189,7 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) return true ; } -void RsGxsNetService::receiveTurtleSearchResults( - TurtleRequestId req, const std::list& group_infos ) +void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list& group_infos ) { std::set groupsToNotifyResults; @@ -5198,20 +5197,44 @@ void RsGxsNetService::receiveTurtleSearchResults( RS_STACK_MUTEX(mNxsMutex); RsGxsGrpMetaTemporaryMap grpMeta; - std::map& - search_results_map(mDistantSearchResults[req]); + std::map& search_results_map(mDistantSearchResults[req]); + + std::cerr << "Received group summary through turtle search for the following groups:" << std::endl; for(const RsGxsGroupSummary& gps : group_infos) + { + std::cerr <<" " << gps.mGroupId << " \"" << gps.mGroupName << "\" " ; + if(search_results_map.find(gps.mGroupId) == search_results_map.end()) + { + std::cerr << "added to search." << std::endl; grpMeta[gps.mGroupId] = nullptr; + } + else + std::cerr << "ignored (already in search results)." << std::endl; + } + + // Do a search for only groups that we dont have yet in the search results. + mDataStore->retrieveGxsGrpMetaData(grpMeta); + std::cerr << "Retrieved data store group data for the following groups:" <mGroupName << std::endl; + for (const RsGxsGroupSummary& gps : group_infos) { #ifndef RS_DEEP_CHANNEL_INDEX /* Only keep groups that are not locally known, and groups that are - * not already in the mDistantSearchResults structure. */ - if(grpMeta[gps.mGroupId]) continue; + * not already in the mDistantSearchResults structure. + * mDataStore may in some situations allocate an empty group meta data, so it's important + * to test that the group meta is both non null and actually corresponds to the group id we seek. */ + + if(grpMeta[gps.mGroupId] != nullptr && grpMeta[gps.mGroupId]->mGroupId == gps.mGroupId) + continue; + + std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl; + #else // ndef RS_DEEP_CHANNEL_INDEX /* When deep search is enabled search results may bring more info * then we already have also about post that are indexed by xapian, From 8e2c6707165954da4e5da657e325b968a74d820a Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 16 Jun 2020 00:00:22 +0200 Subject: [PATCH 02/11] fixed the logic with distant search. --- libretroshare/src/gxs/rsgxsnetservice.cc | 41 ++++++++---------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 139cc21c8..7d9132f1b 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5203,19 +5203,10 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std for(const RsGxsGroupSummary& gps : group_infos) { - std::cerr <<" " << gps.mGroupId << " \"" << gps.mGroupName << "\" " ; - - if(search_results_map.find(gps.mGroupId) == search_results_map.end()) - { - std::cerr << "added to search." << std::endl; - grpMeta[gps.mGroupId] = nullptr; - } - else - std::cerr << "ignored (already in search results)." << std::endl; + std::cerr <<" " << gps.mGroupId << " \"" << gps.mGroupName << "\"" << std::endl; + grpMeta[gps.mGroupId] = nullptr; } - // Do a search for only groups that we dont have yet in the search results. - mDataStore->retrieveGxsGrpMetaData(grpMeta); std::cerr << "Retrieved data store group data for the following groups:" <mGroupId == gps.mGroupId) + auto& meta(grpMeta[gps.mGroupId]); + + if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue; std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl; @@ -5244,21 +5237,15 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std groupsToNotifyResults.insert(grpId); auto it2 = search_results_map.find(grpId); - if(it2 != search_results_map.end()) - { - // update existing data - RsGxsGroupSummary& eGpS(it2->second); - eGpS.mPopularity++; - eGpS.mNumberOfMessages = std::max( - eGpS.mNumberOfMessages, - gps.mNumberOfMessages ); - } - else - { - search_results_map[grpId] = gps; - // number of results so far - search_results_map[grpId].mPopularity = 1; - } + + RsGxsGroupSummary& eGpS(search_results_map[grpId]); + + int popularity = eGpS.mPopularity + 1; + int number_of_msg = std::max( eGpS.mNumberOfMessages, gps.mNumberOfMessages ); + + eGpS = gps; + eGpS.mPopularity = popularity; + eGpS.mNumberOfMessages = number_of_msg; } } // end RS_STACK_MUTEX(mNxsMutex); From 33da5c06b8a4b436df642eabd797c7c9db3a2112 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 16 Jun 2020 21:13:55 +0200 Subject: [PATCH 03/11] removed multi-callback based distant search result handling, and use rsEvents instead --- libretroshare/src/gxs/rsgenexchange.cc | 8 +-- libretroshare/src/gxs/rsgxsnetservice.cc | 34 +++++++--- libretroshare/src/gxs/rsgxsnetservice.h | 6 +- libretroshare/src/gxs/rsgxsnotify.h | 10 --- libretroshare/src/gxs/rsnxs.h | 4 +- libretroshare/src/retroshare/rsgxschannels.h | 35 +++++++--- libretroshare/src/retroshare/rsgxsiface.h | 40 ++++++++++++ libretroshare/src/services/p3gxschannels.cc | 65 ++++++++++++------- libretroshare/src/services/p3gxschannels.h | 8 ++- .../src/gui/common/GroupTreeWidget.cpp | 6 ++ .../src/gui/common/GroupTreeWidget.h | 23 ++++--- .../src/gui/gxs/GxsGroupFrameDialog.cpp | 63 ++++++++---------- .../src/gui/gxs/GxsGroupFrameDialog.h | 5 +- .../src/gui/gxschannels/GxsChannelDialog.cpp | 18 ++--- .../src/gui/gxschannels/GxsChannelDialog.h | 2 +- 15 files changed, 204 insertions(+), 123 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index bfc1bbd23..e3ef41eb9 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1675,13 +1675,7 @@ void RsGenExchange::receiveNewMessages(std::vector& messages) void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId) { - std::cerr << __PRETTY_FUNCTION__ << " received result for request " - << std::hex << id << std::dec << std::endl; - - RS_STACK_MUTEX(mGenMtx); - - RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId); - mNotifications.push_back(gc); + std::cerr << __PRETTY_FUNCTION__ << " received result for request " << std::hex << id << std::dec << ": this method should be overloaded in the client service, but it is not. This is a bug!" << std::endl; } void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 7d9132f1b..58a49f35c 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5155,7 +5155,7 @@ static bool termSearch(const std::string& src, const std::string& substring) } #endif // ndef RS_DEEP_CHANNEL_INDEX -bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) +bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) { RS_STACK_MUTEX(mNxsMutex) ; @@ -5167,7 +5167,7 @@ bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map< group_infos = it->second; return true ; } -bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs) +bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSearchResults& gs) { RS_STACK_MUTEX(mNxsMutex) ; for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it) @@ -5197,7 +5197,7 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std RS_STACK_MUTEX(mNxsMutex); RsGxsGrpMetaTemporaryMap grpMeta; - std::map& search_results_map(mDistantSearchResults[req]); + std::map& search_results_map(mDistantSearchResults[req]); std::cerr << "Received group summary through turtle search for the following groups:" << std::endl; @@ -5236,16 +5236,30 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std const RsGxsGroupId& grpId(gps.mGroupId); groupsToNotifyResults.insert(grpId); - auto it2 = search_results_map.find(grpId); - RsGxsGroupSummary& eGpS(search_results_map[grpId]); + // Find search results place for this particular group - int popularity = eGpS.mPopularity + 1; - int number_of_msg = std::max( eGpS.mNumberOfMessages, gps.mNumberOfMessages ); + std::cerr << " Adding gps=" << gps.mGroupId << " name=\"" << gps.mGroupName << "\" gps.mSearchContext=\"" << gps.mSearchContext << "\"" << std::endl; + RsGxsGroupSearchResults& eGpS(search_results_map[grpId]); - eGpS = gps; - eGpS.mPopularity = popularity; - eGpS.mNumberOfMessages = number_of_msg; + if(eGpS.mGroupId != grpId) // not initialized yet. So we do it now. + { + eGpS.mGroupId = gps.mGroupId; + eGpS.mGroupName = gps.mGroupName; + eGpS.mAuthorId = gps.mAuthorId; + eGpS.mPublishTs = gps.mPublishTs; + eGpS.mSignFlags = gps.mSignFlags; + } + // We should check that the above values are always the same for all info that is received. In the end, we'll + // request the group meta and check the signature, but it may be misleading to receive a forged information + // that is not the real one. + + ++eGpS.mPopularity; // increase popularity. This is not a real counting, but therefore some heuristic estimate. + eGpS.mNumberOfMessages = std::max( eGpS.mNumberOfMessages, gps.mNumberOfMessages ); + eGpS.mLastMessageTs = std::max( eGpS.mLastMessageTs, gps.mLastMessageTs ); + + if(gps.mSearchContext != gps.mGroupName) // this is a bit of a hack. We should have flags to tell where the search hit happens + eGpS.mSearchContexts.insert(gps.mSearchContext); } } // end RS_STACK_MUTEX(mNxsMutex); diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index e4056bb06..e4f51f4b8 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -140,9 +140,9 @@ public: virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos); virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len); - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos); + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos); virtual bool clearDistantSearchResults(const TurtleRequestId& id); - virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&); + virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&, RsGxsGroupSearchResults &); /*! * pauses synchronisation of subscribed groups and request for group id @@ -609,7 +609,7 @@ private: std::set mNewPublishKeysToNotify ; // Distant search result map - std::map > mDistantSearchResults ; + std::map > mDistantSearchResults ; void debugDump(); diff --git a/libretroshare/src/gxs/rsgxsnotify.h b/libretroshare/src/gxs/rsgxsnotify.h index 4640f133a..a6fd4c6fb 100644 --- a/libretroshare/src/gxs/rsgxsnotify.h +++ b/libretroshare/src/gxs/rsgxsnotify.h @@ -76,16 +76,6 @@ protected: bool mMetaChange; }; -class RsGxsDistantSearchResultChange: public RsGxsNotify -{ -public: - RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& gid) : RsGxsNotify(gid), mRequestId(id){} - - NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; } - - TurtleRequestId mRequestId ; -}; - /*! * Relevant to message changes */ diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 2752fa6bf..7276f3421 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -128,7 +128,7 @@ public: * \return * false when the request is unknown. */ - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos)=0; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos)=0; /*! * \brief getDistantSearchResults * \param id @@ -136,7 +136,7 @@ public: * \return */ virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; - virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0; + virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0; virtual bool search(const std::string& substring,std::list& group_infos) =0; virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 4b188ad01..c03f22970 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -117,9 +117,7 @@ enum class RsChannelEventCode: uint8_t struct RsGxsChannelEvent: RsEvent { - RsGxsChannelEvent(): - RsEvent(RsEventType::GXS_CHANNELS), - mChannelEventCode(RsChannelEventCode::UNKNOWN) {} + RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {} RsChannelEventCode mChannelEventCode; RsGxsGroupId mChannelGroupId; @@ -138,6 +136,23 @@ struct RsGxsChannelEvent: RsEvent } }; +// This event is used to factor multiple search results notifications in a single event. + +struct RsGxsChannelSearchResultEvent: public RsEvent +{ + RsGxsChannelSearchResultEvent() : RsEvent(RsEventType::GXS_CHANNELS) {} + + std::map > mSearchResultsMap; + + ///* @see RsEvent @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override + { + RsEvent::serial_process(j, ctx); + + RS_SERIAL_PROCESS(mSearchResultsMap); + } +}; + class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService { public: @@ -407,7 +422,7 @@ public: */ virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0; - +#ifdef TO_REMOVE /** * @brief Request remote channels search * @jsonapi{development} @@ -448,6 +463,7 @@ public: const std::string& matchString, const std::function& multiCallback, rstime_t maxWait = 30 ) = 0; +#endif /// default base URL used for channels links @see exportChannelLink static const std::string DEFAULT_CHANNEL_BASE_URL; @@ -695,16 +711,15 @@ public: /// Distant synchronisation methods /// ////////////////////////////////////////////////////////////////////////////// /// - RS_DEPRECATED_FOR(turtleChannelRequest) + /// + // This approach is much cleaner than the "multicallback" system. We should keep it and use rsEvents to warn when + // new results are received. virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; - RS_DEPRECATED virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; - RS_DEPRECATED_FOR(turtleSearchRequest) - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; - RS_DEPRECATED virtual bool clearDistantSearchResults(TurtleRequestId req)=0; - RS_DEPRECATED_FOR(turtleChannelRequest) virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; + ////////////////////////////////////////////////////////////////////////////// ~RsGxsChannels() override; diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index 36bee517e..74e67008f 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -72,6 +72,46 @@ struct RsGxsGroupSummary : RsSerializable ~RsGxsGroupSummary(); }; +/*! + * This structure is used to locally store group search results for a given service. + * It contains the group information as well as a context + * strings to tell where the information was found. It is more compact than a + * GroupMeta object, so as to make search responses as light as possible. + */ +struct RsGxsGroupSearchResults : RsSerializable +{ + RsGxsGroupSearchResults() + : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0) + {} + + RsGxsGroupId mGroupId; + std::string mGroupName; + RsGxsId mAuthorId; + rstime_t mPublishTs; + uint32_t mNumberOfMessages; + rstime_t mLastMessageTs; + uint32_t mSignFlags; + uint32_t mPopularity; + + std::set mSearchContexts; + + /// @see RsSerializable::serial_process + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) + { + RS_SERIAL_PROCESS(mGroupId); + RS_SERIAL_PROCESS(mGroupName); + RS_SERIAL_PROCESS(mAuthorId); + RS_SERIAL_PROCESS(mPublishTs); + RS_SERIAL_PROCESS(mNumberOfMessages); + RS_SERIAL_PROCESS(mLastMessageTs); + RS_SERIAL_PROCESS(mSignFlags); + RS_SERIAL_PROCESS(mPopularity); + RS_SERIAL_PROCESS(mSearchContexts); + } + + virtual ~RsGxsGroupSearchResults() = default; +}; /*! * Stores ids of changed gxs groups and messages. diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 2a699e662..01ddb4357 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -80,12 +80,15 @@ p3GxsChannels::p3GxsChannels( RS_SERVICE_GXS_TYPE_CHANNELS, gixs, channelsAuthenPolicy() ), RsGxsChannels(static_cast(*this)), GxsTokenQueue(this), mSubscribedGroupsMutex("GXS channels subscribed groups cache"), - mKnownChannelsMutex("GXS channels known channels timestamp cache"), + mKnownChannelsMutex("GXS channels known channels timestamp cache") +#ifdef TO_REMOVE mSearchCallbacksMapMutex("GXS channels search callbacks map"), mDistantChannelsCallbacksMapMutex("GXS channels distant channels callbacks map") +#endif { // For Dummy Msgs. mGenActive = false; + mLastDistantSearchNotificationTS = 0; mCommentService = new p3GxsCommentService(this, RS_SERVICE_GXS_TYPE_CHANNELS); RsTickEvent::schedule_in(CHANNEL_PROCESS, 0); @@ -352,18 +355,6 @@ void p3GxsChannels::notifyChanges(std::vector &changes) } - RsGxsDistantSearchResultChange *dsrChange = dynamic_cast(*it); - - if(dsrChange && rsEvents) - { - auto ev = std::make_shared(); - ev->mChannelGroupId = dsrChange->mGroupId; - ev->mChannelEventCode = RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT; - ev->mDistantSearchRequestId = dsrChange->mRequestId; - - rsEvents->postEvent(ev); - } - /* shouldn't need to worry about groups - as they need to be subscribed to */ delete *it; } @@ -383,17 +374,32 @@ void p3GxsChannels::notifyChanges(std::vector &changes) void p3GxsChannels::service_tick() { static rstime_t last_dummy_tick = 0; + rstime_t now = time(NULL); if (time(NULL) > last_dummy_tick + 5) { dummy_tick(); - last_dummy_tick = time(NULL); + last_dummy_tick = now; } RsTickEvent::tick_events(); GxsTokenQueue::checkRequests(); mCommentService->comment_tick(); + + // Notify distant search results, not more than once per sec. Normally we should + // rather send one item for all, but that needs another class type + + if(now > mLastDistantSearchNotificationTS+2 && !mSearchResultsToNotify.empty()) + { + auto ev = std::make_shared(); + ev->mSearchResultsMap = mSearchResultsToNotify; + + mLastDistantSearchNotificationTS = now; + mSearchResultsToNotify.clear(); + + rsEvents->postEvent(ev); + } } bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector &groups) @@ -2210,7 +2216,9 @@ void p3GxsChannels::dummy_tick() } +#ifdef TO_REMOVE cleanTimedOutCallbacks(); +#endif } @@ -2389,18 +2397,18 @@ bool p3GxsChannels::clearDistantSearchResults(TurtleRequestId req) { return netService()->clearDistantSearchResults(req); } -bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map& results) +bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map& results) { return netService()->retrieveDistantSearchResults(req,results); } bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) { - RsGxsGroupSummary gs; + RsGxsGroupSearchResults gs; if(netService()->retrieveDistantGroupSummary(group_id,gs)) { - // This is a placeholder information by the time we receive the full group meta data. + // This is a placeholder information by the time we receive the full group meta data and check the signature. distant_group.mMeta.mGroupId = gs.mGroupId ; distant_group.mMeta.mGroupName = gs.mGroupName; distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ; @@ -2426,6 +2434,7 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann return false ; } +#ifdef TO_REMOVE bool p3GxsChannels::turtleSearchRequest( const std::string& matchString, const std::function& multiCallback, @@ -2505,17 +2514,24 @@ bool p3GxsChannels::localSearchRequest( return true; } +#endif -void p3GxsChannels::receiveDistantSearchResults( - TurtleRequestId id, const RsGxsGroupId& grpId ) +void p3GxsChannels::receiveDistantSearchResults( TurtleRequestId id, const RsGxsGroupId& grpId ) { - std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")" - << std::endl; + if(!rsEvents) + return; + + // We temporise here, in order to avoid notifying clients with many events + // So we put some data in there and will send an event with all of them at once every 1 sec at most. + + mSearchResultsToNotify[id].insert(grpId); + +#ifdef TO_REMOVE + std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")" << std::endl; { RsGenExchange::receiveDistantSearchResults(id, grpId); - RsGxsGroupSummary gs; - gs.mGroupId = grpId; + RsGxsGroupSearchResults gs; netService()->retrieveDistantGroupSummary(grpId, gs); { @@ -2556,8 +2572,10 @@ void p3GxsChannels::receiveDistantSearchResults( return; } } // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex); +#endif } +#ifdef TO_REMOVE void p3GxsChannels::cleanTimedOutCallbacks() { auto now = std::chrono::system_clock::now(); @@ -2586,6 +2604,7 @@ void p3GxsChannels::cleanTimedOutCallbacks() else ++cbpt; } // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex) } +#endif bool p3GxsChannels::exportChannelLink( std::string& link, const RsGxsGroupId& chanId, bool includeGxsData, diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index e6917e252..12c18d5c4 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -68,7 +68,7 @@ protected: virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id); virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) ; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) ; virtual bool clearDistantSearchResults(TurtleRequestId req); virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group); @@ -109,6 +109,7 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled); virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory); virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory); +#ifdef TO_REMOVE /// @see RsGxsChannels::turtleSearchRequest virtual bool turtleSearchRequest(const std::string& matchString, const std::function& multiCallback, @@ -124,6 +125,7 @@ virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::strin virtual bool localSearchRequest(const std::string& matchString, const std::function& multiCallback, rstime_t maxWait = 30 ) override; +#endif /** * Receive results from turtle search @see RsGenExchange @see RsNxsObserver @@ -374,6 +376,9 @@ bool generateGroup(uint32_t &token, std::string groupName); std::map mKnownChannels; RsMutex mKnownChannelsMutex; + rstime_t mLastDistantSearchNotificationTS; + std::map > mSearchResultsToNotify; +#ifdef TO_REMOVE /** Store search callbacks with timeout*/ std::map< TurtleRequestId, @@ -394,4 +399,5 @@ bool generateGroup(uint32_t &token, std::string groupName); /// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap void cleanTimedOutCallbacks(); +#endif }; diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index fa12ebf54..799331e57 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -463,6 +463,12 @@ void GroupTreeWidget::fillGroupItems(QTreeWidgetItem *categoryItem, const QList< item->setData(COLUMN_DATA, ROLE_NAME, itemInfo.name); item->setData(COLUMN_DATA, ROLE_DESCRIPTION, itemInfo.description); + // Add children for context strings. This happens in the search. + while(nullptr != item->takeChild(0)); + + for(auto str:itemInfo.context_strings) + item->addChild(new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str())))); + /* Set last post */ qlonglong lastPost = itemInfo.lastpost.toTime_t(); item->setData(COLUMN_DATA, ROLE_LASTPOST, -lastPost); // negative for correct sorting diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index 23c90a292..9f8d0ccd4 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -21,6 +21,8 @@ #ifndef GROUPTREEWIDGET_H #define GROUPTREEWIDGET_H +#include + #include #include @@ -47,16 +49,17 @@ public: {} public: - QString id; - QString name; - QString description; - int popularity; - QDateTime lastpost; - QIcon icon; - bool publishKey; - bool adminKey; - quint32 subscribeFlags; - quint32 max_visible_posts ; + QString id; + QString name; + QString description; + int popularity; + QDateTime lastpost; + QIcon icon; + bool publishKey; + bool adminKey; + quint32 subscribeFlags; + quint32 max_visible_posts ; + std::set context_strings; }; //cppcheck-suppress noConstructor diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 1245c80b2..59a8bbfd9 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -286,55 +286,48 @@ void GxsGroupFrameDialog::updateDisplay(bool complete) if(complete) // || !getGrpIds().empty() || !getGrpIdsMeta().empty()) { updateGroupSummary(); /* Update group list */ - updateSearchResults() ; +// updateSearchResults() ; } void GxsGroupFrameDialog::updateSearchResults() { - const std::set& reqs = getSearchRequests(); + for(auto& it:mSearchGroupsItems) + updateSearchResults(it.first); +} - for(auto it(reqs.begin());it!=reqs.end();++it) - { - std::cerr << "updating search ID " << std::hex << *it << std::dec << std::endl; +void GxsGroupFrameDialog::updateSearchResults(const TurtleRequestId& sid) +{ + std::cerr << "updating search ID " << std::hex << sid << std::dec << std::endl; - std::map group_infos; + std::map group_infos; - getDistantSearchResults(*it,group_infos) ; + getDistantSearchResults(sid,group_infos) ; - std::cerr << "retrieved " << std::endl; + std::cerr << "retrieved " << std::endl; - auto it2 = mSearchGroupsItems.find(*it); + auto it2 = mSearchGroupsItems.find(sid); - if(mSearchGroupsItems.end() == it2) - { - std::cerr << "GxsGroupFrameDialog::updateSearchResults(): received result notification for req " << std::hex << *it << std::dec << " but no item present!" << std::endl; - continue ; // we could create the item just as well but since this situation is not supposed to happen, I prefer to make this a failure case. - } + QList group_items ; - QList group_items ; + for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) + { + std::cerr << " adding group " << it3->first << " " << it3->second.mGroupId << " \"" << it3->second.mGroupName << "\"" << std::endl; - for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) - if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end()) - { - std::cerr << " adding new group " << it3->first << " " - << it3->second.mGroupId << " \"" - << it3->second.mGroupName << "\"" << std::endl; + GroupItemInfo i; + i.id = QString(it3->second.mGroupId.toStdString().c_str()); + i.name = QString::fromUtf8(it3->second.mGroupName.c_str()); + i.popularity = 0; // could be set to the number of hits + i.lastpost = QDateTime::fromTime_t(it3->second.mLastMessageTs); + i.subscribeFlags = 0; // irrelevant here + i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags); + i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags); + i.max_visible_posts = it3->second.mNumberOfMessages; + i.context_strings = it3->second.mSearchContexts; - GroupItemInfo i; - i.id = QString(it3->second.mGroupId.toStdString().c_str()); - i.name = QString::fromUtf8(it3->second.mGroupName.c_str()); - i.popularity = 0; // could be set to the number of hits - i.lastpost = QDateTime::fromTime_t(it3->second.mLastMessageTs); - i.subscribeFlags = 0; // irrelevant here - i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags); - i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags); - i.max_visible_posts = it3->second.mNumberOfMessages; + group_items.push_back(i); + } - group_items.push_back(i); - } - - ui->groupTreeWidget->fillGroupItems(it2->second, group_items); - } + ui->groupTreeWidget->fillGroupItems(it2->second, group_items); } void GxsGroupFrameDialog::todo() diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 902e49087..b335fc57b 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -161,7 +161,7 @@ private: virtual void groupTreeCustomActions(RsGxsGroupId /*grpId*/, int /*subscribeFlags*/, QList &/*actions*/) {} virtual RsGxsCommentService *getCommentService() { return NULL; } virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; } - virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map& /* group_infos */){ return false ;} + virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map& /* group_infos */){ return false ;} void initUi(); @@ -187,7 +187,8 @@ private: GxsCommentDialog *commentWidget(const RsGxsMessageId &msgId); protected: - void updateSearchResults(); + void updateSearchResults(const TurtleRequestId &sid); + void updateSearchResults(); // update all searches bool mCountChildMsgs; // Count unread child messages? diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 8e25e4ea0..ce58177c0 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -61,9 +61,7 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr ev { const RsGxsChannelEvent *e = dynamic_cast(event.get()); - if(!e) - return; - + if(e) switch(e->mChannelEventCode) { case RsChannelEventCode::NEW_MESSAGE: // [[fallthrough]]; @@ -72,11 +70,6 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr ev updateGroupStatisticsReal(e->mChannelGroupId); // update the list immediately break; - case RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT: - mSearchResults.insert(e->mDistantSearchRequestId); - updateSearchResults(); - break; - case RsChannelEventCode::NEW_CHANNEL: // [[fallthrough]]; case RsChannelEventCode::SUBSCRIBE_STATUS_CHANGED: updateDisplay(true); @@ -89,6 +82,13 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr ev default: break; } + + + const RsGxsChannelSearchResultEvent*f = dynamic_cast(event.get()); + + if(nullptr != f) + for(auto it:f->mSearchResultsMap) + updateSearchResults(it.first); } GxsChannelDialog::~GxsChannelDialog() @@ -401,7 +401,7 @@ TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; } -bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map& group_infos) +bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map& group_infos) { return rsGxsChannels->retrieveDistantSearchResults(id,group_infos); } diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index b0b46205d..e8c23424d 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -43,7 +43,7 @@ public: protected: /* GxsGroupFrameDialog */ - virtual bool getDistantSearchResults(TurtleRequestId id, std::map& group_infos); + virtual bool getDistantSearchResults(TurtleRequestId id, std::map &group_infos); virtual TurtleRequestId distantSearch(const QString& search_string) ; virtual void checkRequestGroup(const RsGxsGroupId& grpId) ; From 9f5e7f20812504a49f0727ade65804d877899529 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 16 Jun 2020 21:29:13 +0200 Subject: [PATCH 04/11] removed debug info --- libretroshare/src/gxs/rsgxsnetservice.cc | 12 +++++++++++- libretroshare/src/gxs/rsgxsnettunnel.cc | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 58a49f35c..743cd298a 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -272,6 +272,7 @@ NXS_NET_DEBUG_6 group sync statistics (e.g. number of posts at nighbour nodes, etc) NXS_NET_DEBUG_7 encryption/decryption of transactions NXS_NET_DEBUG_8 gxs distant sync + NXS_NET_DEBUG_9 gxs distant search ***/ //#define NXS_NET_DEBUG_0 1 @@ -283,6 +284,7 @@ //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 //#define NXS_NET_DEBUG_8 1 +//#define NXS_NET_DEBUG_9 1 //#define NXS_FRAG @@ -318,7 +320,7 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ; #if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) \ || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7) \ - || defined(NXS_NET_DEBUG_8) + || defined(NXS_NET_DEBUG_8) || defined(NXS_NET_DEBUG_9) static const RsPeerId peer_to_print = RsPeerId();//std::string("a97fef0e2dc82ddb19200fb30f9ac575")) ; static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("66052380f5d1d0c5992e2b55dc402ce6")) ; // use this to allow to this group id only, or "" for all IDs @@ -5199,7 +5201,9 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std RsGxsGrpMetaTemporaryMap grpMeta; std::map& search_results_map(mDistantSearchResults[req]); +#ifdef NXS_NET_DEBUG_9 std::cerr << "Received group summary through turtle search for the following groups:" << std::endl; +#endif for(const RsGxsGroupSummary& gps : group_infos) { @@ -5209,9 +5213,11 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std mDataStore->retrieveGxsGrpMetaData(grpMeta); +#ifdef NXS_NET_DEBUG_9 std::cerr << "Retrieved data store group data for the following groups:" <mGroupName << std::endl; +#endif for (const RsGxsGroupSummary& gps : group_infos) { @@ -5226,7 +5232,9 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue; +#ifdef NXS_NET_DEBUG_9 std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl; +#endif #else // ndef RS_DEEP_CHANNEL_INDEX /* When deep search is enabled search results may bring more info @@ -5239,7 +5247,9 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std // Find search results place for this particular group +#ifdef NXS_NET_DEBUG_9 std::cerr << " Adding gps=" << gps.mGroupId << " name=\"" << gps.mGroupName << "\" gps.mSearchContext=\"" << gps.mSearchContext << "\"" << std::endl; +#endif RsGxsGroupSearchResults& eGpS(search_results_map[grpId]); if(eGpS.mGroupId != grpId) // not initialized yet. So we do it now. diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index a58d20f5f..eabf21e7e 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -1090,8 +1090,10 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id { GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; +#ifdef DEBUG_RSGXSNETTUNNEL for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl; +#endif auto it = mSearchableServices.find(result_gs->service) ; From ba486a0029c10ae47e247b55cb00253b8d4eae2d Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 25 Jun 2020 21:15:37 +0200 Subject: [PATCH 05/11] multiple improvements in distant channel search and general display of distant search results --- libretroshare/src/retroshare/rsgxschannels.h | 2 +- libretroshare/src/services/p3gxschannels.cc | 2 +- libretroshare/src/services/p3gxschannels.h | 2 +- .../src/gui/common/GroupTreeWidget.cpp | 3 ++- .../src/gui/gxs/GxsGroupFrameDialog.cpp | 21 +++++++++++++------ .../src/gui/gxs/GxsGroupFrameDialog.h | 1 + .../src/gui/gxschannels/GxsChannelDialog.cpp | 12 ++++++++++- .../src/gui/gxschannels/GxsChannelDialog.h | 7 ++++--- .../gui/gxschannels/GxsChannelPostsWidget.cpp | 5 +++-- .../GxsChannelPostsWidgetWithModel.cpp | 17 +++++++-------- .../GxsChannelPostsWidgetWithModel.ui | 2 +- 11 files changed, 47 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index c03f22970..8fd02a3b1 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -717,7 +717,7 @@ public: virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; virtual bool clearDistantSearchResults(TurtleRequestId req)=0; - virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; + virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; ////////////////////////////////////////////////////////////////////////////// diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 01ddb4357..bdb1f9981 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -2402,7 +2402,7 @@ bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::mapretrieveDistantSearchResults(req,results); } -bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) +bool p3GxsChannels::getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) { RsGxsGroupSearchResults gs; diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 12c18d5c4..da274690b 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -70,7 +70,7 @@ protected: virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) ; virtual bool clearDistantSearchResults(TurtleRequestId req); - virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group); + virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group); // Overloaded to cache new groups. virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index 799331e57..c67dfbc20 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -467,7 +467,8 @@ void GroupTreeWidget::fillGroupItems(QTreeWidgetItem *categoryItem, const QList< while(nullptr != item->takeChild(0)); for(auto str:itemInfo.context_strings) - item->addChild(new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str())))); + if(!str.empty()) + item->addChild(new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str())))); /* Set last post */ qlonglong lastPost = itemInfo.lastpost.toTime_t(); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 7d5ac1f7c..49b91866d 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -316,6 +316,8 @@ void GxsGroupFrameDialog::updateSearchResults(const TurtleRequestId& sid) for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) { std::cerr << " adding group " << it3->first << " " << it3->second.mGroupId << " \"" << it3->second.mGroupName << "\"" << std::endl; + for(auto s:it3->second.mSearchContexts) + std::cerr << " Context string \"" << s << "\"" << std::endl; GroupItemInfo i; i.id = QString(it3->second.mGroupId.toStdString().c_str()); @@ -1067,15 +1069,20 @@ void GxsGroupFrameDialog::updateGroupSummary() { RsThread::async([this]() { - std::list groupInfo; + auto groupInfo = new std::list() ; - if(!getGroupData(groupInfo)) + if(!getGroupData(*groupInfo)) { - std::cerr << __PRETTY_FUNCTION__ << " failed to collect group info " << std::endl; + std::cerr << __PRETTY_FUNCTION__ << " failed to collect group info." << std::endl; + delete groupInfo; return; } - if(groupInfo.empty()) + if(groupInfo->empty()) + { + std::cerr << __PRETTY_FUNCTION__ << " no group info collected." << std::endl; + delete groupInfo; return; + } RsQThreadUtils::postToObject( [this,groupInfo]() { @@ -1085,7 +1092,7 @@ void GxsGroupFrameDialog::updateGroupSummary() * Qt::QueuedConnection is important! */ - insertGroupsData(groupInfo); + insertGroupsData(*groupInfo); updateSearchResults(); mStateHelper->setLoading(TOKEN_TYPE_GROUP_SUMMARY, false); @@ -1104,12 +1111,14 @@ void GxsGroupFrameDialog::updateGroupSummary() // now delete the data that is not used anymore - for(auto& g:groupInfo) + for(auto& g:*groupInfo) { mCachedGroupMetas[g->mMeta.mGroupId] = g->mMeta; delete g; } + delete groupInfo; + }, this ); }); } diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 5c3385d52..05b311b04 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -162,6 +162,7 @@ private: virtual RsGxsCommentService *getCommentService() { return NULL; } virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; } virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map& /* group_infos */){ return false ;} + virtual RsGxsGenericGroupData *getDistantSearchResultGroupData(const RsGxsGroupId& group_id){ return nullptr ;} void initUi(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 8af68b01e..8cbeccfbc 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -406,11 +406,21 @@ bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::mapretrieveDistantSearchResults(id,group_infos); } +RsGxsGenericGroupData *GxsChannelDialog::getDistantSearchResultGroupData(const RsGxsGroupId& group_id) +{ + RsGxsChannelGroup channel_group; + + if(rsGxsChannels->getDistantSearchResultGroupData(group_id,channel_group)) + return new RsGxsGenericGroupData(channel_group); + else + return nullptr; +} + void GxsChannelDialog::checkRequestGroup(const RsGxsGroupId& grpId) { RsGxsChannelGroup distant_group; - if( rsGxsChannels->retrieveDistantGroup(grpId,distant_group)) // normally we should also check that the group meta is not already here. + if( rsGxsChannels->getDistantSearchResultGroupData(grpId,distant_group)) // normally we should also check that the group meta is not already here. { std::cerr << "GxsChannelDialog::checkRequestGroup() sending turtle request for group data for group " << grpId << std::endl; rsGxsChannels->turtleGroupRequest(grpId); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index e8c23424d..7902b7361 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -43,10 +43,11 @@ public: protected: /* GxsGroupFrameDialog */ - virtual bool getDistantSearchResults(TurtleRequestId id, std::map &group_infos); + virtual bool getDistantSearchResults(TurtleRequestId id, std::map &group_infos) override; + virtual RsGxsGenericGroupData *getDistantSearchResultGroupData(const RsGxsGroupId& group_id) override; - virtual TurtleRequestId distantSearch(const QString& search_string) ; - virtual void checkRequestGroup(const RsGxsGroupId& grpId) ; + virtual TurtleRequestId distantSearch(const QString& search_string) override; + virtual void checkRequestGroup(const RsGxsGroupId& grpId) override ; // Implementation of some abstract methods in GxsGroupFrameDialog diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp index 5d6974442..bbbb26a42 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp @@ -858,11 +858,12 @@ bool GxsChannelPostsWidget::getGroupData(RsGxsGenericGroupData *& data) { RsGxsChannelGroup distant_group; - if(rsGxsChannels->retrieveDistantGroup(groupId(),distant_group)) + if(rsGxsChannels->getDistantSearchResultGroupData(groupId(),distant_group)) { insertChannelDetails(distant_group); - data = new RsGxsChannelGroup(distant_group); + data = new RsGxsChannelGroup(distant_group); mGroup = distant_group; // make a local copy to pass on to items + return true ; } } diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp index 73508c1e8..6843a80f0 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp @@ -538,23 +538,20 @@ void GxsChannelPostsWidgetWithModel::updateGroupData() RsThread::async([this]() { + RsGxsChannelGroup group; std::vector groups; - if(!rsGxsChannels->getChannelsInfo(std::list{ groupId() }, groups)) + if(rsGxsChannels->getChannelsInfo(std::list{ groupId() }, groups) && groups.size()==1) + group = groups[0]; + else if(!rsGxsChannels->getDistantSearchResultGroupData(groupId(),group)) { - std::cerr << __PRETTY_FUNCTION__ << " failed to get autodownload value for channel: " << groupId() << std::endl; + std::cerr << __PRETTY_FUNCTION__ << " failed to get group data for channel: " << groupId() << std::endl; return; } - if(groups.size() != 1) - { - RsErr() << __PRETTY_FUNCTION__ << " cannot retrieve channel data for group ID " << groupId() << ": ERROR." << std::endl; - return; - } - - RsQThreadUtils::postToObject( [this,groups]() + RsQThreadUtils::postToObject( [this,group]() { - mGroup = groups[0]; + mGroup = group; mChannelPostsModel->updateChannel(groupId()); insertChannelDetails(mGroup); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.ui b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.ui index 1f51702a2..2886e8dbe 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.ui +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.ui @@ -161,7 +161,7 @@ - 1 + 0 From c58aa688d164263e070a41fea3d1a4d4192336b7 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 25 Jun 2020 21:25:22 +0200 Subject: [PATCH 06/11] fixed proper display of group for context strings --- retroshare-gui/src/gui/common/GroupTreeWidget.cpp | 6 +++++- .../src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index c67dfbc20..2fa8de0c0 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -468,7 +468,11 @@ void GroupTreeWidget::fillGroupItems(QTreeWidgetItem *categoryItem, const QList< for(auto str:itemInfo.context_strings) if(!str.empty()) - item->addChild(new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str())))); + { + QTreeWidgetItem *it = new QTreeWidgetItem(QStringList(QString::fromUtf8(str.c_str()))); + it->setData(COLUMN_DATA,ROLE_ID,itemInfo.id); + item->addChild(it); + } /* Set last post */ qlonglong lastPost = itemInfo.lastpost.toTime_t(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp index 6843a80f0..cc60e003b 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidgetWithModel.cpp @@ -59,6 +59,7 @@ static const int mTokenTypeGroupData = 1; static const int CHANNEL_TABS_DETAILS= 0; static const int CHANNEL_TABS_POSTS = 1; +static const int CHANNEL_TABS_FILES = 2; /* View mode */ #define VIEW_MODE_FEEDS 1 @@ -776,12 +777,14 @@ void GxsChannelPostsWidgetWithModel::insertChannelDetails(const RsGxsChannelGrou //ui->feedToolButton->setEnabled(true); //ui->fileToolButton->setEnabled(true); ui->channel_TW->setTabEnabled(CHANNEL_TABS_POSTS,true); + ui->channel_TW->setTabEnabled(CHANNEL_TABS_FILES,true); ui->details_TW->setEnabled(true); } else { ui->details_TW->setEnabled(false); ui->channel_TW->setTabEnabled(CHANNEL_TABS_POSTS,false); + ui->channel_TW->setTabEnabled(CHANNEL_TABS_FILES,false); } From 9133adbde89975daf8bb264d07a676291b8a0975 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 29 Jun 2020 21:03:17 +0200 Subject: [PATCH 07/11] added anti-clogging strategy in GXS. To be tested. --- libretroshare/src/gxs/rsgxsnetservice.cc | 48 ++++++++++++++++---- libretroshare/src/gxs/rsgxsnetservice.h | 1 + libretroshare/src/rsitems/rsgxsupdateitems.h | 18 +++++++- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 72a078075..48b12caa5 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -274,7 +274,7 @@ NXS_NET_DEBUG_8 gxs distant sync ***/ -//#define NXS_NET_DEBUG_0 1 +#define NXS_NET_DEBUG_0 1 //#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 @@ -306,6 +306,7 @@ static const uint32_t GROUP_STATS_UPDATE_DELAY = 240; // static const uint32_t GROUP_STATS_UPDATE_NB_PEERS = 2; // number of peers to which the group stats are asked static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE = 199000; // 200,000 bytes including signature and headers static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH = 40; // dont search same group more than every 40 secs. +static const uint32_t SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE = 1800; // avoid re-sending the same msg list to a peer who asks twice for the same update in less than this time static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN = 0x00 ; static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR = 0x01 ; @@ -3993,7 +3994,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item) GXSNETDEBUG_P_(item->PeerId()) << " local modification time stamp: " << std::dec<< time(NULL) - mGrpServerUpdate.grpUpdateTS << " secs ago. Update sent: " << ((item->updateTS < mGrpServerUpdate.grpUpdateTS)?"YES":"NO") << std::endl; #endif - return item->updateTS < mGrpServerUpdate.grpUpdateTS; + return item->updateTS < mGrpServerUpdate.grpUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),RsGxsGroupId(),item->updateTS,mGrpServerUpdate.grpUpdateTsRecords[item->PeerId()]) ; } void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrpReqItem *item) @@ -4238,10 +4239,33 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs return true; } +bool RsGxsNetService::locked_checkResendingOfUpdates(const RsPeerId& pid,const RsGxsGroupId& grpId,rstime_t incoming_ts,RsPeerUpdateTsRecord& rec) +{ + rstime_t now = time(NULL); + + // Now we check if the peer is sending the same outdated TS for the same time in a short while. This would mean the peer + // hasn't finished processing the updates we're sending and we shouldn't send new data anymore. Of course the peer might + // have disconnected or so, which means that we need to be careful about not sending. As a compromise we still send, but + // after waiting for a while (See + + if(rec.mLastTsReceived == incoming_ts && rec.mTs + SAFETY_DELAY_FOR_UNSUCCESSFUL_UPDATE > now) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(pid,grpId) << "(II) peer " << pid << " already sent the same TS " << (long int)now-(long int)rec.mTs << " secs ago for that group ID. Will not send msg list again for a while to prevent clogging..." << std::endl; +#endif + return false; + } + + rec.mLastTsReceived = incoming_ts; + rec.mTs = now; + + return true; +} + bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& grp_is_known) { // Do we have new updates for this peer? - // Here we compare times in the same clock: the friend's clock, so it should be fine. + // Here we compare times in the same clock: our own clock, so it should be fine. grp_is_known = false ; @@ -4250,7 +4274,7 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr // Item contains the hashed group ID in order to protect is from friends who don't know it. So we de-hash it using bruteforce over known group IDs for this peer. // We could save the de-hash result. But the cost is quite light, since the number of encrypted groups per service is usually low. - for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it) + for(ServerMsgMap::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it) if(item->grpId == hashGrpId(it->first,item->PeerId())) { item->grpId = it->first ; @@ -4260,20 +4284,25 @@ bool RsGxsNetService::locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item,bool& gr #endif grp_is_known = true ; - return item->updateTS < it->second.msgUpdateTS ; + // The order of tests below is important because we want to only modify the map of requests records if the request actually is a valid requests instead of + // a simple check that nothing's changed. + + return item->updateTS < it->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,it->second.msgUpdateTsRecords[item->PeerId()]) ; } return false ; } - ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); + ServerMsgMap::iterator cit = mServerMsgUpdateMap.find(item->grpId); + if(cit != mServerMsgUpdateMap.end()) { #ifdef NXS_NET_DEBUG_0 GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " local time stamp: " << std::dec<< time(NULL) - cit->second.msgUpdateTS << " secs ago. Update sent: " << (item->updateTS < cit->second.msgUpdateTS) << std::endl; #endif grp_is_known = true ; - return item->updateTS < cit->second.msgUpdateTS ; + + return item->updateTS < cit->second.msgUpdateTS && locked_checkResendingOfUpdates(item->PeerId(),item->grpId,item->updateTS,cit->second.msgUpdateTsRecords[item->PeerId()]) ; } #ifdef NXS_NET_DEBUG_0 @@ -4294,6 +4323,9 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ bool grp_is_known = false; bool was_circle_protected = item_was_encrypted || bool(item->flag & RsNxsSyncMsgReqItem::FLAG_USE_HASHED_GROUP_ID); + // This call determines if the peer can receive updates from us, meaning that our last TS is larger than what the peer sent. + // It also changes the items' group id into the un-hashed group ID if the group is a distant group. + bool peer_can_receive_update = locked_CanReceiveUpdate(item, grp_is_known); if(item_was_encrypted) @@ -4309,7 +4341,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ // We update suppliers in two cases: // Case 1: the grp is known because it is the hash of an existing group, but it's not yet in the server config map - // Case 2: the gtp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only + // Case 2: the grp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only // gathered from known suppliers. So statistics never add new suppliers. These are only added here. if(grp_is_known || mServerGrpConfigMap.find(item->grpId)!=mServerGrpConfigMap.end()) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index e4056bb06..f7abdc176 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -439,6 +439,7 @@ private: bool locked_CanReceiveUpdate(const RsNxsSyncGrpReqItem *item); bool locked_CanReceiveUpdate(RsNxsSyncMsgReqItem *item, bool &grp_is_known); void locked_resetClientTS(const RsGxsGroupId& grpId); + bool locked_checkResendingOfUpdates(const RsPeerId& pid, const RsGxsGroupId &grpId, rstime_t incoming_ts, RsPeerUpdateTsRecord& rec); static RsGxsGroupId hashGrpId(const RsGxsGroupId& gid,const RsPeerId& pid) ; diff --git a/libretroshare/src/rsitems/rsgxsupdateitems.h b/libretroshare/src/rsitems/rsgxsupdateitems.h index 9e7a194ac..03bd21800 100644 --- a/libretroshare/src/rsitems/rsgxsupdateitems.h +++ b/libretroshare/src/rsitems/rsgxsupdateitems.h @@ -106,12 +106,22 @@ public: RsPeerId peerID; }; +struct RsPeerUpdateTsRecord +{ + RsPeerUpdateTsRecord() : mLastTsReceived(0), mTs(0) {} + + rstime_t mLastTsReceived; // last TS that was sent for this group by this peer ID. + rstime_t mTs; // time at which this TS was sent. +}; + class RsGxsServerGrpUpdate { public: RsGxsServerGrpUpdate() { grpUpdateTS = 0 ; } uint32_t grpUpdateTS; + + std::map grpUpdateTsRecords; }; class RsGxsServerGrpUpdateItem : public RsGxsNetServiceItem, public RsGxsServerGrpUpdate @@ -168,7 +178,13 @@ class RsGxsServerMsgUpdate public: RsGxsServerMsgUpdate() { msgUpdateTS = 0 ;} - uint32_t msgUpdateTS; // local time stamp this group last received a new msg + uint32_t msgUpdateTS; // local time stamp at which this group last received a new msg + + // Now we also store for each peer the last own TS the peer sent and when it did so. This allows to detect when transactions are stuck because of + // outqueues clogging. If that happens, we receive multiple times the same TS from the friend, in which case we do not send the list of msgs + // again until a significant amount of time has passed. These values are obviously initialized to 0. + + std::map msgUpdateTsRecords; }; class RsGxsServerMsgUpdateItem : public RsGxsNetServiceItem, public RsGxsServerMsgUpdate From 8d40224c37a26029f93445f85d3fdd35cc02215e Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 30 Jun 2020 18:10:28 +0200 Subject: [PATCH 08/11] fixed sizes in forums and removed duplicate text on distribution column --- .../src/gui/gxsforums/GxsForumModel.cpp | 8 +++++--- .../gui/gxsforums/GxsForumThreadWidget.cpp | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumModel.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumModel.cpp index f72649dfc..e4d6f13e9 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumModel.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumModel.cpp @@ -669,8 +669,11 @@ QVariant RsGxsForumModel::displayRole(const ForumModelPostEntry& fmpe,int col) c return QVariant(DateTime::formatDateTime(qtime)); } - case COLUMN_THREAD_DISTRIBUTION: - case COLUMN_THREAD_AUTHOR:{ + case COLUMN_THREAD_DISTRIBUTION: // passthrough // handled by delegate. + case COLUMN_THREAD_MSGID: + return QVariant(); + case COLUMN_THREAD_AUTHOR: + { QString name; RsGxsId id = RsGxsId(fmpe.mAuthorId.toStdString()); @@ -680,7 +683,6 @@ QVariant RsGxsForumModel::displayRole(const ForumModelPostEntry& fmpe,int col) c return name; return QVariant(tr("[Unknown]")); } - case COLUMN_THREAD_MSGID: return QVariant(); #ifdef TODO if (filterColumn == COLUMN_THREAD_CONTENT) { // need content for filter diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 29cfa5b44..fd9c591ca 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -134,6 +134,13 @@ public: const QPoint p = QPoint((r.width() - pix.width())/2, (r.height() - pix.height())/2); painter->drawPixmap(r.topLeft() + p, pix); } + + virtual QSize sizeHint(const QStyleOptionViewItem& option, const QModelIndex& index) const override + { + static auto img(FilesDefs::getPixmapFromQtResourcePath(IMAGE_WARNING_YELLOW)); + + return QSize(img.width()*1.2,option.rect.height()); + } }; class ReadStatusItemDelegate: public QStyledItemDelegate @@ -183,6 +190,13 @@ public: const QPoint p = QPoint((r.width() - pix.width())/2, (r.height() - pix.height())/2); painter->drawPixmap(r.topLeft() + p, pix); } + + virtual QSize sizeHint(const QStyleOptionViewItem& option, const QModelIndex& index) const override + { + static auto img(FilesDefs::getPixmapFromQtResourcePath(":/images/message-state-unread.png")); + + return QSize(img.width()*1.2,option.rect.height()); + } }; class ForumPostSortFilterProxyModel: public QSortFilterProxyModel @@ -299,9 +313,10 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget QHeaderView * ttheader = ui->threadTreeWidget->header () ; ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_DATE, 140*f); ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_TITLE, 440*f); - ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_DISTRIBUTION, 24*f); ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_AUTHOR, 150*f); - ttheader->resizeSection (RsGxsForumModel::COLUMN_THREAD_READ, 24*f); + + ui->threadTreeWidget->resizeColumnToContents(RsGxsForumModel::COLUMN_THREAD_DISTRIBUTION); + ui->threadTreeWidget->resizeColumnToContents(RsGxsForumModel::COLUMN_THREAD_READ); QHeaderView_setSectionResizeModeColumn(ttheader, RsGxsForumModel::COLUMN_THREAD_TITLE, QHeaderView::Interactive); QHeaderView_setSectionResizeModeColumn(ttheader, RsGxsForumModel::COLUMN_THREAD_DATE, QHeaderView::Interactive); From 6510af7ab6f44fd8341296521fe0e499f6576a75 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 3 Jul 2020 18:07:07 +0200 Subject: [PATCH 09/11] Enable JSON API for channel turtle search Add event subtype to RsGxsChannelSearchResultEvent so its type can be recognized from JSON API Remove unused member remnant of the past from RsGxsChannelEvent --- libretroshare/src/retroshare/rsgxschannels.h | 128 +++++++++---------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 8fd02a3b1..106147dc6 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -122,7 +122,6 @@ struct RsGxsChannelEvent: RsEvent RsChannelEventCode mChannelEventCode; RsGxsGroupId mChannelGroupId; RsGxsMessageId mChannelMsgId; - TurtleRequestId mDistantSearchRequestId; ///* @see RsEvent @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override @@ -132,25 +131,28 @@ struct RsGxsChannelEvent: RsEvent RS_SERIAL_PROCESS(mChannelEventCode); RS_SERIAL_PROCESS(mChannelGroupId); RS_SERIAL_PROCESS(mChannelMsgId); - RS_SERIAL_PROCESS(mDistantSearchRequestId); - } + } }; // This event is used to factor multiple search results notifications in a single event. -struct RsGxsChannelSearchResultEvent: public RsEvent +struct RsGxsChannelSearchResultEvent: RsEvent { - RsGxsChannelSearchResultEvent() : RsEvent(RsEventType::GXS_CHANNELS) {} + RsGxsChannelSearchResultEvent(): + RsEvent(RsEventType::GXS_CHANNELS), + mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {} - std::map > mSearchResultsMap; + RsChannelEventCode mChannelEventCode; + std::map > mSearchResultsMap; ///* @see RsEvent @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override { RsEvent::serial_process(j, ctx); + RS_SERIAL_PROCESS(mChannelEventCode); RS_SERIAL_PROCESS(mSearchResultsMap); - } + } }; class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService @@ -422,49 +424,6 @@ public: */ virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0; -#ifdef TO_REMOVE - /** - * @brief Request remote channels search - * @jsonapi{development} - * @param[in] matchString string to look for in the search - * @param multiCallback function that will be called each time a search - * result is received - * @param[in] maxWait maximum wait time in seconds for search results - * @return false on error, true otherwise - */ - virtual bool turtleSearchRequest( - const std::string& matchString, - const std::function& multiCallback, - rstime_t maxWait = 300 ) = 0; - - /** - * @brief Request remote channel - * @jsonapi{development} - * @param[in] channelId id of the channel to request to distants peers - * @param multiCallback function that will be called each time a result is - * received - * @param[in] maxWait maximum wait time in seconds for search results - * @return false on error, true otherwise - */ - virtual bool turtleChannelRequest( - const RsGxsGroupId& channelId, - const std::function& multiCallback, - rstime_t maxWait = 300 ) = 0; - - /** - * @brief Search local channels - * @jsonapi{development} - * @param[in] matchString string to look for in the search - * @param multiCallback function that will be called for each result - * @param[in] maxWait maximum wait time in seconds for search results - * @return false on error, true otherwise - */ - virtual bool localSearchRequest( - const std::string& matchString, - const std::function& multiCallback, - rstime_t maxWait = 30 ) = 0; -#endif - /// default base URL used for channels links @see exportChannelLink static const std::string DEFAULT_CHANNEL_BASE_URL; @@ -509,6 +468,7 @@ public: /** * @brief Import channel from full link + * @jsonapi{development} * @param[in] link channel link either in radix or link format * @param[out] chanId optional storage for parsed channel id * @param[out] errMsg optional storage for error message, meaningful only in @@ -520,7 +480,58 @@ public: RsGxsGroupId& chanId = RS_DEFAULT_STORAGE_PARAM(RsGxsGroupId), std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; + /** + * @brief Search the turtle reachable network for matching channels + * @jsonapi{development} + * An @see RsGxsChannelSearchResultEvent is emitted when matching channels + * arrives from the network + * @param[in] matchString string to search into the channels + * @return search id + */ + virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0; + /** + * @brief Retrieve available search results + * @jsonapi{development} + * @param[in] searchId search id + * @param[out] results storage for search results + * @return false on error, true otherwise + */ + virtual bool retrieveDistantSearchResults( + TurtleRequestId searchId, + std::map& results ) = 0; + + /** + * @brief Request distant channel details + * @jsonapi{development} + * An @see RsGxsChannelSearchResultEvent is emitted once details are + * retrieved from the network + * @param[in] groupId if of the group to request to the network + * @return search id + */ + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& groupId) = 0; + + /** + * @brief Retrieve previously requested distant group + * @jsonapi{development} + * @param[in] groupId if of teh group + * @param[out] distantGroup storage for group data + * @return false on error, true otherwise + */ + virtual bool getDistantSearchResultGroupData( + const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; + + /** + * @brief Clear accumulated search results + * @jsonapi{development} + * @param[in] reqId search id + * @return false on error, true otherwise + */ + virtual bool clearDistantSearchResults(TurtleRequestId reqId) = 0; + + ~RsGxsChannels() override; + + //////////////////////////////////////////////////////////////////////////// /* Following functions are deprecated and should not be considered a safe to * use API */ @@ -706,21 +717,4 @@ public: */ RS_DEPRECATED_FOR(editChannel) virtual bool updateGroup(uint32_t& token, RsGxsChannelGroup& group) = 0; - - ////////////////////////////////////////////////////////////////////////////// - /// Distant synchronisation methods /// - ////////////////////////////////////////////////////////////////////////////// - /// - /// - // This approach is much cleaner than the "multicallback" system. We should keep it and use rsEvents to warn when - // new results are received. - virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; - virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; - virtual bool clearDistantSearchResults(TurtleRequestId req)=0; - virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; - - ////////////////////////////////////////////////////////////////////////////// - - ~RsGxsChannels() override; }; From 4d872d5778d8db4670ee0da75ebc14bec414e657 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 3 Jul 2020 22:48:33 +0200 Subject: [PATCH 10/11] added missing cleaning of distant search results in GUI --- retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp | 10 +++++++++- retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h | 1 + .../src/gui/gxschannels/GxsChannelDialog.cpp | 5 +++++ retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 49b91866d..069b682ce 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -359,13 +359,21 @@ void GxsGroupFrameDialog::removeCurrentSearch() mSearchGroupsItems.erase(it); mKnownGroups.erase(search_request_id); + + clearDistantSearchResults(search_request_id); } void GxsGroupFrameDialog::removeAllSearches() { for(auto it(mSearchGroupsItems.begin());it!=mSearchGroupsItems.end();++it) - ui->groupTreeWidget->removeSearchItem(it->second) ; + { + TurtleRequestId search_request_id = 0 ; + if(ui->groupTreeWidget->isSearchRequestItem(point,search_request_id)) + clearDistantSearchResults(search_request_id); + + ui->groupTreeWidget->removeSearchItem(it->second) ; + } mSearchGroupsItems.clear(); mKnownGroups.clear(); } diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 05b311b04..006f593a4 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -162,6 +162,7 @@ private: virtual RsGxsCommentService *getCommentService() { return NULL; } virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; } virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map& /* group_infos */){ return false ;} + virtual void clearDistantSearchResults(TurtleRequestId /* id */) {} virtual RsGxsGenericGroupData *getDistantSearchResultGroupData(const RsGxsGroupId& group_id){ return nullptr ;} void initUi(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 8cbeccfbc..5e1c40fe4 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -396,6 +396,11 @@ void GxsChannelDialog::groupInfoToGroupItemInfo(const RsGxsGenericGroupData *gro groupItemInfo.description = QString::fromUtf8(channelGroupData->mDescription.c_str()); } +void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id) +{ + rsGxsChannels->clearDistantSearchResults(id); +} + TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) { return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index 7902b7361..63b6ba38a 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -48,6 +48,7 @@ protected: virtual TurtleRequestId distantSearch(const QString& search_string) override; virtual void checkRequestGroup(const RsGxsGroupId& grpId) override ; + virtual void clearDistantSearchResults(TurtleRequestId id) override; // Implementation of some abstract methods in GxsGroupFrameDialog From 07655e287ec1d8e09eb276ec586bf3b8b905729f Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 4 Jul 2020 00:05:29 +0200 Subject: [PATCH 11/11] fixed cleaning up of search results --- retroshare-gui/src/gui/common/GroupTreeWidget.cpp | 13 +++++++++++++ retroshare-gui/src/gui/common/GroupTreeWidget.h | 1 + retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp | 6 ++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index 2fa8de0c0..db9b487b4 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -397,6 +397,19 @@ bool GroupTreeWidget::isSearchRequestResult(QPoint &point,QString& group_id,uint return search_req_id > 0; } +bool GroupTreeWidget::isSearchRequestResultItem(QTreeWidgetItem *item,QString& group_id,uint32_t& search_req_id) +{ + QTreeWidgetItem *parent = item->parent(); + + if(parent == NULL) + return false ; + + search_req_id = parent->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt(); + group_id = itemId(item) ; + + return search_req_id > 0; +} + bool GroupTreeWidget::isSearchRequestItem(QPoint &point,uint32_t& search_req_id) { QTreeWidgetItem *item = ui->treeWidget->itemAt(point); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index 9f8d0ccd4..5b40a9487 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -97,6 +97,7 @@ public: bool isSearchRequestItem(QPoint &point,uint32_t& search_req_id); bool isSearchRequestResult(QPoint &point, QString &group_id, uint32_t& search_req_id); + bool isSearchRequestResultItem(QTreeWidgetItem *item,QString& group_id,uint32_t& search_req_id); QTreeWidgetItem *getItemFromId(const QString &id); QTreeWidgetItem *activateId(const QString &id, bool focus); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 069b682ce..b7f3fcf20 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -29,6 +29,7 @@ #include "gui/settings/rsharesettings.h" #include "gui/RetroShareLink.h" #include "gui/gxs/GxsGroupShareKey.h" +#include "gui/common/GroupTreeWidget.h" #include "gui/common/RSTreeWidget.h" #include "gui/notifyqt.h" #include "gui/common/UIStateHelper.h" @@ -367,9 +368,10 @@ void GxsGroupFrameDialog::removeAllSearches() { for(auto it(mSearchGroupsItems.begin());it!=mSearchGroupsItems.end();++it) { - TurtleRequestId search_request_id = 0 ; + QString group_id; + TurtleRequestId search_request_id; - if(ui->groupTreeWidget->isSearchRequestItem(point,search_request_id)) + if(ui->groupTreeWidget->isSearchRequestResultItem(it->second,group_id,search_request_id)) clearDistantSearchResults(search_request_id); ui->groupTreeWidget->removeSearchItem(it->second) ;