From 0c3b8641af987c12cc515263fc610ae3ddd8f4f5 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 9 May 2020 18:07:56 +0200 Subject: [PATCH] fixing some bugs in circles backend --- libretroshare/src/gxs/rsgenexchange.cc | 4 +- libretroshare/src/gxs/rsgxsnetservice.cc | 18 ++++--- libretroshare/src/services/p3gxscircles.cc | 58 ++++++++++++++------- libretroshare/src/services/p3gxscircles.h | 27 +++++++++- retroshare-gui/src/gui/Identity/IdDialog.ui | 12 ++--- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index aa2526548..bcd197abe 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -61,7 +61,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key #define GXS_MASK "GXS_MASK_HACK" -//#define GEN_EXCH_DEBUG 1 +#define GEN_EXCH_DEBUG 1 static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes @@ -937,7 +937,7 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin RsReputationLevel::LOCALLY_NEGATIVE ) { #ifdef GEN_EXCH_DEBUG - std::cerr << "RsGenExchange::validateMsg(): message from " << metaData.mAuthorId << ", rejected because reputation level (" << details.mReputation.mOverallReputationLevel <<") indicate that you banned this ID." << std::endl; + std::cerr << "RsGenExchange::validateMsg(): message from " << metaData.mAuthorId << ", rejected because reputation level (" << static_cast(details.mReputation.mOverallReputationLevel) <<") indicate that you banned this ID." << std::endl; #endif idValidate = false ; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index ba2e287ad..5fac5d731 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -274,12 +274,12 @@ NXS_NET_DEBUG_8 gxs distant sync ***/ -//#define NXS_NET_DEBUG_0 1 -//#define NXS_NET_DEBUG_1 1 +#define NXS_NET_DEBUG_0 1 +#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 -//#define NXS_NET_DEBUG_4 1 -//#define NXS_NET_DEBUG_5 1 +#define NXS_NET_DEBUG_4 1 +#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 //#define NXS_NET_DEBUG_8 1 @@ -320,9 +320,9 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ; || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7) \ || defined(NXS_NET_DEBUG_8) -static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; -static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ; // use this to allow to this group id only, or "" for all IDs -static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_CHANNELS ; // use this to allow to this service id only, or 0 for all services +static const RsPeerId peer_to_print = RsPeerId();//std::string("a97fef0e2dc82ddb19200fb30f9ac575")) ; +static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("66052380f5d1d0c5992e2b55dc402ce6")) ; // use this to allow to this group id only, or "" for all IDs +static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_GXSCIRCLE; // use this to allow to this service id only, or 0 for all services // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) class nullstream: public std::ostream {}; @@ -3598,6 +3598,10 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) msg->count = 1; // only one piece. This is to keep compatibility if we ever implement fragmenting in the future. msg->pos = 0; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),msg->grpId) << " sending msg Id " << msg->msgId << " in Group " << msg->grpId << std::endl; +#endif + newTr->mItems.push_back(msg); msgSize++; #endif diff --git a/libretroshare/src/services/p3gxscircles.cc b/libretroshare/src/services/p3gxscircles.cc index c94ecc0d8..04fcac3e8 100644 --- a/libretroshare/src/services/p3gxscircles.cc +++ b/libretroshare/src/services/p3gxscircles.cc @@ -126,7 +126,7 @@ p3GxsCircles::p3GxsCircles( RsGeneralDataService *gds, RsNetworkExchangeService RsGxsCircles(static_cast(*this)), GxsTokenQueue(this), RsTickEvent(), mIdentities(identities), mPgpUtils(pgpUtils), mCircleMtx("p3GxsCircles"), - mCircleCache(DEFAULT_MEM_CACHE_SIZE, "GxsCircleCache" ), +// mCircleCache(DEFAULT_MEM_CACHE_SIZE, "GxsCircleCache" ), mShouldSendCacheUpdateNotification(false) { // Kick off Cache Testing, + Others. @@ -1071,6 +1071,10 @@ RsGxsCircleCache::RsGxsCircleCache() bool RsGxsCircleCache::loadBaseCircle(const RsGxsCircleGroup& circle) { +#ifdef DEBUG_CIRCLES + std::cerr << "RsGxsCircleCache::loadBaseCircle(" << mCircleId << "):" << std::endl; +#endif // DEBUG_CIRCLES + mCircleId = RsGxsCircleId(circle.mMeta.mGroupId); mCircleName = circle.mMeta.mGroupName; // mProcessedCircles.insert(mCircleId); @@ -1084,19 +1088,23 @@ bool RsGxsCircleCache::loadBaseCircle(const RsGxsCircleGroup& circle) mAllowedNodes = circle.mLocalFriends ; mRestrictedCircleId = circle.mMeta.mCircleId ; - mMembershipStatus.clear() ; + // We do not clear mMembershipStatus because this might be an update and if we do, it will clear membership requests + // that are not in the invited list! for(std::set::const_iterator it(circle.mInvitedMembers.begin());it!=circle.mInvitedMembers.end();++it) { RsGxsCircleMembershipStatus& s(mMembershipStatus[*it]) ; + s.subscription_flags |= GXS_EXTERNAL_CIRCLE_FLAGS_IN_ADMIN_LIST ; + + // This one can be cleared because it will anyway be updated to the latest when loading subscription request messages and it wil only + // be used there as well. + s.last_subscription_TS = 0 ; - s.subscription_flags = GXS_EXTERNAL_CIRCLE_FLAGS_IN_ADMIN_LIST ; - } #ifdef DEBUG_CIRCLES - std::cerr << "RsGxsCircleCache::loadBaseCircle(" << mCircleId << ")"; - std::cerr << std::endl; + std::cerr << " Invited member " << *it << " Initializing/updating membership status to " << std::hex << s.subscription_flags << std::dec << std::endl; #endif // DEBUG_CIRCLES + } return true; } @@ -1251,7 +1259,6 @@ bool p3GxsCircles::cache_request_load(const RsGxsCircleId &id) cache.mStatus = CircleEntryCacheStatus::LOADING; mCirclesToLoad.insert(id); - mShouldSendCacheUpdateNotification = true; } if (RsTickEvent::event_count(CIRCLE_EVENT_CACHELOAD) > 0) /* its already scheduled */ @@ -1480,9 +1487,6 @@ bool p3GxsCircles::cache_reloadids(const RsGxsCircleId &circleId) cache.mStatus = CircleEntryCacheStatus::CHECKING_MEMBERSHIP; locked_checkCircleCacheForMembershipUpdate(cache); - /* move straight into the cache */ - mCircleCache.resize(); - std::cerr << " Loading complete." << std::endl; return true ; @@ -1923,6 +1927,7 @@ bool p3GxsCircles::processMembershipRequests(uint32_t token) RsGxsCircleId circle_id(it->first); RsGxsCircleCache& cache( mCircleCache[circle_id] ); + #ifdef DEBUG_CIRCLES std::cerr << " Circle found in cache!" << std::endl; std::cerr << " Retrieving messages..." << std::endl; @@ -1931,24 +1936,26 @@ bool p3GxsCircles::processMembershipRequests(uint32_t token) for(uint32_t i=0;isecond.size();++i) { #ifdef DEBUG_CIRCLES - std::cerr << " Group ID: " << it->second[i]->meta.mGroupId << ", Message ID: " << it->second[i]->meta.mMsgId << ": " ; + std::cerr << " Group ID: " << it->second[i]->meta.mGroupId << ", Message ID: " << it->second[i]->meta.mMsgId << ", thread ID: " << it->second[i]->meta.mThreadId << ", author: " << it->second[i]->meta.mAuthorId << ": " ; #endif RsGxsCircleSubscriptionRequestItem *item = dynamic_cast(it->second[i]) ; if(item == NULL) { - std::cerr << " (EE) item is not a RsGxsCircleSubscriptionRequestItem. Weird." << std::endl; + std::cerr << " (EE) item is not a RsGxsCircleSubscriptionRequestItem. Weird. Scheduling for deletion." << std::endl; + + messages_to_delete[RsGxsGroupId(circle_id)].insert(it->second[i]->meta.mMsgId); continue ; } RsGxsCircleMembershipStatus& info(cache.mMembershipStatus[item->meta.mAuthorId]) ; #ifdef DEBUG_CIRCLES - std::cerr << " is from id " << item->meta.mAuthorId << " " << time(NULL) - item->time_stamp << " seconds ago, " ; + std::cerr << " " << time(NULL) - item->time_stamp << " seconds ago, " ; #endif - if(info.last_subscription_TS < item->time_stamp) + if(info.last_subscription_TS <= item->time_stamp) // the <= here allows to make sure we update the flags is something happenned { info.last_subscription_TS = item->time_stamp ; @@ -1961,15 +1968,28 @@ bool p3GxsCircles::processMembershipRequests(uint32_t token) mShouldSendCacheUpdateNotification = true; #ifdef DEBUG_CIRCLES - std::cerr << " UPDATING" << std::endl; + std::cerr << " UPDATING status to " << std::hex << info.subscription_flags << std::dec << std::endl; #endif } - else if(info.last_subscription_TS > item->time_stamp) + else if(info.last_subscription_TS > item->time_stamp) + std::cerr << " Too old: item->TS=" << item->time_stamp << ", last_subscription_TS=" << info.last_subscription_TS << ". IGNORING." << std::endl; + } + + // now do another sweep and remove all msgs that are older than the latest + + std::cerr << " Cleaning old messages..." << std::endl; + + for(uint32_t i=0;isecond.size();++i) + { + RsGxsCircleMembershipStatus& info(cache.mMembershipStatus[it->second[i]->meta.mAuthorId]) ; + RsGxsCircleSubscriptionRequestItem *item = dynamic_cast(it->second[i]) ; + + if(item && info.last_subscription_TS > item->time_stamp) { #ifdef DEBUG_CIRCLES - std::cerr << " Older than last known (" << time(NULL)-info.last_subscription_TS << " seconds ago): deleting." << std::endl; + std::cerr << " " << item->meta.mMsgId << ": Older than last known (" << (long int)info.last_subscription_TS - (long int)item->time_stamp << " seconds before): deleting." << std::endl; #endif - messages_to_delete[RsGxsGroupId(circle_id)].insert(it->second[i]->meta.mMsgId) ; + messages_to_delete[RsGxsGroupId(circle_id)].insert(item->meta.mMsgId) ; } } @@ -2002,7 +2022,7 @@ void p3GxsCircles::debug_dumpCache() { std::cerr << "Debug dump of CircleCache:" << std::endl; - mCircleCache.printStats(std::cerr); + mCircleCache.printStats(); mCircleCache.applyToAllCachedEntries(*this,&p3GxsCircles::debug_dumpCacheEntry); } diff --git a/libretroshare/src/services/p3gxscircles.h b/libretroshare/src/services/p3gxscircles.h index e57cd7652..1b7986014 100644 --- a/libretroshare/src/services/p3gxscircles.h +++ b/libretroshare/src/services/p3gxscircles.h @@ -128,7 +128,7 @@ public: uint32_t subscription_flags ; // combination of GXS_EXTERNAL_CIRCLE_FLAGS_IN_ADMIN_LIST and GXS_EXTERNAL_CIRCLE_FLAGS_SUBSCRIBED }; -typedef enum CircleEntryCacheStatus: uint8_t { +enum CircleEntryCacheStatus: uint8_t { UNKNOWN = 0x00, // Used to detect uninitialized memory NO_DATA_YET = 0x01, // Used in the constuctor LOADING = 0x02, // When the token request to load cache has been sent and no data is present @@ -185,6 +185,28 @@ public: class PgpAuxUtils; +class RsCirclesMemCache : public std::map +{ +public: + RsCirclesMemCache() : std::map(){} + + bool is_cached(const RsGxsCircleId& id) { return end() != find(id) ; } + RsGxsCircleCache& ref(const RsGxsCircleId& id) { return operator[](id) ; } + + void printStats() { std::cerr << "CircleMemCache: " << size() << " elements." << std::endl; } + + template void applyToAllCachedEntries(ClientClass& c,bool (ClientClass::*method)(RsGxsCircleCache&)) + { + for(auto& it:*this) + (c.*method)(it.second); + } + template void applyToAllCachedEntries(ClientClass& c,bool (ClientClass::*method)(const RsGxsCircleCache&)) + { + for(const auto& it:*this) + (c.*method)(it.second); + } +}; + class p3GxsCircles: public RsGxsCircleExchange, public RsGxsCircles, public GxsTokenQueue, public RsTickEvent { @@ -341,7 +363,8 @@ public: //std::map > mCacheLoad_SubCircle; std::set mCirclesToLoad; // list of circles to update/load, so that we can treat them by groups. - RsMemCache mCircleCache; // actual cache data + RsCirclesMemCache mCircleCache; + //RsMemCache mCircleCache; // actual cache data void debug_dumpCache(); // debug method to overview what's going on bool debug_dumpCacheEntry(RsGxsCircleCache &cache); diff --git a/retroshare-gui/src/gui/Identity/IdDialog.ui b/retroshare-gui/src/gui/Identity/IdDialog.ui index c881ed39f..127f7a85a 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.ui +++ b/retroshare-gui/src/gui/Identity/IdDialog.ui @@ -120,7 +120,7 @@ Qt::NoFocus - + :/icons/help_64.png:/icons/help_64.png @@ -268,7 +268,7 @@ - 0 + 1 @@ -289,8 +289,8 @@ 0 0 - 634 - 523 + 1372 + 719 @@ -1052,7 +1052,7 @@ border-image: url(:/images/closepressed.png) - + :/images/toaster/chat.png:/images/toaster/chat.png @@ -1094,8 +1094,8 @@ border-image: url(:/images/closepressed.png) idTreeWidget - +