From 6c4497a18c89af3d0af093cb70719a9c3a5df23e Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 7 Dec 2016 22:41:05 +0100 Subject: [PATCH] improved debug output in rsgxsutil. Added disabled code for removing old unused groups --- libretroshare/src/gxs/rsgenexchange.cc | 2 +- libretroshare/src/gxs/rsgenexchange.h | 3 +- libretroshare/src/gxs/rsgxsutil.cc | 72 +++++++++++++++++--------- libretroshare/src/gxs/rsgxsutil.h | 4 +- 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 940c3a591..4fed3ec29 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -230,7 +230,7 @@ void RsGenExchange::tick() } else { - mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore,mGixs); + mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore,this,mGixs); mIntegrityCheck->start("gxs integrity"); mChecking = true; } diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 5ee37add3..a391d47e0 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -655,6 +655,7 @@ public: virtual uint32_t getSyncPeriod(const RsGxsGroupId& grpId) ; virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ; + uint16_t serviceType() const { return mServType ; } protected: /** Notifications **/ @@ -827,8 +828,6 @@ private: */ void removeDeleteExistingMessages(RsGeneralDataService::MsgStoreMap& msgs, GxsMsgReq& msgIdsNotify); -private: - RsMutex mGenMtx; RsGxsDataAccess* mDataAccess; RsGeneralDataService* mDataStore; diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index a61e77529..164420517 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -33,7 +33,9 @@ static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!) -//#define GXSUTIL_DEBUG 1 +#define DEBUG_GXSUTIL 1 + +#define GXSUTIL_DEBUG() std::cerr << time(NULL) << " : GXS_UTIL : " << __FUNCTION__ << " : " RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize) : mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize) @@ -45,9 +47,7 @@ RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService std::map::iterator cit = grpMeta.begin(); for(;cit != grpMeta.end(); ++cit) - { mGrpMeta.push_back(cit->second); - } } @@ -57,6 +57,9 @@ bool RsGxsMessageCleanUp::clean() time_t now = time(NULL); +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " Cleaning up groups in service" << std::hex << mGenExchangeClient->serviceType() << std::dec << std::endl; +#endif while(!mGrpMeta.empty()) { RsGxsGrpMetaData* grpMeta = mGrpMeta.back(); @@ -70,6 +73,9 @@ bool RsGxsMessageCleanUp::clean() GxsMsgMetaResult::iterator mit = result.begin(); +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl; +#endif req.clear(); uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ; @@ -97,7 +103,7 @@ bool RsGxsMessageCleanUp::clean() { req[grpId].push_back(meta->mMsgId); - std::cerr << "Scheduling msg id " << meta->mMsgId << " in grp " << grpId << " for removal." << std::endl; + GXSUTIL_DEBUG() << " Scheduling msg id " << meta->mMsgId << " in grp " << grpId << " for removal." << std::endl; } delete meta; @@ -116,8 +122,8 @@ bool RsGxsMessageCleanUp::clean() return mGrpMeta.empty(); } -RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs) : - mDs(dataService), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs) +RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGenExchange *genex, RsGixs *gixs) : + mDs(dataService),mGenExchangeClient(genex), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs) { } void RsGxsIntegrityCheck::run() @@ -162,8 +168,8 @@ bool RsGxsIntegrityCheck::check() if(!grp->metaData->mAuthorId.isNull()) { -#ifdef GXSUTIL_DEBUG - std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl; #endif if(rsIdentity!=NULL && !rsIdentity->isBanned(grp->metaData->mAuthorId)) @@ -182,6 +188,22 @@ bool RsGxsIntegrityCheck::check() { grpsToDel.push_back(grp->grpId); } + +#ifdef TODO + if(!(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)) + { + RsGroupNetworkStats stats ; + mGenExchangeClient->getGroupNetworkStats(grp->grpId,stats); + + if(stats.mSuppliers == 0 && stats.mMaxVisibleCount == 0) + { + GXSUTIL_DEBUG() << "Scheduling group \"" << grp->metaData->mGroupName << "\" ID=" << grp->grpId << " for deletion because it has no suppliers not any visible data at friends." << std::endl; +#warning Should we do that here? What happens for groups that are normally empty such as identities? + grpsToDel.push_back(grp->grpId); + } + } +#endif + delete grp; } @@ -245,8 +267,8 @@ bool RsGxsIntegrityCheck::check() } else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end()) { -#ifdef GXSUTIL_DEBUG - std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl; #endif if(rsIdentity!=NULL && !rsIdentity->isBanned(msg->metaData->mAuthorId)) used_gxs_ids.insert(msg->metaData->mAuthorId) ; @@ -268,9 +290,9 @@ bool RsGxsIntegrityCheck::check() } mDeletedMsgs = msgsToDel; -#ifdef GXSUTIL_DEBUG - std::cerr << "At end of pass, this is the list used GXS ids: " << std::endl; - std::cerr << " requesting them to GXS identity service to enforce loading." << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << "At end of pass, this is the list used GXS ids: " << std::endl; + GXSUTIL_DEBUG() << " requesting them to GXS identity service to enforce loading." << std::endl; #endif std::list connected_friends ; @@ -281,14 +303,14 @@ bool RsGxsIntegrityCheck::check() for(std::set::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it) { gxs_ids.push_back(*it) ; -#ifdef GXSUTIL_DEBUG - std::cerr << " " << *it << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " " << *it << std::endl; #endif } uint32_t nb_requested_not_in_cache = 0; -#ifdef GXSUTIL_DEBUG - std::cerr << " issuing random get on friends for non existing IDs" << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " issuing random get on friends for non existing IDs" << std::endl; #endif // now request a cache update for them, which triggers downloading from friends, if missing. @@ -296,8 +318,8 @@ bool RsGxsIntegrityCheck::check() for(;nb_requested_not_in_cachehaveKey(gxs_ids[n])) // checks if we have it already in the cache (conservative way to ensure that we atually have it) @@ -305,14 +327,14 @@ bool RsGxsIntegrityCheck::check() mGixs->requestKey(gxs_ids[n],connected_friends); ++nb_requested_not_in_cache ; -#ifdef GXSUTIL_DEBUG - std::cerr << " ... from cache/net" << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " ... from cache/net" << std::endl; #endif } else { -#ifdef GXSUTIL_DEBUG - std::cerr << " ... already in cache" << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " ... already in cache" << std::endl; #endif // Note: we could time_stamp even in the case where the id is not cached. Anyway, it's not really a problem here, since IDs have a high chance of @@ -324,8 +346,8 @@ bool RsGxsIntegrityCheck::check() gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ; gxs_ids.pop_back() ; } -#ifdef GXSUTIL_DEBUG - std::cerr << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl; +#ifdef DEBUG_GXSUTIL + GXSUTIL_DEBUG() << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl; #endif return true; diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index b662acfc9..cfd777c90 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -115,8 +115,7 @@ public: * @param chunkSize * @param sleepPeriod */ - RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs); - + RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGenExchange *genex, RsGixs *gixs); bool check(); bool isDone(); @@ -128,6 +127,7 @@ public: private: RsGeneralDataService* const mDs; + RsGenExchange *mGenExchangeClient; bool mDone; RsMutex mIntegrityMutex; std::list mDeletedGrps;