Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-02-25 17:21:27 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

improved debug output in rsgxsutil. Added disabled code for removing old unused groups

Browse Source
This commit is contained in:
csoler 2016-12-07 22:41:05 +01:00
parent d9416d3e3d
commit 6c4497a18c
4 changed files with 51 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -230,7 +230,7 @@ void RsGenExchange::tick()
} }
else else
{ {
mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore,mGixs); mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore,this,mGixs);
mIntegrityCheck->start("gxs integrity"); mIntegrityCheck->start("gxs integrity");
mChecking = true; mChecking = true;
} }

3
libretroshare/src/gxs/rsgenexchange.h
View File

@ -655,6 +655,7 @@ public:
virtual uint32_t getSyncPeriod(const RsGxsGroupId& grpId) ; virtual uint32_t getSyncPeriod(const RsGxsGroupId& grpId) ;
virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ; virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ;
uint16_t serviceType() const { return mServType ; }
protected: protected:
/** Notifications **/ /** Notifications **/
@ -827,8 +828,6 @@ private:
*/ */
void removeDeleteExistingMessages(RsGeneralDataService::MsgStoreMap& msgs, GxsMsgReq& msgIdsNotify); void removeDeleteExistingMessages(RsGeneralDataService::MsgStoreMap& msgs, GxsMsgReq& msgIdsNotify);
private:
RsMutex mGenMtx; RsMutex mGenMtx;
RsGxsDataAccess* mDataAccess; RsGxsDataAccess* mDataAccess;
RsGeneralDataService* mDataStore; RsGeneralDataService* mDataStore;

72
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -33,7 +33,9 @@
static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!) static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!)
//#define GXSUTIL_DEBUG 1 #define DEBUG_GXSUTIL 1
#define GXSUTIL_DEBUG() std::cerr << time(NULL) << " : GXS_UTIL : " << __FUNCTION__ << " : "
RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize) RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize)
: mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize) : mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize)
@ -45,9 +47,7 @@ RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService
std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator cit = grpMeta.begin(); std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator cit = grpMeta.begin();
for(;cit != grpMeta.end(); ++cit) for(;cit != grpMeta.end(); ++cit)
{
mGrpMeta.push_back(cit->second); mGrpMeta.push_back(cit->second);
}
} }
@ -57,6 +57,9 @@ bool RsGxsMessageCleanUp::clean()
time_t now = time(NULL); time_t now = time(NULL);
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " Cleaning up groups in service" << std::hex << mGenExchangeClient->serviceType() << std::dec << std::endl;
#endif
while(!mGrpMeta.empty()) while(!mGrpMeta.empty())
{ {
RsGxsGrpMetaData* grpMeta = mGrpMeta.back(); RsGxsGrpMetaData* grpMeta = mGrpMeta.back();
@ -70,6 +73,9 @@ bool RsGxsMessageCleanUp::clean()
GxsMsgMetaResult::iterator mit = result.begin(); GxsMsgMetaResult::iterator mit = result.begin();
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl;
#endif
req.clear(); req.clear();
uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ; uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ;
@ -97,7 +103,7 @@ bool RsGxsMessageCleanUp::clean()
{ {
req[grpId].push_back(meta->mMsgId); req[grpId].push_back(meta->mMsgId);
std::cerr << "Scheduling msg id " << meta->mMsgId << " in grp " << grpId << " for removal." << std::endl; GXSUTIL_DEBUG() << " Scheduling msg id " << meta->mMsgId << " in grp " << grpId << " for removal." << std::endl;
} }
delete meta; delete meta;
@ -116,8 +122,8 @@ bool RsGxsMessageCleanUp::clean()
return mGrpMeta.empty(); return mGrpMeta.empty();
} }
RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs) : RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGenExchange *genex, RsGixs *gixs) :
mDs(dataService), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs) mDs(dataService),mGenExchangeClient(genex), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs)
{ } { }
void RsGxsIntegrityCheck::run() void RsGxsIntegrityCheck::run()
@ -162,8 +168,8 @@ bool RsGxsIntegrityCheck::check()
if(!grp->metaData->mAuthorId.isNull()) if(!grp->metaData->mAuthorId.isNull())
{ {
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl; GXSUTIL_DEBUG() << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl;
#endif #endif
if(rsIdentity!=NULL && !rsIdentity->isBanned(grp->metaData->mAuthorId)) if(rsIdentity!=NULL && !rsIdentity->isBanned(grp->metaData->mAuthorId))
@ -182,6 +188,22 @@ bool RsGxsIntegrityCheck::check()
{ {
grpsToDel.push_back(grp->grpId); grpsToDel.push_back(grp->grpId);
} }
#ifdef TODO
if(!(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED))
{
RsGroupNetworkStats stats ;
mGenExchangeClient->getGroupNetworkStats(grp->grpId,stats);
if(stats.mSuppliers == 0 && stats.mMaxVisibleCount == 0)
{
GXSUTIL_DEBUG() << "Scheduling group \"" << grp->metaData->mGroupName << "\" ID=" << grp->grpId << " for deletion because it has no suppliers not any visible data at friends." << std::endl;
#warning Should we do that here? What happens for groups that are normally empty such as identities?
grpsToDel.push_back(grp->grpId);
}
}
#endif
delete grp; delete grp;
} }
@ -245,8 +267,8 @@ bool RsGxsIntegrityCheck::check()
} }
else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end()) else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end())
{ {
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl; GXSUTIL_DEBUG() << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl;
#endif #endif
if(rsIdentity!=NULL && !rsIdentity->isBanned(msg->metaData->mAuthorId)) if(rsIdentity!=NULL && !rsIdentity->isBanned(msg->metaData->mAuthorId))
used_gxs_ids.insert(msg->metaData->mAuthorId) ; used_gxs_ids.insert(msg->metaData->mAuthorId) ;
@ -268,9 +290,9 @@ bool RsGxsIntegrityCheck::check()
} }
mDeletedMsgs = msgsToDel; mDeletedMsgs = msgsToDel;
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << "At end of pass, this is the list used GXS ids: " << std::endl; GXSUTIL_DEBUG() << "At end of pass, this is the list used GXS ids: " << std::endl;
std::cerr << " requesting them to GXS identity service to enforce loading." << std::endl; GXSUTIL_DEBUG() << " requesting them to GXS identity service to enforce loading." << std::endl;
#endif #endif
std::list<RsPeerId> connected_friends ; std::list<RsPeerId> connected_friends ;
@ -281,14 +303,14 @@ bool RsGxsIntegrityCheck::check()
for(std::set<RsGxsId>::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it) for(std::set<RsGxsId>::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it)
{ {
gxs_ids.push_back(*it) ; gxs_ids.push_back(*it) ;
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " " << *it << std::endl; GXSUTIL_DEBUG() << " " << *it << std::endl;
#endif #endif
} }
uint32_t nb_requested_not_in_cache = 0; uint32_t nb_requested_not_in_cache = 0;
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " issuing random get on friends for non existing IDs" << std::endl; GXSUTIL_DEBUG() << " issuing random get on friends for non existing IDs" << std::endl;
#endif #endif
// now request a cache update for them, which triggers downloading from friends, if missing. // now request a cache update for them, which triggers downloading from friends, if missing.
@ -296,8 +318,8 @@ bool RsGxsIntegrityCheck::check()
for(;nb_requested_not_in_cache<MAX_GXS_IDS_REQUESTS_NET && !gxs_ids.empty();) for(;nb_requested_not_in_cache<MAX_GXS_IDS_REQUESTS_NET && !gxs_ids.empty();)
{ {
uint32_t n = RSRandom::random_u32() % gxs_ids.size() ; uint32_t n = RSRandom::random_u32() % gxs_ids.size() ;
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " requesting ID " << gxs_ids[n] ; GXSUTIL_DEBUG() << " requesting ID " << gxs_ids[n] ;
#endif #endif
if(!mGixs->haveKey(gxs_ids[n])) // checks if we have it already in the cache (conservative way to ensure that we atually have it) if(!mGixs->haveKey(gxs_ids[n])) // checks if we have it already in the cache (conservative way to ensure that we atually have it)
@ -305,14 +327,14 @@ bool RsGxsIntegrityCheck::check()
mGixs->requestKey(gxs_ids[n],connected_friends); mGixs->requestKey(gxs_ids[n],connected_friends);
++nb_requested_not_in_cache ; ++nb_requested_not_in_cache ;
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " ... from cache/net" << std::endl; GXSUTIL_DEBUG() << " ... from cache/net" << std::endl;
#endif #endif
} }
else else
{ {
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " ... already in cache" << std::endl; GXSUTIL_DEBUG() << " ... already in cache" << std::endl;
#endif #endif
// Note: we could time_stamp even in the case where the id is not cached. Anyway, it's not really a problem here, since IDs have a high chance of // Note: we could time_stamp even in the case where the id is not cached. Anyway, it's not really a problem here, since IDs have a high chance of
@ -324,8 +346,8 @@ bool RsGxsIntegrityCheck::check()
gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ; gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ;
gxs_ids.pop_back() ; gxs_ids.pop_back() ;
} }
#ifdef GXSUTIL_DEBUG #ifdef DEBUG_GXSUTIL
std::cerr << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl; GXSUTIL_DEBUG() << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl;
#endif #endif
return true; return true;

4
libretroshare/src/gxs/rsgxsutil.h
View File

@ -115,8 +115,7 @@ public:
* @param chunkSize * @param chunkSize
* @param sleepPeriod * @param sleepPeriod
*/ */
RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs); RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGenExchange *genex, RsGixs *gixs);
bool check(); bool check();
bool isDone(); bool isDone();
@ -128,6 +127,7 @@ public:
private: private:
RsGeneralDataService* const mDs; RsGeneralDataService* const mDs;
RsGenExchange *mGenExchangeClient;
bool mDone; bool mDone;
RsMutex mIntegrityMutex; RsMutex mIntegrityMutex;
std::list<RsGxsGroupId> mDeletedGrps; std::list<RsGxsGroupId> mDeletedGrps;