From b52ab0f542398659253e7d7e473d1c04d52e4d23 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 19 Jan 2017 23:59:26 +0100 Subject: [PATCH 1/2] added auto-clean of unused groups when grpAutoSync is enabled (so it does not touch identities) --- libretroshare/src/gxs/rsgds.h | 2 + libretroshare/src/gxs/rsgenexchange.cc | 12 +++++- libretroshare/src/gxs/rsgenexchange.h | 1 + libretroshare/src/gxs/rsgxsnetservice.cc | 48 ++++++++++++++++++++++-- libretroshare/src/gxs/rsgxsnetservice.h | 1 + libretroshare/src/gxs/rsgxsutil.cc | 9 ++--- libretroshare/src/gxs/rsnxs.h | 8 ++++ 7 files changed, 70 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index 0593610ce..c56b47c12 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -90,6 +90,8 @@ public: uint32_t mSuppliers ; uint32_t mMaxVisibleCount ; + bool mGrpAutoSync ; + bool mAllowMsgSync; }; typedef std::map > NxsMsgDataResult; diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 9c484b79e..3396b1e94 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -81,7 +81,7 @@ RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService mLastClean((int)time(NULL) - (int)(RSRandom::random_u32() % MSG_CLEANUP_PERIOD)), // this helps unsynchronising the checks for the different services mMsgCleanUp(NULL), mChecking(false), - mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD)), // this helps unsynchronising the checks for the different services + mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD) + 120), // this helps unsynchronising the checks for the different services, with 2 min security to avoid checking right away before statistics come up. mIntegrityCheck(NULL), CREATE_FAIL(0), CREATE_SUCCESS(1), @@ -211,6 +211,11 @@ void RsGenExchange::tick() std::cerr << " " << *it << std::endl; #endif mNotifications.push_back(gc); + + // also notify the network exchange service that these groups no longer exist. + + if(mNetService) + mNetService->removeGroups(grpIds) ; } if (!msgIds.empty()) { @@ -1639,6 +1644,11 @@ uint32_t RsGenExchange::getSyncPeriod(const RsGxsGroupId& grpId) return RS_GXS_DEFAULT_MSG_REQ_PERIOD; } +bool RsGenExchange::getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats) +{ + return (!mNetService) || mNetService->getGroupNetworkStats(grpId,stats) ; +} + void RsGenExchange::setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) { if(mNetService != NULL) diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 706ed9401..959faeda6 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -646,6 +646,7 @@ public: virtual uint32_t getDefaultSyncPeriod(); virtual uint32_t getSyncPeriod(const RsGxsGroupId& grpId) ; virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ; + virtual bool getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats); uint16_t serviceType() const { return mServType ; } uint32_t serviceFullType() const { return ((uint32_t)mServType << 8) + (((uint32_t) RS_PKT_VERSION_SERVICE) << 24); } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 7653c4e20..51d8e3d04 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -225,12 +225,12 @@ NXS_NET_DEBUG_7 encryption/decryption of transactions ***/ -//#define NXS_NET_DEBUG_0 1 +#define NXS_NET_DEBUG_0 1 //#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 -//#define NXS_NET_DEBUG_5 1 +#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 @@ -2311,6 +2311,8 @@ bool RsGxsNetService::getGroupNetworkStats(const RsGxsGroupId& gid,RsGroupNetwor stats.mSuppliers = it->second.suppliers.ids.size(); stats.mMaxVisibleCount = it->second.max_visible_count ; + stats.mAllowMsgSync = mAllowMsgSync ; + stats.mGrpAutoSync = mGrpAutoSync ; return true ; } @@ -4046,7 +4048,12 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "handleRecvSyncMsg(): Received last update TS of group " << item->grpId << ", for peer " << peer << ", TS = " << time(NULL) - item->updateTS << " secs ago." ; #endif - if(grp_is_known) + // We update suppliers in two cases: + // Case 1: the grp is known because it is the hash of an existing group, but it's not yet in the server config map + // Case 2: the gtp is not known, possibly because it was deleted, but there's an entry in mServerGrpConfigMap due to statistics gathering. Still, statistics are only + // gathered from known suppliers. So statistics never add new suppliers. These are only added here. + + if(grp_is_known || mServerGrpConfigMap.find(item->grpId)!=mServerGrpConfigMap.end()) { RsGxsGrpConfig & rec(mServerGrpConfigMap[item->grpId]) ; // this creates it if needed. When the grp is unknown (and hashed) this will would create a unused entry rec.suppliers.ids.insert(peer) ; @@ -4725,7 +4732,7 @@ void RsGxsNetService::handleRecvPublishKeys(RsNxsGroupPublishKeyItem *item) if(ret) { -#ifdef NXS_NET_DEBUG +#ifdef NXS_NET_DEBUG_3 GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " updated database with new publish keys." << std::endl; #endif mNewPublishKeysToNotify.insert(item->grpId) ; @@ -4752,6 +4759,39 @@ bool RsGxsNetService::getGroupServerUpdateTS(const RsGxsGroupId& gid,time_t& gro return true ; } +bool RsGxsNetService::removeGroups(const std::list& groups) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + GXSNETDEBUG___ << "Removing group information from deleted groups:" << std::endl; + + for(std::list::const_iterator git(groups.begin());git!=groups.end();++git) + { + GXSNETDEBUG__G(*git) << " deleting info for group " << *git << std::endl; + + GrpConfigMap::iterator it = mServerGrpConfigMap.find(*git) ; + + if(it != mServerGrpConfigMap.end()) + { + it->second.suppliers.TlvClear(); // we dont erase the entry, because we want to keep the user-defined sync parameters. + it->second.max_visible_count = 0; + } + + mServerMsgUpdateMap.erase(*git) ; + + for(ClientMsgMap::iterator it(mClientMsgUpdateMap.begin());it!=mClientMsgUpdateMap.end();++it) + it->second.msgUpdateInfos.erase(*git) ; + + // This last step is very important: it makes RS re-sync all groups after deleting, with every new peer. If may happen indeed that groups + // are deleted because there's no suppliers since the actual supplier friend is offline for too long. In this case, the group needs + // to re-appear when the friend who is a subscriber comes online again. + + mClientGrpUpdateMap.clear(); + } + + IndicateConfigChanged(); +} + bool RsGxsNetService::stampMsgServerUpdateTS(const RsGxsGroupId& gid) { RS_STACK_MUTEX(mNxsMutex) ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 6aae36462..b14d55058 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -161,6 +161,7 @@ public: virtual bool getGroupServerUpdateTS(const RsGxsGroupId& gid,time_t& grp_server_update_TS,time_t& msg_server_update_TS) ; virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) ; + virtual bool removeGroups(const std::list& groups); /* p3Config methods */ public: diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index aa5c5558a..2e9ce5204 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -188,20 +188,17 @@ bool RsGxsIntegrityCheck::check() grpsToDel.push_back(grp->grpId); } -#ifdef TODO - if(!(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)) + if(!(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) && !(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_ADMIN) && !(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_PUBLISH)) { RsGroupNetworkStats stats ; mGenExchangeClient->getGroupNetworkStats(grp->grpId,stats); - if(stats.mSuppliers == 0 && stats.mMaxVisibleCount == 0) + if(stats.mSuppliers == 0 && stats.mMaxVisibleCount == 0 && stats.mGrpAutoSync) { - GXSUTIL_DEBUG() << "Scheduling group \"" << grp->metaData->mGroupName << "\" ID=" << grp->grpId << " for deletion because it has no suppliers not any visible data at friends." << std::endl; -#warning Should we do that here? What happens for groups that are normally empty such as identities? + GXSUTIL_DEBUG() << "Scheduling group \"" << grp->metaData->mGroupName << "\" ID=" << grp->grpId << " in service " << std::hex << mGenExchangeClient->serviceType() << std::dec << " for deletion because it has no suppliers not any visible data at friends." << std::endl; grpsToDel.push_back(grp->grpId); } } -#endif delete grp; } diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 9b879d2d0..ef86caa28 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -162,6 +162,14 @@ public: */ virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) =0; + /*! + * \brief removeGroups + * Removes time stamp information from the list of groups. This allows to re-sync them if suppliers are present. + * \param groups list of groups to remove from the update maps + * \return true if nothing bad happens. + */ + virtual bool removeGroups(const std::list& groups)=0; + /*! * \brief minReputationForForwardingMessages * Encodes the policy for sending/requesting messages depending on anti-spam settings. From ee4d722225f228ee14675f5087370bca7c014fa3 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 20 Jan 2017 23:27:18 +0100 Subject: [PATCH 2/2] fixed cache deletion of GXS grp meta that caused deleted groups to not come back when available --- libretroshare/src/gxs/rsdataservice.cc | 4 ++++ libretroshare/src/gxs/rsgenexchange.cc | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index fbf8ad9a6..8486e95d0 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1634,10 +1634,14 @@ bool RsDataService::locked_removeGroupEntries(const std::vector& g const RsGxsGroupId& grpId = *vit; mDb->sqlDelete(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", ""); + + // also remove the group meta from cache. + mGrpMetaDataCache.erase(*vit) ; } ret &= mDb->commitTransaction(); + mGrpMetaDataCache_ContainsAllDatabase = false ; return ret; } uint32_t RsDataService::cacheSize() const { diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 48adc6b35..7551d68c3 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -63,8 +63,8 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key //#define GEN_EXCH_DEBUG 1 -#define MSG_CLEANUP_PERIOD 60*59 // 59 minutes -#define INTEGRITY_CHECK_PERIOD 60*31 // 31 minutes +static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes +static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService *ns, RsSerialType *serviceSerialiser, uint16_t servType, RsGixs* gixs,