From 2fd15134c946d12f77825037321eeedd3cc8d5f7 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 22 Nov 2020 21:45:21 +0100 Subject: [PATCH] added auto-cleaning of unused forums, channels, posted. Works for all GXS groups if the correct method is supplied. --- libretroshare/src/gxs/rsgds.h | 2 + libretroshare/src/gxs/rsgenexchange.cc | 14 +- libretroshare/src/gxs/rsgenexchange.h | 17 ++- libretroshare/src/gxs/rsgxsnetservice.h | 2 + libretroshare/src/gxs/rsgxsutil.cc | 148 +++++++++++--------- libretroshare/src/gxs/rsgxsutil.h | 4 +- libretroshare/src/gxs/rsnxs.h | 3 + libretroshare/src/retroshare/rsgxsflags.h | 3 +- libretroshare/src/services/p3gxschannels.cc | 82 +++++++++-- libretroshare/src/services/p3gxschannels.h | 3 + libretroshare/src/services/p3gxscircles.cc | 1 - libretroshare/src/services/p3gxsforums.cc | 72 +++++++++- libretroshare/src/services/p3gxsforums.h | 33 ++--- libretroshare/src/services/p3postbase.cc | 77 +++++++++- libretroshare/src/services/p3postbase.h | 10 +- libretroshare/src/services/p3posted.cc | 3 + libretroshare/src/services/p3posted.h | 15 +- 17 files changed, 363 insertions(+), 126 deletions(-) diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index fb985b9f7..d0879b2c9 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -159,6 +159,8 @@ public: /*! * Retrieves meta data of all groups stored (most current versions only) + * Memory is owned by the service, not the caller. Therefore the pointers in the temporary map + * shouldn't be destroyed. * * @param grp if null grpIds entries are made, only meta for those grpId are retrieved \n * , if grpId is failed to be retrieved it will be erased from map diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 2b88ad730..966eb18d4 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -131,7 +131,8 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key // | // +--- processRoutingClues() ; // -static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes +//static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes +static const uint32_t MSG_CLEANUP_PERIOD = 60*5; // 59 minutes static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes RsGenExchange::RsGenExchange( @@ -257,10 +258,16 @@ void RsGenExchange::tick() rstime_t now = time(NULL); - if((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning) + // Cleanup unused data. This is only needed when auto-synchronization is needed, which is not the case + // of identities. This is why idendities do their own cleaning. + + if((mNetService->msgAutoSync() || mNetService->grpAutoSync()) + && ((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning)) { if(mMsgCleanUp) { + RS_STACK_MUTEX(mGenMtx); + if(mMsgCleanUp->clean()) { mCleaning = false; @@ -268,11 +275,10 @@ void RsGenExchange::tick() mMsgCleanUp = NULL; mLastClean = time(NULL); } - } else { - mMsgCleanUp = new RsGxsMessageCleanUp(mDataStore, this, 1); + mMsgCleanUp = new RsGxsCleanUp(mDataStore, this, 1); mCleaning = true; } } diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index a72562391..a4ba8a50f 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -648,6 +648,19 @@ protected: */ virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); + /*! + * \brief service_checkIfGroupIsStillUsed + * Re-implement this function to help GXS cleaning, by telling that some particular group + * is not used anymore. This usually depends on subscription, the fact that friend nodes send + * some info or not, and particular cleaning strategy of each service. + * Besides, groups in some services are used by other services (e.g. identities, circles, are used in + * forums and so on), so deciding on a group usage can only be left to the specific service it is used in. + * \return + * true if the group is still used, false otherwise, meaning that the group can be deleted. Default is + * that the group is always in use. + */ + virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& /* meta */) { return true; } // see RsGenExchange + public: /*! @@ -959,7 +972,7 @@ private: bool mCleaning; rstime_t mLastClean; - RsGxsMessageCleanUp* mMsgCleanUp; + RsGxsCleanUp* mMsgCleanUp; bool mChecking, mCheckStarted; @@ -982,6 +995,8 @@ private: std::vector mMsgDeletePublish; std::map > mRoutingClues ; + + friend class RsGxsCleanUp; }; #endif // RSGENEXCHANGE_H diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 9322811fc..e6eb36246 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -128,6 +128,8 @@ public: virtual void setDefaultKeepAge(uint32_t t) override { mDefaultMsgStorePeriod = t ; } virtual void setDefaultSyncAge(uint32_t t) override { mDefaultMsgSyncPeriod = t ; } + virtual bool msgAutoSync() const override { return mAllowMsgSync; } + virtual bool grpAutoSync() const override { return mGrpAutoSync; } /*! * \brief Search methods. * These four methods are used to request distant search and receive the results. diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index 491c6e3c2..10620b2cf 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -35,15 +35,19 @@ # include "rsitems/rsgxschannelitems.h" #endif +// The goals of this set of methods is to check GXS messages and groups for consistency, mostly +// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't +// happen anyway, but we still conduct these test as an extra safety measure. + static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!) -//#define DEBUG_GXSUTIL 1 +#define DEBUG_GXSUTIL 1 #ifdef DEBUG_GXSUTIL #define GXSUTIL_DEBUG() std::cerr << "[" << time(NULL) << "] : GXS_UTIL : " << __FUNCTION__ << " : " #endif -RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize) +RsGxsCleanUp::RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize) : mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize) { RsGxsGrpMetaTemporaryMap grpMeta; @@ -53,96 +57,111 @@ RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService mGrpMeta.push_back(cit->second); } -bool RsGxsMessageCleanUp::clean() +bool RsGxsCleanUp::clean() { - uint32_t i = 1; + uint32_t i = 1; - rstime_t now = time(NULL); + rstime_t now = time(NULL); + std::vector grps_to_delete; #ifdef DEBUG_GXSUTIL - uint16_t service_type = mGenExchangeClient->serviceType() ; - GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << std::endl; + uint16_t service_type = mGenExchangeClient->serviceType() ; + GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << std::endl; #endif - while(!mGrpMeta.empty()) - { - const RsGxsGrpMetaData* grpMeta = mGrpMeta.back(); - const RsGxsGroupId& grpId = grpMeta->mGroupId; - mGrpMeta.pop_back(); - GxsMsgReq req; - GxsMsgMetaResult result; + while(!mGrpMeta.empty()) + { + const RsGxsGrpMetaData* grpMeta = mGrpMeta.back(); - req[grpId] = std::set(); - mDs->retrieveGxsMsgMetaData(req, result); + // first check if we keep the group or not - GxsMsgMetaResult::iterator mit = result.begin(); + if(!mGenExchangeClient->service_checkIfGroupIsStillUsed(*grpMeta)) + { +#ifdef DEBUG_GXSUTIL + std::cerr << " Scheduling group " << grpMeta->mGroupId << " for removal." << std::endl; +#endif + grps_to_delete.push_back(grpMeta->mGroupId); + } + else + { + const RsGxsGroupId& grpId = grpMeta->mGroupId; + mGrpMeta.pop_back(); + GxsMsgReq req; + GxsMsgMetaResult result; + + req[grpId] = std::set(); + mDs->retrieveGxsMsgMetaData(req, result); + + GxsMsgMetaResult::iterator mit = result.begin(); #ifdef DEBUG_GXSUTIL - GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl; + GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl; #endif - req.clear(); + GxsMsgReq messages_to_delete; - uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ; + uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ; - for(; mit != result.end(); ++mit) - { - std::vector& metaV = mit->second; + for(; mit != result.end(); ++mit) + { + std::vector& metaV = mit->second; - // First, make a map of which message have a child message. This allows to only delete messages that dont have child messages. - // A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the - // end the message tree will be deleted slice after slice, which should still be reasonnably fast. - // - std::set messages_with_kids ; + // First, make a map of which message have a child message. This allows to only delete messages that dont have child messages. + // A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the + // end the message tree will be deleted slice after slice, which should still be reasonnably fast. + // + std::set messages_with_kids ; - for( uint32_t i=0;imParentId.isNull()) - messages_with_kids.insert(metaV[i]->mParentId) ; + for( uint32_t i=0;imParentId.isNull()) + messages_with_kids.insert(metaV[i]->mParentId) ; - for( uint32_t i=0;imMsgId)!=messages_with_kids.end()); + bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end()); - // check if expired - bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids; + // check if expired + bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids; - // check client does not want the message kept regardless of age - remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER); + // check client does not want the message kept regardless of age + remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER); - // if not subscribed remove messages (can optimise this really) - remove = remove || (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED); - remove = remove || !(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED); + // if not subscribed remove messages (can optimise this really) + remove = remove || (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED); + remove = remove || !(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED); #ifdef DEBUG_GXSUTIL - GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP) - << " subscribed: " << bool(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period - << " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ; + GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER) + << " subscribed: " << bool(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period + << " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ; #endif - if( remove ) - { - req[grpId].insert(meta->mMsgId); - + if( remove ) + { + messages_to_delete[grpId].insert(meta->mMsgId); + #ifdef DEBUG_GXSUTIL - std::cerr << " Scheduling for removal." << std::endl; + std::cerr << " Scheduling for removal." << std::endl; #endif - } + } #ifdef DEBUG_GXSUTIL - else - std::cerr << std::endl; + else + std::cerr << std::endl; #endif + //delete meta; + } + } - //delete meta; - } - } + mDs->removeMsgs(messages_to_delete); - mDs->removeMsgs(req); + i++; + if(i > CHUNK_SIZE) break; + } + } - i++; - if(i > CHUNK_SIZE) break; - } + //mDs->removeGroups(grps_to_delete); - return mGrpMeta.empty(); + return mGrpMeta.empty(); } RsGxsIntegrityCheck::RsGxsIntegrityCheck( @@ -223,6 +242,8 @@ bool RsGxsIntegrityCheck::check() else msgIds.erase(msgIds.find(grp->grpId)); #ifdef RS_DEEP_CHANNEL_INDEX + // This should be moved to p3gxschannels. It is really not the place for this here! + if( isGxsChannels && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) @@ -359,8 +380,9 @@ bool RsGxsIntegrityCheck::check() else if (subscribed_groups.count(msg->metaData->mGroupId)) { #ifdef RS_DEEP_CHANNEL_INDEX - if( isGxsChannels - && indexedGroups.count(msg->metaData->mGroupId) ) + // This should be moved to p3gxschannels. It is really not the place for this here! + + if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) ) { RsGxsMsgMetaData meta; meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len); diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index 50e749f3a..4cc9f65ee 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -125,7 +125,7 @@ inline RsGxsGrpMsgIdPair getMsgIdPair(RsGxsMsgItem& msg) * Does message clean up based on individual group expirations first * if avialable. If not then deletion s */ -class RsGxsMessageCleanUp +class RsGxsCleanUp { public: @@ -136,7 +136,7 @@ public: * @param chunkSize * @param sleepPeriod */ - RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize); + RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize); /*! * On construction this should be called to progress deletions diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index ea12e146a..fb6238f92 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -82,6 +82,9 @@ public: virtual uint32_t getDefaultSyncAge() =0; virtual uint32_t getDefaultKeepAge() =0; + virtual bool msgAutoSync() const =0; + virtual bool grpAutoSync() const =0; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// DISTANT SEARCH FUNCTIONS /// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/libretroshare/src/retroshare/rsgxsflags.h b/libretroshare/src/retroshare/rsgxsflags.h index 10357118a..7ef6d2c49 100644 --- a/libretroshare/src/retroshare/rsgxsflags.h +++ b/libretroshare/src/retroshare/rsgxsflags.h @@ -107,8 +107,7 @@ namespace GXS_SERV { static const uint32_t GXS_MSG_STATUS_UNPROCESSED = 0x00000001; // Flags to store the read/process status of group messages. static const uint32_t GXS_MSG_STATUS_GUI_UNREAD = 0x00000002; // The actual meaning may depend on the type of service. static const uint32_t GXS_MSG_STATUS_GUI_NEW = 0x00000004; // - /** Do not delete message even if older then group maximum storage time */ - static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008; + static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008; // Do not delete message even if older then group maximum storage time static const uint32_t GXS_MSG_STATUS_DELETE = 0x00000020; // /** END GXS Msg status flags **/ diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 7d181d021..b61880cd5 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -64,11 +64,13 @@ #define CHANNEL_PROCESS 0x0001 #define CHANNEL_TESTEVENT_DUMMYDATA 0x0002 -#define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated. +#define DUMMYDATA_PERIOD 60 // Long enough for some RsIdentities to be generated. + +#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7) +#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-) +#define CHANNEL_UNUSED_BY_FRIENDS_DELAY (3600*24*60) // Two months. Will be used to delete a channel if too old +#define CHANNEL_DELAY_FOR_CHECKING_AND_DELETING_OLD_GROUPS 300 // check for old channels every 30 mins. Far too often than above delay by RS needs to run it at least once per session -#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7) -#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-) - /********************************************************************************/ /******************* Startup / Tick ******************************************/ /********************************************************************************/ @@ -209,7 +211,7 @@ bool p3GxsChannels::loadList(std::list& loadList) mKnownChannels.clear(); for(auto it(fnr->records.begin());it!=fnr->records.end();++it) - if( now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE) + if(now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE) mKnownChannels.insert(*it) ; } @@ -332,28 +334,36 @@ void p3GxsChannels::notifyChanges(std::vector &changes) ev->mChannelGroupId = grpChange->mGroupId; ev->mChannelEventCode = RsChannelEventCode::STATISTICS_CHANGED; rsEvents->postEvent(ev); - } + + // also update channel usage. Statistics are updated when a friend sends some sync packets + RS_STACK_MUTEX(mKnownChannelsMutex); + mKnownChannels[grpChange->mGroupId] = time(NULL); + IndicateConfigChanged(); + } break; case RsGxsNotify::TYPE_PUBLISHED: case RsGxsNotify::TYPE_RECEIVED_NEW: { - /* group received */ + /* group received or updated */ - RS_STACK_MUTEX(mKnownChannelsMutex); + bool unknown ; + { + RS_STACK_MUTEX(mKnownChannelsMutex); + + unknown = (mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end()); + mKnownChannels[grpChange->mGroupId] = time(NULL); + IndicateConfigChanged(); + } #ifdef GXSCHANNEL_DEBUG RsDbg() << " Type = Published/New " << std::endl; #endif - if(mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end()) + if(unknown) { #ifdef GXSCHANNEL_DEBUG RsDbg() << " Status: unknown. Sending notification event." << std::endl; #endif - - mKnownChannels.insert(std::make_pair(grpChange->mGroupId,time(NULL))) ; - IndicateConfigChanged(); - auto ev = std::make_shared(); ev->mChannelGroupId = grpChange->mGroupId; ev->mChannelEventCode = RsChannelEventCode::NEW_CHANNEL; @@ -431,6 +441,52 @@ void p3GxsChannels::service_tick() } } +bool p3GxsChannels::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) +{ + std::cerr << "p3gxsChannels: Checking unused channel: called by GxsCleaning." << std::endl; + + // request all group infos at once + + rstime_t now = time(nullptr); + + RS_STACK_MUTEX(mKnownChannelsMutex); + + auto it = mKnownChannels.find(meta.mGroupId); + bool unknown_channel = it == mKnownChannels.end(); + + std::cerr << " Channel " << meta.mGroupId ; + + if(unknown_channel) + { + // This case should normally not happen. It does because this channel was never registered since it may + // arrived before this code was here + + std::cerr << ". Not known yet. Adding current time as new TS." << std::endl; + mKnownChannels[meta.mGroupId] = now; + IndicateConfigChanged(); + + return true; + } + else + { + bool used_by_friends = (now < it->second + CHANNEL_UNUSED_BY_FRIENDS_DELAY); + bool subscribed = static_cast(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED); + + std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)"; + + if(!subscribed && !used_by_friends) + { + std::cerr << ". Scheduling for deletion" << std::endl; + return false; + } + else + { + std::cerr << ". Keeping!" << std::endl; + return true; + } + } +} + bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector &groups) { #ifdef GXSCHANNELS_DEBUG diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index fd6eefd51..e353881e2 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -62,6 +62,7 @@ public: protected: + virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override; // see RsGenExchange virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser() virtual bool saveList(bool &cleanup, std::list&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list&) @@ -332,6 +333,7 @@ static uint32_t channelsAuthenPolicy(); void clearUnsubscribedGroup(const RsGxsGroupId &id); bool setAutoDownload(const RsGxsGroupId &groupId, bool enabled); bool autoDownloadEnabled(const RsGxsGroupId &groupId, bool &enabled); + bool checkForOldAndUnusedChannels(); // DUMMY DATA, virtual bool generateDummyData(); @@ -379,6 +381,7 @@ bool generateGroup(uint32_t &token, std::string groupName); RsMutex mKnownChannelsMutex; rstime_t mLastDistantSearchNotificationTS; + std::map > mSearchResultsToNotify; #ifdef TO_REMOVE /** Store search callbacks with timeout*/ diff --git a/libretroshare/src/services/p3gxscircles.cc b/libretroshare/src/services/p3gxscircles.cc index 96f2db03d..931b82c7d 100644 --- a/libretroshare/src/services/p3gxscircles.cc +++ b/libretroshare/src/services/p3gxscircles.cc @@ -40,7 +40,6 @@ /**** * #define DEBUG_CIRCLES 1 ****/ - #define DEBUG_CIRCLES 1 /*extern*/ RsGxsCircles* rsGxsCircles = nullptr; diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 1a7b984fc..4d45abef2 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -48,6 +48,7 @@ RsGxsForums *rsGxsForums = NULL; #define FORUM_TESTEVENT_DUMMYDATA 0x0001 #define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated. +#define FORUM_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // unused forums are deleted after 2 months /********************************************************************************/ /******************* Startup / Tick ******************************************/ @@ -145,7 +146,10 @@ bool p3GxsForums::saveList(bool &cleanup, std::list&saveList) RsGxsForumNotifyRecordsItem *item = new RsGxsForumNotifyRecordsItem ; - item->records = mKnownForums ; + { + RS_STACK_MUTEX(mKnownForumsMutex); + item->records = mKnownForums ; + } saveList.push_back(item) ; return true; @@ -164,6 +168,8 @@ bool p3GxsForums::loadList(std::list& loadList) if(fnr != NULL) { + RS_STACK_MUTEX(mKnownForumsMutex); + mKnownForums.clear(); for(auto it(fnr->records.begin());it!=fnr->records.end();++it) @@ -270,13 +276,16 @@ void p3GxsForums::notifyChanges(std::vector &changes) { /* group received */ - RS_STACK_MUTEX(mKnownForumsMutex); + bool unknown; + { + RS_STACK_MUTEX(mKnownForumsMutex); + unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end()); + mKnownForums[grpChange->mGroupId] = time(nullptr); + IndicateConfigChanged(); + } - if(mKnownForums.find(grpChange->mGroupId) == mKnownForums.end()) + if(unknown) { - mKnownForums.insert( std::make_pair(grpChange->mGroupId, time(nullptr))); - IndicateConfigChanged(); - auto ev = std::make_shared(); ev->mForumGroupId = grpChange->mGroupId; ev->mForumEventCode = RsForumEventCode::NEW_FORUM; @@ -295,7 +304,11 @@ void p3GxsForums::notifyChanges(std::vector &changes) ev->mForumGroupId = grpChange->mGroupId; ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED; rsEvents->postEvent(ev); - } + + RS_STACK_MUTEX(mKnownForumsMutex); + mKnownForums[grpChange->mGroupId] = time(nullptr); + IndicateConfigChanged(); + } break; case RsGxsNotify::TYPE_UPDATED: @@ -383,6 +396,51 @@ void p3GxsForums::service_tick() return; } +bool p3GxsForums::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) +{ + std::cerr << "p3gxsForums: Checking unused forums: called by GxsCleaning." << std::endl; + + // request all group infos at once + + rstime_t now = time(nullptr); + + RS_STACK_MUTEX(mKnownForumsMutex); + + auto it = mKnownForums.find(meta.mGroupId); + bool unknown_forum = it == mKnownForums.end(); + + std::cerr << " Forum " << meta.mGroupId ; + + if(unknown_forum) + { + // This case should normally not happen. It does because this forum was never registered since it may + // arrived before this code was here + + std::cerr << ". Not known yet. Adding current time as new TS." << std::endl; + mKnownForums[meta.mGroupId] = now; + IndicateConfigChanged(); + + return true; + } + else + { + bool used_by_friends = (now < it->second + FORUM_UNUSED_BY_FRIENDS_DELAY); + bool subscribed = static_cast(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED); + + std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)"; + + if(!subscribed && !used_by_friends) + { + std::cerr << ". Scheduling for deletion" << std::endl; + return false; + } + else + { + std::cerr << ". Keeping!" << std::endl; + return true; + } + } +} bool p3GxsForums::getGroupData(const uint32_t &token, std::vector &groups) { std::vector grpData; diff --git a/libretroshare/src/services/p3gxsforums.h b/libretroshare/src/services/p3gxsforums.h index 5ddce2ea2..da45afcdc 100644 --- a/libretroshare/src/services/p3gxsforums.h +++ b/libretroshare/src/services/p3gxsforums.h @@ -40,18 +40,19 @@ public: p3GxsForums( RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs); - virtual RsServiceInfo getServiceInfo(); - virtual void service_tick(); + virtual RsServiceInfo getServiceInfo() override; + virtual void service_tick() override; protected: - virtual void notifyChanges(std::vector& changes); + virtual void notifyChanges(std::vector& changes) override; /// Overloaded from RsTickEvent. - virtual void handle_event(uint32_t event_type, const std::string &elabel); + virtual void handle_event(uint32_t event_type, const std::string &elabel) override; - virtual RsSerialiser* setupSerialiser(); // @see p3Config::setupSerialiser() - virtual bool saveList(bool &cleanup, std::list&saveList); // @see p3Config::saveList(bool &cleanup, std::list&) - virtual bool loadList(std::list& loadList); // @see p3Config::loadList(std::list&) + virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser() + virtual bool saveList(bool &cleanup, std::list&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list&) + virtual bool loadList(std::list& loadList) override; // @see p3Config::loadList(std::list&) + virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override; public: /// @see RsGxsForums::createForumV2 bool createForumV2( @@ -78,22 +79,22 @@ public: /// @see RsGxsForums::createForum @deprecated RS_DEPRECATED_FOR(createForumV2) - virtual bool createForum(RsGxsForumGroup& forum); + virtual bool createForum(RsGxsForumGroup& forum) override; /// @see RsGxsForums::createMessage @deprecated RS_DEPRECATED_FOR(createPost) - virtual bool createMessage(RsGxsForumMsg& message); + virtual bool createMessage(RsGxsForumMsg& message) override; /// @see RsGxsForums::editForum virtual bool editForum(RsGxsForumGroup& forum) override; /// @see RsGxsForums::getForumsSummaries - virtual bool getForumsSummaries(std::list& forums); + virtual bool getForumsSummaries(std::list& forums) override; /// @see RsGxsForums::getForumsInfo virtual bool getForumsInfo( const std::list& forumIds, - std::vector& forumsInfo ); + std::vector& forumsInfo ) override; /// Implementation of @see RsGxsForums::getForumStatistics bool getForumStatistics(const RsGxsGroupId& ForumId,GxsGroupStatistic& stat) override; @@ -102,20 +103,20 @@ public: bool getForumServiceStatistics(GxsServiceStatistic& stat) override; /// @see RsGxsForums::getForumMsgMetaData - virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector& msg_metas) ; + virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector& msg_metas) override; /// @see RsGxsForums::getForumContent virtual bool getForumContent( const RsGxsGroupId& forumId, const std::set& msgs_to_request, - std::vector& msgs ); + std::vector& msgs ) override; /// @see RsGxsForums::markRead - virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read); + virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read) override; /// @see RsGxsForums::subscribeToForum virtual bool subscribeToForum( const RsGxsGroupId& forumId, - bool subscribe ); + bool subscribe ) override; /// @see RsGxsForums bool exportForumLink( @@ -140,7 +141,7 @@ public: /// @see RsGxsForums std::error_condition setPostKeepForever( const RsGxsGroupId& forumId, const RsGxsMessageId& postId, - bool keepForever ); + bool keepForever ) override; /// implementation of rsGxsGorums /// diff --git a/libretroshare/src/services/p3postbase.cc b/libretroshare/src/services/p3postbase.cc index f02ca7f75..bfd160672 100644 --- a/libretroshare/src/services/p3postbase.cc +++ b/libretroshare/src/services/p3postbase.cc @@ -48,13 +48,18 @@ #define POSTBASE_UNPROCESSED_MSGS 0x0012 #define POSTBASE_ALL_MSGS 0x0013 #define POSTBASE_BG_POST_META 0x0014 + +#define POSTED_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // delete unused posted groups after 2 months + /********************************************************************************/ /******************* Startup / Tick ******************************************/ /********************************************************************************/ p3PostBase::p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs, RsSerialType* serviceSerialiser, uint16_t serviceType) - : RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(), mPostBaseMtx("PostBaseMtx") + : RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(), + mPostBaseMtx("PostBaseMutex"), + mKnownPostedMutex("PostBaseKnownPostedMutex") { mBgProcessing = false; @@ -185,6 +190,10 @@ void p3PostBase::notifyChanges(std::vector &changes) ev->mPostedGroupId = group_id; ev->mPostedEventCode = RsPostedEventCode::STATISTICS_CHANGED; rsEvents->postEvent(ev); + + RS_STACK_MUTEX(mKnownPostedMutex); + mKnownPosted[group_id] = time(nullptr); + IndicateConfigChanged(); } break; @@ -193,11 +202,16 @@ void p3PostBase::notifyChanges(std::vector &changes) { /* group received */ - if(mKnownPosted.find(group_id) == mKnownPosted.end()) + bool unknown; { - mKnownPosted.insert(std::make_pair(group_id, time(nullptr))); - IndicateConfigChanged(); + RS_STACK_MUTEX(mKnownPostedMutex); + unknown = (mKnownPosted.find(grpChange->mGroupId) == mKnownPosted.end()); + mKnownPosted[group_id] = time(nullptr); + IndicateConfigChanged(); + } + if(unknown) + { auto ev = std::make_shared(); ev->mPostedGroupId = group_id; ev->mPostedEventCode = RsPostedEventCode::NEW_POSTED_GROUP; @@ -883,13 +897,62 @@ public: } }; -bool p3PostBase::saveList(bool &cleanup, std::list&saveList) +bool p3PostBase::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) +{ + std::cerr << "p3gxsChannels: Checking unused board: called by GxsCleaning." << std::endl; + + // request all group infos at once + + rstime_t now = time(nullptr); + + RS_STACK_MUTEX(mKnownPostedMutex); + + auto it = mKnownPosted.find(meta.mGroupId); + bool unknown_posted = (it == mKnownPosted.end()); + + std::cerr << " Board " << meta.mGroupId ; + + if(unknown_posted) + { + // This case should normally not happen. It does because this board was never registered since it may + // arrived before this code was here + + std::cerr << ". Not known yet. Adding current time as new TS." << std::endl; + mKnownPosted[meta.mGroupId] = now; + IndicateConfigChanged(); + + return true; + } + else + { + bool used_by_friends = (now < it->second + POSTED_UNUSED_BY_FRIENDS_DELAY); + bool subscribed = static_cast(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED); + + std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)"; + + if(!subscribed && !used_by_friends) + { + std::cerr << ". Scheduling for deletion" << std::endl; + return false; + } + else + { + std::cerr << ". Keeping!" << std::endl; + return true; + } + } +} + +bool p3PostBase::saveList(bool& cleanup, std::list&saveList) { cleanup = true ; RsGxsPostedNotifyRecordsItem *item = new RsGxsPostedNotifyRecordsItem ; - item->records = mKnownPosted ; + { + RS_STACK_MUTEX(mKnownPostedMutex); + item->records = mKnownPosted ; + } saveList.push_back(item) ; return true; @@ -908,6 +971,8 @@ bool p3PostBase::loadList(std::list& loadList) if(fnr != NULL) { + RS_STACK_MUTEX(mKnownPostedMutex); + mKnownPosted.clear(); for(auto it(fnr->records.begin());it!=fnr->records.end();++it) diff --git a/libretroshare/src/services/p3postbase.h b/libretroshare/src/services/p3postbase.h index 40804be1f..864d57206 100644 --- a/libretroshare/src/services/p3postbase.h +++ b/libretroshare/src/services/p3postbase.h @@ -70,23 +70,24 @@ public: p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs, RsSerialType* serviceSerialiser, uint16_t serviceType); - virtual void service_tick(); + virtual void service_tick() override; protected: - virtual void notifyChanges(std::vector& changes); + virtual void notifyChanges(std::vector& changes) override; // Overloaded from GxsTokenQueue for Request callbacks. - virtual void handleResponse(uint32_t token, uint32_t req_type); + virtual void handleResponse(uint32_t token, uint32_t req_type) override; // Overloaded from RsTickEvent. - virtual void handle_event(uint32_t event_type, const std::string &elabel); + virtual void handle_event(uint32_t event_type, const std::string &elabel) override; // overloads p3Config virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser() virtual bool saveList(bool &cleanup, std::list&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list&) virtual bool loadList(std::list& loadList) override; // @see p3Config::loadList(std::list&) + virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override; public: ////////////////////////////////////////////////////////////////////////////// @@ -122,6 +123,7 @@ private: RsMutex mPostBaseMtx; + RsMutex mKnownPostedMutex; bool mBgProcessing; bool mBgIncremental; diff --git a/libretroshare/src/services/p3posted.cc b/libretroshare/src/services/p3posted.cc index 251bf10cd..b38adbcec 100644 --- a/libretroshare/src/services/p3posted.cc +++ b/libretroshare/src/services/p3posted.cc @@ -49,6 +49,9 @@ const uint16_t GXS_POSTED_APP_MINOR_VERSION = 0; const uint16_t GXS_POSTED_MIN_MAJOR_VERSION = 1; const uint16_t GXS_POSTED_MIN_MINOR_VERSION = 0; +static const uint32_t GXS_POSTED_CONFIG_MAX_TIME_NOTIFY_STORAGE = 86400*30*2 ; // ignore notifications for 2 months +static const uint8_t GXS_POSTED_CONFIG_SUBTYPE_NOTIFY_RECORD = 0x01 ; + RsServiceInfo p3Posted::getServiceInfo() { return RsServiceInfo(RS_SERVICE_GXS_TYPE_POSTED, diff --git a/libretroshare/src/services/p3posted.h b/libretroshare/src/services/p3posted.h index 8f71a97a1..08eb0b2d7 100644 --- a/libretroshare/src/services/p3posted.h +++ b/libretroshare/src/services/p3posted.h @@ -92,19 +92,20 @@ virtual void receiveHelperChanges(std::vector& changes) virtual bool getPostData(const uint32_t &token, std::vector &posts, std::vector &cmts, std::vector &vots) override; virtual bool getPostData(const uint32_t &token, std::vector &posts, std::vector &cmts) override; virtual bool getPostData(const uint32_t &token, std::vector &posts) override; + //Not currently used //virtual bool getRelatedPosts(const uint32_t &token, std::vector &posts); -virtual bool createGroup(uint32_t &token, RsPostedGroup &group); -virtual bool createPost(uint32_t &token, RsPostedPost &post); +virtual bool createGroup(uint32_t &token, RsPostedGroup &group) override; +virtual bool createPost(uint32_t &token, RsPostedPost &post) override; -virtual bool updateGroup(uint32_t &token, RsPostedGroup &group); -virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set& peers); +virtual bool updateGroup(uint32_t &token, RsPostedGroup &group) override; +virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set& peers) override; ////////////////////////////////////////////////////////////////////////////// // WRAPPERS due to the separate Interface. -virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) +virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) override { return p3PostBase::setMessageReadStatus(token, msgId, read); } @@ -112,11 +113,11 @@ virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgI /** Comment service - Provide RsGxsCommentService - * redirect to p3GxsCommentService */ - virtual bool getCommentData(uint32_t token, std::vector &msgs) + virtual bool getCommentData(uint32_t token, std::vector &msgs) override { return mCommentService->getGxsCommentData(token, msgs); } virtual bool getRelatedComments( uint32_t token, - std::vector &msgs ) + std::vector &msgs ) override { return mCommentService->getGxsRelatedComments(token, msgs); } virtual bool createNewComment(uint32_t &token, const RsGxsComment &msg) override