Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

added auto-cleaning of unused forums, channels, posted. Works for all GXS groups if the correct method is supplied.

Browse Source
This commit is contained in:
csoler 2020-11-22 21:45:21 +01:00
parent c1c6d33add
commit 2fd15134c9
17 changed files with 363 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libretroshare/src/gxs/rsgds.h
View File

@ -159,6 +159,8 @@ public:
/*!
* Retrieves meta data of all groups stored (most current versions only)
* Memory is owned by the service, not the caller. Therefore the pointers in the temporary map
* shouldn't be destroyed.
*
* @param grp if null grpIds entries are made, only meta for those grpId are retrieved \n
* , if grpId is failed to be retrieved it will be erased from map

14
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -131,7 +131,8 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key
// |
// +--- processRoutingClues() ;
//
static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes
//static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes
static const uint32_t MSG_CLEANUP_PERIOD = 60*5; // 59 minutes
static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes
RsGenExchange::RsGenExchange(
@ -257,10 +258,16 @@ void RsGenExchange::tick()
rstime_t now = time(NULL);
if((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning)
// Cleanup unused data. This is only needed when auto-synchronization is needed, which is not the case
// of identities. This is why idendities do their own cleaning.
if((mNetService->msgAutoSync() || mNetService->grpAutoSync())
&& ((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning))
{
if(mMsgCleanUp)
{
RS_STACK_MUTEX(mGenMtx);
if(mMsgCleanUp->clean())
{
mCleaning = false;
@ -268,11 +275,10 @@ void RsGenExchange::tick()
mMsgCleanUp = NULL;
mLastClean = time(NULL);
}
}
else
{
mMsgCleanUp = new RsGxsMessageCleanUp(mDataStore, this, 1);
mMsgCleanUp = new RsGxsCleanUp(mDataStore, this, 1);
mCleaning = true;
}
}

17
libretroshare/src/gxs/rsgenexchange.h
View File

@ -648,6 +648,19 @@ protected:
*/
virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
/*!
* \brief service_checkIfGroupIsStillUsed
* Re-implement this function to help GXS cleaning, by telling that some particular group
* is not used anymore. This usually depends on subscription, the fact that friend nodes send
* some info or not, and particular cleaning strategy of each service.
* Besides, groups in some services are used by other services (e.g. identities, circles, are used in
* forums and so on), so deciding on a group usage can only be left to the specific service it is used in.
* \return
* true if the group is still used, false otherwise, meaning that the group can be deleted. Default is
* that the group is always in use.
*/
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& /* meta */) { return true; } // see RsGenExchange
public:
/*!
@ -959,7 +972,7 @@ private:
bool mCleaning;
rstime_t mLastClean;
RsGxsMessageCleanUp* mMsgCleanUp;
RsGxsCleanUp* mMsgCleanUp;
bool mChecking, mCheckStarted;
@ -982,6 +995,8 @@ private:
std::vector<MsgDeletePublish> mMsgDeletePublish;
std::map<RsGxsId,std::set<RsPeerId> > mRoutingClues ;
friend class RsGxsCleanUp;
};
#endif // RSGENEXCHANGE_H

2
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -128,6 +128,8 @@ public:
virtual void setDefaultKeepAge(uint32_t t) override { mDefaultMsgStorePeriod = t ; }
virtual void setDefaultSyncAge(uint32_t t) override { mDefaultMsgSyncPeriod = t ; }
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.

148
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -35,15 +35,19 @@
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!)
//#define DEBUG_GXSUTIL 1
#define DEBUG_GXSUTIL 1
#ifdef DEBUG_GXSUTIL
#define GXSUTIL_DEBUG() std::cerr << "[" << time(NULL) << "] : GXS_UTIL : " << __FUNCTION__ << " : "
#endif
RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize)
RsGxsCleanUp::RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize)
: mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize)
{
RsGxsGrpMetaTemporaryMap grpMeta;
@ -53,96 +57,111 @@ RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService
mGrpMeta.push_back(cit->second);
}
bool RsGxsMessageCleanUp::clean()
bool RsGxsCleanUp::clean()
{
uint32_t i = 1;
uint32_t i = 1;
rstime_t now = time(NULL);
rstime_t now = time(NULL);
std::vector<RsGxsGroupId> grps_to_delete;
#ifdef DEBUG_GXSUTIL
uint16_t service_type = mGenExchangeClient->serviceType() ;
GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << std::endl;
uint16_t service_type = mGenExchangeClient->serviceType() ;
GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << std::endl;
#endif
while(!mGrpMeta.empty())
{
const RsGxsGrpMetaData* grpMeta = mGrpMeta.back();
const RsGxsGroupId& grpId = grpMeta->mGroupId;
mGrpMeta.pop_back();
GxsMsgReq req;
GxsMsgMetaResult result;
while(!mGrpMeta.empty())
{
const RsGxsGrpMetaData* grpMeta = mGrpMeta.back();
req[grpId] = std::set<RsGxsMessageId>();
mDs->retrieveGxsMsgMetaData(req, result);
// first check if we keep the group or not
GxsMsgMetaResult::iterator mit = result.begin();
if(!mGenExchangeClient->service_checkIfGroupIsStillUsed(*grpMeta))
{
#ifdef DEBUG_GXSUTIL
std::cerr << " Scheduling group " << grpMeta->mGroupId << " for removal." << std::endl;
#endif
grps_to_delete.push_back(grpMeta->mGroupId);
}
else
{
const RsGxsGroupId& grpId = grpMeta->mGroupId;
mGrpMeta.pop_back();
GxsMsgReq req;
GxsMsgMetaResult result;
req[grpId] = std::set<RsGxsMessageId>();
mDs->retrieveGxsMsgMetaData(req, result);
GxsMsgMetaResult::iterator mit = result.begin();
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl;
GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl;
#endif
req.clear();
GxsMsgReq messages_to_delete;
uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ;
uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ;
for(; mit != result.end(); ++mit)
{
std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
for(; mit != result.end(); ++mit)
{
std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
// First, make a map of which message have a child message. This allows to only delete messages that dont have child messages.
// A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the
// end the message tree will be deleted slice after slice, which should still be reasonnably fast.
//
std::set<RsGxsMessageId> messages_with_kids ;
// First, make a map of which message have a child message. This allows to only delete messages that dont have child messages.
// A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the
// end the message tree will be deleted slice after slice, which should still be reasonnably fast.
//
std::set<RsGxsMessageId> messages_with_kids ;
for( uint32_t i=0;i<metaV.size();++i)
if(!metaV[i]->mParentId.isNull())
messages_with_kids.insert(metaV[i]->mParentId) ;
for( uint32_t i=0;i<metaV.size();++i)
if(!metaV[i]->mParentId.isNull())
messages_with_kids.insert(metaV[i]->mParentId) ;
for( uint32_t i=0;i<metaV.size();++i)
{
const RsGxsMsgMetaData* meta = metaV[i];
for( uint32_t i=0;i<metaV.size();++i)
{
const RsGxsMsgMetaData* meta = metaV[i];
bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end());
bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end());
// check if expired
bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids;
// check if expired
bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids;
// check client does not want the message kept regardless of age
remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER);
// check client does not want the message kept regardless of age
remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER);
// if not subscribed remove messages (can optimise this really)
remove = remove || (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED);
remove = remove || !(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
// if not subscribed remove messages (can optimise this really)
remove = remove || (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED);
remove = remove || !(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP)
<< " subscribed: " << bool(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period
<< " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ;
GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER)
<< " subscribed: " << bool(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period
<< " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ;
#endif
if( remove )
{
req[grpId].insert(meta->mMsgId);
if( remove )
{
messages_to_delete[grpId].insert(meta->mMsgId);
#ifdef DEBUG_GXSUTIL
std::cerr << " Scheduling for removal." << std::endl;
std::cerr << " Scheduling for removal." << std::endl;
#endif
}
}
#ifdef DEBUG_GXSUTIL
else
std::cerr << std::endl;
else
std::cerr << std::endl;
#endif
//delete meta;
}
}
//delete meta;
}
}
mDs->removeMsgs(messages_to_delete);
mDs->removeMsgs(req);
i++;
if(i > CHUNK_SIZE) break;
}
}
i++;
if(i > CHUNK_SIZE) break;
}
//mDs->removeGroups(grps_to_delete);
return mGrpMeta.empty();
return mGrpMeta.empty();
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
@ -223,6 +242,8 @@ bool RsGxsIntegrityCheck::check()
else msgIds.erase(msgIds.find(grp->grpId));
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
@ -359,8 +380,9 @@ bool RsGxsIntegrityCheck::check()
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
if( isGxsChannels
&& indexedGroups.count(msg->metaData->mGroupId) )
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);

4
libretroshare/src/gxs/rsgxsutil.h
View File

@ -125,7 +125,7 @@ inline RsGxsGrpMsgIdPair getMsgIdPair(RsGxsMsgItem& msg)
* Does message clean up based on individual group expirations first
* if avialable. If not then deletion s
*/
class RsGxsMessageCleanUp
class RsGxsCleanUp
{
public:
@ -136,7 +136,7 @@ public:
* @param chunkSize
* @param sleepPeriod
*/
RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize);
RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize);
/*!
* On construction this should be called to progress deletions

3
libretroshare/src/gxs/rsnxs.h
View File

@ -82,6 +82,9 @@ public:
virtual uint32_t getDefaultSyncAge() =0;
virtual uint32_t getDefaultKeepAge() =0;
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////

3
libretroshare/src/retroshare/rsgxsflags.h
View File

@ -107,8 +107,7 @@ namespace GXS_SERV {
static const uint32_t GXS_MSG_STATUS_UNPROCESSED = 0x00000001; // Flags to store the read/process status of group messages.
static const uint32_t GXS_MSG_STATUS_GUI_UNREAD = 0x00000002; // The actual meaning may depend on the type of service.
static const uint32_t GXS_MSG_STATUS_GUI_NEW = 0x00000004; //
/** Do not delete message even if older then group maximum storage time */
static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008;
static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008; // Do not delete message even if older then group maximum storage time
static const uint32_t GXS_MSG_STATUS_DELETE = 0x00000020; //
/** END GXS Msg status flags **/

82
libretroshare/src/services/p3gxschannels.cc
View File

@ -64,11 +64,13 @@
#define CHANNEL_PROCESS 0x0001
#define CHANNEL_TESTEVENT_DUMMYDATA 0x0002
#define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated.
#define DUMMYDATA_PERIOD 60 // Long enough for some RsIdentities to be generated.
#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7)
#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-)
#define CHANNEL_UNUSED_BY_FRIENDS_DELAY (3600*24*60) // Two months. Will be used to delete a channel if too old
#define CHANNEL_DELAY_FOR_CHECKING_AND_DELETING_OLD_GROUPS 300 // check for old channels every 30 mins. Far too often than above delay by RS needs to run it at least once per session
#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7)
#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-)
/********************************************************************************/
/******************* Startup / Tick ******************************************/
/********************************************************************************/
@ -209,7 +211,7 @@ bool p3GxsChannels::loadList(std::list<RsItem *>& loadList)
mKnownChannels.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)
if( now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE)
if(now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE)
mKnownChannels.insert(*it) ;
}
@ -332,28 +334,36 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mChannelGroupId = grpChange->mGroupId;
ev->mChannelEventCode = RsChannelEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
}
// also update channel usage. Statistics are updated when a friend sends some sync packets
RS_STACK_MUTEX(mKnownChannelsMutex);
mKnownChannels[grpChange->mGroupId] = time(NULL);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_PUBLISHED:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
/* group received */
/* group received or updated */
RS_STACK_MUTEX(mKnownChannelsMutex);
bool unknown ;
{
RS_STACK_MUTEX(mKnownChannelsMutex);
unknown = (mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end());
mKnownChannels[grpChange->mGroupId] = time(NULL);
IndicateConfigChanged();
}
#ifdef GXSCHANNEL_DEBUG
RsDbg() << " Type = Published/New " << std::endl;
#endif
if(mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end())
if(unknown)
{
#ifdef GXSCHANNEL_DEBUG
RsDbg() << " Status: unknown. Sending notification event." << std::endl;
#endif
mKnownChannels.insert(std::make_pair(grpChange->mGroupId,time(NULL))) ;
IndicateConfigChanged();
auto ev = std::make_shared<RsGxsChannelEvent>();
ev->mChannelGroupId = grpChange->mGroupId;
ev->mChannelEventCode = RsChannelEventCode::NEW_CHANNEL;
@ -431,6 +441,52 @@ void p3GxsChannels::service_tick()
}
}
bool p3GxsChannels::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
std::cerr << "p3gxsChannels: Checking unused channel: called by GxsCleaning." << std::endl;
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownChannelsMutex);
auto it = mKnownChannels.find(meta.mGroupId);
bool unknown_channel = it == mKnownChannels.end();
std::cerr << " Channel " << meta.mGroupId ;
if(unknown_channel)
{
// This case should normally not happen. It does because this channel was never registered since it may
// arrived before this code was here
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
mKnownChannels[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + CHANNEL_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
if(!subscribed && !used_by_friends)
{
std::cerr << ". Scheduling for deletion" << std::endl;
return false;
}
else
{
std::cerr << ". Keeping!" << std::endl;
return true;
}
}
}
bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups)
{
#ifdef GXSCHANNELS_DEBUG

3
libretroshare/src/services/p3gxschannels.h
View File

@ -62,6 +62,7 @@ public:
protected:
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override; // see RsGenExchange
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
@ -332,6 +333,7 @@ static uint32_t channelsAuthenPolicy();
void clearUnsubscribedGroup(const RsGxsGroupId &id);
bool setAutoDownload(const RsGxsGroupId &groupId, bool enabled);
bool autoDownloadEnabled(const RsGxsGroupId &groupId, bool &enabled);
bool checkForOldAndUnusedChannels();
// DUMMY DATA,
virtual bool generateDummyData();
@ -379,6 +381,7 @@ bool generateGroup(uint32_t &token, std::string groupName);
RsMutex mKnownChannelsMutex;
rstime_t mLastDistantSearchNotificationTS;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
#ifdef TO_REMOVE
/** Store search callbacks with timeout*/

1
libretroshare/src/services/p3gxscircles.cc
View File

@ -40,7 +40,6 @@
/****
* #define DEBUG_CIRCLES 1
****/
#define DEBUG_CIRCLES 1
/*extern*/ RsGxsCircles* rsGxsCircles = nullptr;

72
libretroshare/src/services/p3gxsforums.cc
View File

@ -48,6 +48,7 @@ RsGxsForums *rsGxsForums = NULL;
#define FORUM_TESTEVENT_DUMMYDATA 0x0001
#define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated.
#define FORUM_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // unused forums are deleted after 2 months
/********************************************************************************/
/******************* Startup / Tick ******************************************/
@ -145,7 +146,10 @@ bool p3GxsForums::saveList(bool &cleanup, std::list<RsItem *>&saveList)
RsGxsForumNotifyRecordsItem *item = new RsGxsForumNotifyRecordsItem ;
item->records = mKnownForums ;
{
RS_STACK_MUTEX(mKnownForumsMutex);
item->records = mKnownForums ;
}
saveList.push_back(item) ;
return true;
@ -164,6 +168,8 @@ bool p3GxsForums::loadList(std::list<RsItem *>& loadList)
if(fnr != NULL)
{
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)
@ -270,13 +276,16 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
{
/* group received */
RS_STACK_MUTEX(mKnownForumsMutex);
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(mKnownForums.find(grpChange->mGroupId) == mKnownForums.end())
if(unknown)
{
mKnownForums.insert( std::make_pair(grpChange->mGroupId, time(nullptr)));
IndicateConfigChanged();
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
@ -295,7 +304,11 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
}
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_UPDATED:
@ -383,6 +396,51 @@ void p3GxsForums::service_tick()
return;
}
bool p3GxsForums::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
std::cerr << "p3gxsForums: Checking unused forums: called by GxsCleaning." << std::endl;
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownForumsMutex);
auto it = mKnownForums.find(meta.mGroupId);
bool unknown_forum = it == mKnownForums.end();
std::cerr << " Forum " << meta.mGroupId ;
if(unknown_forum)
{
// This case should normally not happen. It does because this forum was never registered since it may
// arrived before this code was here
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
mKnownForums[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + FORUM_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
if(!subscribed && !used_by_friends)
{
std::cerr << ". Scheduling for deletion" << std::endl;
return false;
}
else
{
std::cerr << ". Keeping!" << std::endl;
return true;
}
}
}
bool p3GxsForums::getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups)
{
std::vector<RsGxsGrpItem*> grpData;

33
libretroshare/src/services/p3gxsforums.h
View File

@ -40,18 +40,19 @@ public:
p3GxsForums(
RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs);
virtual RsServiceInfo getServiceInfo();
virtual void service_tick();
virtual RsServiceInfo getServiceInfo() override;
virtual void service_tick() override;
protected:
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) override;
/// Overloaded from RsTickEvent.
virtual void handle_event(uint32_t event_type, const std::string &elabel);
virtual void handle_event(uint32_t event_type, const std::string &elabel) override;
virtual RsSerialiser* setupSerialiser(); // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList); // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList); // @see p3Config::loadList(std::list<RsItem *>&)
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override;
public:
/// @see RsGxsForums::createForumV2
bool createForumV2(
@ -78,22 +79,22 @@ public:
/// @see RsGxsForums::createForum @deprecated
RS_DEPRECATED_FOR(createForumV2)
virtual bool createForum(RsGxsForumGroup& forum);
virtual bool createForum(RsGxsForumGroup& forum) override;
/// @see RsGxsForums::createMessage @deprecated
RS_DEPRECATED_FOR(createPost)
virtual bool createMessage(RsGxsForumMsg& message);
virtual bool createMessage(RsGxsForumMsg& message) override;
/// @see RsGxsForums::editForum
virtual bool editForum(RsGxsForumGroup& forum) override;
/// @see RsGxsForums::getForumsSummaries
virtual bool getForumsSummaries(std::list<RsGroupMetaData>& forums);
virtual bool getForumsSummaries(std::list<RsGroupMetaData>& forums) override;
/// @see RsGxsForums::getForumsInfo
virtual bool getForumsInfo(
const std::list<RsGxsGroupId>& forumIds,
std::vector<RsGxsForumGroup>& forumsInfo );
std::vector<RsGxsForumGroup>& forumsInfo ) override;
/// Implementation of @see RsGxsForums::getForumStatistics
bool getForumStatistics(const RsGxsGroupId& ForumId,GxsGroupStatistic& stat) override;
@ -102,20 +103,20 @@ public:
bool getForumServiceStatistics(GxsServiceStatistic& stat) override;
/// @see RsGxsForums::getForumMsgMetaData
virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector<RsMsgMetaData>& msg_metas) ;
virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector<RsMsgMetaData>& msg_metas) override;
/// @see RsGxsForums::getForumContent
virtual bool getForumContent(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& msgs_to_request,
std::vector<RsGxsForumMsg>& msgs );
std::vector<RsGxsForumMsg>& msgs ) override;
/// @see RsGxsForums::markRead
virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read);
virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read) override;
/// @see RsGxsForums::subscribeToForum
virtual bool subscribeToForum( const RsGxsGroupId& forumId,
bool subscribe );
bool subscribe ) override;
/// @see RsGxsForums
bool exportForumLink(
@ -140,7 +141,7 @@ public:
/// @see RsGxsForums
std::error_condition setPostKeepForever(
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever );
bool keepForever ) override;
/// implementation of rsGxsGorums
///

77
libretroshare/src/services/p3postbase.cc
View File

@ -48,13 +48,18 @@
#define POSTBASE_UNPROCESSED_MSGS 0x0012
#define POSTBASE_ALL_MSGS 0x0013
#define POSTBASE_BG_POST_META 0x0014
#define POSTED_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // delete unused posted groups after 2 months
/********************************************************************************/
/******************* Startup / Tick ******************************************/
/********************************************************************************/
p3PostBase::p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs,
RsSerialType* serviceSerialiser, uint16_t serviceType)
: RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(), mPostBaseMtx("PostBaseMtx")
: RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(),
mPostBaseMtx("PostBaseMutex"),
mKnownPostedMutex("PostBaseKnownPostedMutex")
{
mBgProcessing = false;
@ -185,6 +190,10 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mPostedGroupId = group_id;
ev->mPostedEventCode = RsPostedEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownPostedMutex);
mKnownPosted[group_id] = time(nullptr);
IndicateConfigChanged();
}
break;
@ -193,11 +202,16 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
{
/* group received */
if(mKnownPosted.find(group_id) == mKnownPosted.end())
bool unknown;
{
mKnownPosted.insert(std::make_pair(group_id, time(nullptr)));
IndicateConfigChanged();
RS_STACK_MUTEX(mKnownPostedMutex);
unknown = (mKnownPosted.find(grpChange->mGroupId) == mKnownPosted.end());
mKnownPosted[group_id] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsPostedEvent>();
ev->mPostedGroupId = group_id;
ev->mPostedEventCode = RsPostedEventCode::NEW_POSTED_GROUP;
@ -883,13 +897,62 @@ public:
}
};
bool p3PostBase::saveList(bool &cleanup, std::list<RsItem *>&saveList)
bool p3PostBase::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
std::cerr << "p3gxsChannels: Checking unused board: called by GxsCleaning." << std::endl;
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownPostedMutex);
auto it = mKnownPosted.find(meta.mGroupId);
bool unknown_posted = (it == mKnownPosted.end());
std::cerr << " Board " << meta.mGroupId ;
if(unknown_posted)
{
// This case should normally not happen. It does because this board was never registered since it may
// arrived before this code was here
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
mKnownPosted[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + POSTED_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
if(!subscribed && !used_by_friends)
{
std::cerr << ". Scheduling for deletion" << std::endl;
return false;
}
else
{
std::cerr << ". Keeping!" << std::endl;
return true;
}
}
}
bool p3PostBase::saveList(bool& cleanup, std::list<RsItem *>&saveList)
{
cleanup = true ;
RsGxsPostedNotifyRecordsItem *item = new RsGxsPostedNotifyRecordsItem ;
item->records = mKnownPosted ;
{
RS_STACK_MUTEX(mKnownPostedMutex);
item->records = mKnownPosted ;
}
saveList.push_back(item) ;
return true;
@ -908,6 +971,8 @@ bool p3PostBase::loadList(std::list<RsItem *>& loadList)
if(fnr != NULL)
{
RS_STACK_MUTEX(mKnownPostedMutex);
mKnownPosted.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)

10
libretroshare/src/services/p3postbase.h
View File

@ -70,23 +70,24 @@ public:
p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs,
RsSerialType* serviceSerialiser, uint16_t serviceType);
virtual void service_tick();
virtual void service_tick() override;
protected:
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) override;
// Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type);
virtual void handleResponse(uint32_t token, uint32_t req_type) override;
// Overloaded from RsTickEvent.
virtual void handle_event(uint32_t event_type, const std::string &elabel);
virtual void handle_event(uint32_t event_type, const std::string &elabel) override;
// overloads p3Config
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override;
public:
//////////////////////////////////////////////////////////////////////////////
@ -122,6 +123,7 @@ private:
RsMutex mPostBaseMtx;
RsMutex mKnownPostedMutex;
bool mBgProcessing;
bool mBgIncremental;

3
libretroshare/src/services/p3posted.cc
View File

@ -49,6 +49,9 @@ const uint16_t GXS_POSTED_APP_MINOR_VERSION = 0;
const uint16_t GXS_POSTED_MIN_MAJOR_VERSION = 1;
const uint16_t GXS_POSTED_MIN_MINOR_VERSION = 0;
static const uint32_t GXS_POSTED_CONFIG_MAX_TIME_NOTIFY_STORAGE = 86400*30*2 ; // ignore notifications for 2 months
static const uint8_t GXS_POSTED_CONFIG_SUBTYPE_NOTIFY_RECORD = 0x01 ;
RsServiceInfo p3Posted::getServiceInfo()
{
return RsServiceInfo(RS_SERVICE_GXS_TYPE_POSTED,

15
libretroshare/src/services/p3posted.h
View File

@ -92,19 +92,20 @@ virtual void receiveHelperChanges(std::vector<RsGxsNotify*>& changes)
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts, std::vector<RsGxsComment> &cmts, std::vector<RsGxsVote> &vots) override;
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts, std::vector<RsGxsComment> &cmts) override;
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts) override;
//Not currently used
//virtual bool getRelatedPosts(const uint32_t &token, std::vector<RsPostedPost> &posts);
virtual bool createGroup(uint32_t &token, RsPostedGroup &group);
virtual bool createPost(uint32_t &token, RsPostedPost &post);
virtual bool createGroup(uint32_t &token, RsPostedGroup &group) override;
virtual bool createPost(uint32_t &token, RsPostedPost &post) override;
virtual bool updateGroup(uint32_t &token, RsPostedGroup &group);
virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set<RsPeerId>& peers);
virtual bool updateGroup(uint32_t &token, RsPostedGroup &group) override;
virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set<RsPeerId>& peers) override;
//////////////////////////////////////////////////////////////////////////////
// WRAPPERS due to the separate Interface.
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read)
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) override
{
return p3PostBase::setMessageReadStatus(token, msgId, read);
}
@ -112,11 +113,11 @@ virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgI
/** Comment service - Provide RsGxsCommentService -
* redirect to p3GxsCommentService */
virtual bool getCommentData(uint32_t token, std::vector<RsGxsComment> &msgs)
virtual bool getCommentData(uint32_t token, std::vector<RsGxsComment> &msgs) override
{ return mCommentService->getGxsCommentData(token, msgs); }
virtual bool getRelatedComments( uint32_t token,
std::vector<RsGxsComment> &msgs )
std::vector<RsGxsComment> &msgs ) override
{ return mCommentService->getGxsRelatedComments(token, msgs); }
virtual bool createNewComment(uint32_t &token, const RsGxsComment &msg) override