Merge pull request #2123 from csoler/v0.6-BugFixing_4

GXS cleaning
This commit is contained in:
csoler 2020-11-30 21:42:14 +01:00 committed by GitHub
commit 2bb03e53d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 763 additions and 248 deletions

View File

@ -159,6 +159,8 @@ public:
/*!
* Retrieves meta data of all groups stored (most current versions only)
* Memory is owned by the service, not the caller. Therefore the pointers in the temporary map
* shouldn't be destroyed.
*
* @param grp if null grpIds entries are made, only meta for those grpId are retrieved \n
* , if grpId is failed to be retrieved it will be erased from map

View File

@ -131,6 +131,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key
// |
// +--- processRoutingClues() ;
//
static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes
static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes
@ -147,7 +148,6 @@ RsGenExchange::RsGenExchange(
mAuthenPolicy(authenPolicy),
mCleaning(false),
mLastClean((int)time(NULL) - (int)(RSRandom::random_u32() % MSG_CLEANUP_PERIOD)), // this helps unsynchronising the checks for the different services
mMsgCleanUp(NULL),
mChecking(false),
mCheckStarted(false),
mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD) + 120), // this helps unsynchronising the checks for the different services, with 2 min security to avoid checking right away before statistics come up.
@ -257,27 +257,30 @@ void RsGenExchange::tick()
rstime_t now = time(NULL);
if((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning)
// Cleanup unused data. This is only needed when auto-synchronization is needed, which is not the case
// of identities. This is why idendities do their own cleaning.
now = time(NULL);
if( (mNetService && (mNetService->msgAutoSync() || mNetService->grpAutoSync())) && (mLastClean + MSG_CLEANUP_PERIOD < now) )
{
if(mMsgCleanUp)
{
if(mMsgCleanUp->clean())
{
mCleaning = false;
delete mMsgCleanUp;
mMsgCleanUp = NULL;
mLastClean = time(NULL);
}
GxsMsgReq msgs_to_delete;
std::vector<RsGxsGroupId> grps_to_delete;
}
else
{
mMsgCleanUp = new RsGxsMessageCleanUp(mDataStore, this, 1);
mCleaning = true;
}
}
RsGxsCleanUp(mDataStore,this,1).clean(mNextGroupToCheck,grps_to_delete,msgs_to_delete); // no need to lock here, because all access below (RsGenExchange, RsDataStore) are properly mutexed
uint32_t token1=0;
deleteMsgs(token1,msgs_to_delete);
for(auto& grpId: grps_to_delete)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
RS_STACK_MUTEX(mGenMtx) ;
mLastClean = now;
}
now = time(NULL);
if(mChecking || (mLastCheck + INTEGRITY_CHECK_PERIOD < now))
{
mLastCheck = time(NULL);

View File

@ -648,6 +648,19 @@ protected:
*/
virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
/*!
* \brief service_checkIfGroupIsStillUsed
* Re-implement this function to help GXS cleaning, by telling that some particular group
* is not used anymore. This usually depends on subscription, the fact that friend nodes send
* some info or not, and particular cleaning strategy of each service.
* Besides, groups in some services are used by other services (e.g. identities, circles, are used in
* forums and so on), so deciding on a group usage can only be left to the specific service it is used in.
* \return
* true if the group is still used, false otherwise, meaning that the group can be deleted. Default is
* that the group is always in use.
*/
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& /* meta */) { return true; } // see RsGenExchange
public:
/*!
@ -959,12 +972,11 @@ private:
bool mCleaning;
rstime_t mLastClean;
RsGxsMessageCleanUp* mMsgCleanUp;
bool mChecking, mCheckStarted;
rstime_t mLastCheck;
RsGxsIntegrityCheck* mIntegrityCheck;
RsGxsGroupId mNextGroupToCheck ;
protected:
enum CreateStatus { CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER };
@ -982,6 +994,8 @@ private:
std::vector<MsgDeletePublish> mMsgDeletePublish;
std::map<RsGxsId,std::set<RsPeerId> > mRoutingClues ;
friend class RsGxsCleanUp;
};
#endif // RSGENEXCHANGE_H

View File

@ -128,6 +128,8 @@ public:
virtual void setDefaultKeepAge(uint32_t t) override { mDefaultMsgStorePeriod = t ; }
virtual void setDefaultSyncAge(uint32_t t) override { mDefaultMsgSyncPeriod = t ; }
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.

View File

@ -35,114 +35,161 @@
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!)
//#define DEBUG_GXSUTIL 1
#define DEBUG_GXSUTIL 1
#ifdef DEBUG_GXSUTIL
#define GXSUTIL_DEBUG() std::cerr << "[" << time(NULL) << "] : GXS_UTIL : " << __FUNCTION__ << " : "
#endif
RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize)
RsGxsCleanUp::RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize)
: mDs(dataService), mGenExchangeClient(genex), CHUNK_SIZE(chunkSize)
{
RsGxsGrpMetaTemporaryMap grpMeta;
mDs->retrieveGxsGrpMetaData(grpMeta);
for(auto cit=grpMeta.begin();cit != grpMeta.end(); ++cit)
mGrpMeta.push_back(cit->second);
}
bool RsGxsMessageCleanUp::clean()
bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGroupId>& grps_to_delete,GxsMsgReq& messages_to_delete)
{
uint32_t i = 1;
RsGxsGrpMetaTemporaryMap grpMetaMap;
mDs->retrieveGxsGrpMetaData(grpMetaMap);
rstime_t now = time(NULL);
rstime_t now = time(NULL);
#ifdef DEBUG_GXSUTIL
uint16_t service_type = mGenExchangeClient->serviceType() ;
GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << std::endl;
uint16_t service_type = mGenExchangeClient->serviceType() ;
GXSUTIL_DEBUG() << " Cleaning up groups in service " << std::hex << service_type << std::dec << " starting at group " << next_group_to_check << std::endl;
#endif
while(!mGrpMeta.empty())
{
const RsGxsGrpMetaData* grpMeta = mGrpMeta.back();
const RsGxsGroupId& grpId = grpMeta->mGroupId;
mGrpMeta.pop_back();
GxsMsgReq req;
GxsMsgMetaResult result;
// This method stores/takes the next group to check. This allows to limit group checking to a small part of the total groups
// in the situation where it takes too much time. So when arriving here, we must start again from where we left last time.
req[grpId] = std::set<RsGxsMessageId>();
mDs->retrieveGxsMsgMetaData(req, result);
if(grpMetaMap.empty()) // nothing to do.
{
next_group_to_check.clear();
return true;
}
GxsMsgMetaResult::iterator mit = result.begin();
auto it = next_group_to_check.isNull()?grpMetaMap.begin() : grpMetaMap.find(next_group_to_check);
if(it == grpMetaMap.end()) // group wasn't found
it = grpMetaMap.begin();
bool full_round = false; // did we have the time to test all groups?
next_group_to_check = it->first; // covers the case where next_group_to_check is null or not found
while(true) // check all groups, starting from the one indicated as parameter
{
const RsGxsGrpMetaData& grpMeta = *(it->second);
// first check if we keep the group or not
if(!mGenExchangeClient->service_checkIfGroupIsStillUsed(grpMeta))
{
#ifdef DEBUG_GXSUTIL
std::cerr << " Scheduling group " << grpMeta.mGroupId << " for removal." << std::endl;
#endif
grps_to_delete.push_back(grpMeta.mGroupId);
}
else
{
const RsGxsGroupId& grpId = grpMeta.mGroupId;
GxsMsgReq req;
GxsMsgMetaResult result;
req[grpId] = std::set<RsGxsMessageId>();
mDs->retrieveGxsMsgMetaData(req, result);
GxsMsgMetaResult::iterator mit = result.begin();
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl;
GXSUTIL_DEBUG() << " Cleaning up group message for group ID " << grpId << std::endl;
#endif
req.clear();
uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ;
uint32_t store_period = mGenExchangeClient->getStoragePeriod(grpId) ;
for(; mit != result.end(); ++mit)
{
std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
for(; mit != result.end(); ++mit)
{
std::vector<const RsGxsMsgMetaData*>& metaV = mit->second;
// First, make a map of which message have a child message. This allows to only delete messages that dont have child messages.
// A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the
// end the message tree will be deleted slice after slice, which should still be reasonnably fast.
//
std::set<RsGxsMessageId> messages_with_kids ;
// First, make a map of which message have a child message. This allows to only delete messages that dont have child messages.
// A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the
// end the message tree will be deleted slice after slice, which should still be reasonnably fast.
//
std::set<RsGxsMessageId> messages_with_kids ;
for( uint32_t i=0;i<metaV.size();++i)
if(!metaV[i]->mParentId.isNull())
messages_with_kids.insert(metaV[i]->mParentId) ;
for( uint32_t i=0;i<metaV.size();++i)
if(!metaV[i]->mParentId.isNull())
messages_with_kids.insert(metaV[i]->mParentId) ;
for( uint32_t i=0;i<metaV.size();++i)
{
const RsGxsMsgMetaData* meta = metaV[i];
for( uint32_t i=0;i<metaV.size();++i)
{
const RsGxsMsgMetaData* meta = metaV[i];
bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end());
bool have_kids = (messages_with_kids.find(meta->mMsgId)!=messages_with_kids.end());
// check if expired
bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids;
// check if expired
bool remove = store_period > 0 && ((meta->mPublishTs + store_period) < now) && !have_kids;
// check client does not want the message kept regardless of age
remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER);
// check client does not want the message kept regardless of age
remove &= !(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER);
// if not subscribed remove messages (can optimise this really)
remove = remove || (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED);
remove = remove || !(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
// if not subscribed remove messages (can optimise this really)
remove = remove || (grpMeta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED);
remove = remove || !(grpMeta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP)
<< " subscribed: " << bool(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period
<< " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ;
GXSUTIL_DEBUG() << " msg id " << meta->mMsgId << " in grp " << grpId << ": keep_flag=" << bool(meta->mMsgStatus & GXS_SERV::GXS_MSG_STATUS_KEEP_FOREVER)
<< " subscribed: " << bool(grpMeta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) << " store_period: " << store_period
<< " kids: " << have_kids << " now - meta->mPublishTs: " << now - meta->mPublishTs ;
#endif
if( remove )
{
req[grpId].insert(meta->mMsgId);
if( remove )
{
messages_to_delete[grpId].insert(meta->mMsgId);
#ifdef DEBUG_GXSUTIL
std::cerr << " Scheduling for removal." << std::endl;
std::cerr << " Scheduling for removal." << std::endl;
#endif
}
}
#ifdef DEBUG_GXSUTIL
else
std::cerr << std::endl;
else
std::cerr << std::endl;
#endif
//delete meta;
}
}
}
//delete meta;
}
}
++it;
mDs->removeMsgs(req);
if(it == grpMetaMap.end())
it = grpMetaMap.begin();
i++;
if(i > CHUNK_SIZE) break;
}
// check if we looped already
return mGrpMeta.empty();
if(it->first == next_group_to_check)
{
GXSUTIL_DEBUG() << "Had the time to test all groups. Will start again at " << it->first << std::endl;
full_round = true;
break;
}
// now check if we spent too much time on this already
rstime_t tm = time(nullptr);
//if(tm > now + 1) // we spent more than 1 sec on the job already
if(tm > now) // we spent more than 1 sec on the job already
{
GXSUTIL_DEBUG() << "Aborting cleanup because it took too much time already. Next group left to be " << it->first << std::endl;
next_group_to_check = it->first;
full_round = false;
break;
}
}
return full_round;
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
@ -223,6 +270,8 @@ bool RsGxsIntegrityCheck::check()
else msgIds.erase(msgIds.find(grp->grpId));
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
@ -359,8 +408,9 @@ bool RsGxsIntegrityCheck::check()
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
if( isGxsChannels
&& indexedGroups.count(msg->metaData->mGroupId) )
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);

View File

@ -36,10 +36,10 @@ class RsGeneralDataService ;
class non_copiable
{
public:
non_copiable() {}
non_copiable() {}
private:
non_copiable& operator=(const non_copiable&) { return *this ;}
non_copiable(const non_copiable&) {}
non_copiable& operator=(const non_copiable&) { return *this ;}
non_copiable(const non_copiable&) {}
};
template<class IdClass,class IdData>
@ -55,7 +55,7 @@ public:
{
for(typename t_RsGxsGenericDataTemporaryMap<IdClass,IdData>::iterator it = this->begin();it!=this->end();++it)
if(it->second != NULL)
delete it->second ;
delete it->second ;
std::map<IdClass,IdData*>::clear() ;
}
@ -75,7 +75,7 @@ public:
for(typename t_RsGxsGenericDataTemporaryMapVector<T>::iterator it = this->begin();it!=this->end();++it)
{
for(uint32_t i=0;i<it->second.size();++i)
delete it->second[i] ;
delete it->second[i] ;
it->second.clear();
}
@ -113,44 +113,43 @@ typedef t_RsGxsGenericDataTemporaryList<RsNxsMsg> RsNxsMsgDa
inline RsGxsGrpMsgIdPair getMsgIdPair(RsNxsMsg& msg)
{
return RsGxsGrpMsgIdPair(std::make_pair(msg.grpId, msg.msgId));
return RsGxsGrpMsgIdPair(std::make_pair(msg.grpId, msg.msgId));
}
inline RsGxsGrpMsgIdPair getMsgIdPair(RsGxsMsgItem& msg)
{
return RsGxsGrpMsgIdPair(std::make_pair(msg.meta.mGroupId, msg.meta.mMsgId));
return RsGxsGrpMsgIdPair(std::make_pair(msg.meta.mGroupId, msg.meta.mMsgId));
}
/*!
* Does message clean up based on individual group expirations first
* if avialable. If not then deletion s
*/
class RsGxsMessageCleanUp
class RsGxsCleanUp
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsMessageCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize);
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsCleanUp(RsGeneralDataService* const dataService, RsGenExchange *genex, uint32_t chunkSize);
/*!
* On construction this should be called to progress deletions
* Deletion will process by chunk size
* @return true if no more messages to delete, false otherwise
*/
bool clean();
/*!
* On construction this should be called to progress deletions
* Deletion will process by chunk size
* @return true if no more messages to delete, false otherwise
*/
bool clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGroupId>& grps_to_delete,GxsMsgReq& messages_to_delete);
private:
RsGeneralDataService* const mDs;
RsGeneralDataService* const mDs;
RsGenExchange *mGenExchangeClient;
uint32_t CHUNK_SIZE;
std::vector<const RsGxsGrpMetaData*> mGrpMeta;
uint32_t CHUNK_SIZE;
};
/*!
@ -160,52 +159,52 @@ private:
class RsGxsIntegrityCheck : public RsThread
{
enum CheckState { CheckStart, CheckChecking };
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType& gxsSerialiser,
RsGixs *gixs);
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType& gxsSerialiser,
RsGixs *gixs);
bool check();
bool isDone();
bool check();
bool isDone();
void run();
void run();
void getDeletedIds(std::list<RsGxsGroupId>& grpIds, std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgIds);
void getDeletedIds(std::list<RsGxsGroupId>& grpIds, std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgIds);
private:
RsGeneralDataService* const mDs;
RsGenExchange *mGenExchangeClient;
RsGeneralDataService* const mDs;
RsGenExchange *mGenExchangeClient;
#ifdef RS_DEEP_CHANNEL_INDEX
RsSerialType& mSerializer;
RsSerialType& mSerializer;
#endif
bool mDone;
RsMutex mIntegrityMutex;
std::list<RsGxsGroupId> mDeletedGrps;
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mDeletedMsgs;
bool mDone;
RsMutex mIntegrityMutex;
std::list<RsGxsGroupId> mDeletedGrps;
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mDeletedMsgs;
RsGixs* mGixs;
RsGixs* mGixs;
};
class GroupUpdate
{
public:
GroupUpdate() : oldGrpMeta(NULL), newGrp(NULL), validUpdate(false)
{}
const RsGxsGrpMetaData* oldGrpMeta;
RsNxsGrp* newGrp;
bool validUpdate;
GroupUpdate() : oldGrpMeta(NULL), newGrp(NULL), validUpdate(false)
{}
const RsGxsGrpMetaData* oldGrpMeta;
RsNxsGrp* newGrp;
bool validUpdate;
};
class GroupUpdatePublish
@ -213,8 +212,8 @@ class GroupUpdatePublish
public:
GroupUpdatePublish(RsGxsGrpItem* item, uint32_t token)
: grpItem(item), mToken(token) {}
RsGxsGrpItem* grpItem;
uint32_t mToken;
RsGxsGrpItem* grpItem;
uint32_t mToken;
};
class GroupDeletePublish
@ -222,8 +221,8 @@ class GroupDeletePublish
public:
GroupDeletePublish(const RsGxsGroupId& grpId, uint32_t token)
: mGroupId(grpId), mToken(token) {}
RsGxsGroupId mGroupId;
uint32_t mToken;
RsGxsGroupId mGroupId;
uint32_t mToken;
};
@ -232,7 +231,7 @@ class MsgDeletePublish
public:
MsgDeletePublish(const GxsMsgReq& msgs, uint32_t token)
: mMsgs(msgs), mToken(token) {}
GxsMsgReq mMsgs ;
uint32_t mToken;
GxsMsgReq mMsgs ;
uint32_t mToken;
};

View File

@ -82,6 +82,9 @@ public:
virtual uint32_t getDefaultSyncAge() =0;
virtual uint32_t getDefaultKeepAge() =0;
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View File

@ -118,6 +118,7 @@ enum class RsChannelEventCode: uint8_t
SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed
NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message
NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system
};
struct RsGxsChannelEvent: RsEvent

View File

@ -235,6 +235,11 @@ enum class RsGxsCircleEventCode: uint8_t
*
* no additional information. Simply means that the info previously from the cache has changed. */
CACHE_DATA_UPDATED = 0x06,
/**
* The circle has been deleted by auto-cleaning.
* */
CIRCLE_DELETED = 0x07,
};
struct RsGxsCircleEvent: RsEvent

View File

@ -107,8 +107,7 @@ namespace GXS_SERV {
static const uint32_t GXS_MSG_STATUS_UNPROCESSED = 0x00000001; // Flags to store the read/process status of group messages.
static const uint32_t GXS_MSG_STATUS_GUI_UNREAD = 0x00000002; // The actual meaning may depend on the type of service.
static const uint32_t GXS_MSG_STATUS_GUI_NEW = 0x00000004; //
/** Do not delete message even if older then group maximum storage time */
static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008;
static const uint32_t GXS_MSG_STATUS_KEEP_FOREVER = 0x00000008; // Do not delete message even if older then group maximum storage time
static const uint32_t GXS_MSG_STATUS_DELETE = 0x00000020; //
/** END GXS Msg status flags **/

View File

@ -117,6 +117,7 @@ enum class RsForumEventCode: uint8_t
MODERATOR_LIST_CHANGED = 0x08, /// forum moderation list has changed.
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
};
struct RsGxsForumEvent: RsEvent

View File

@ -119,6 +119,7 @@ enum class RsPostedEventCode: uint8_t
SYNC_PARAMETERS_UPDATED = 0x09,
NEW_COMMENT = 0x0a,
NEW_VOTE = 0x0b,
BOARD_DELETED = 0x0c,
};

View File

@ -128,6 +128,7 @@ RS_DEPRECATED_FOR(RsServiceType) const uint16_t RS_SERVICE_TYPE_JSONAPI
RS_DEPRECATED_FOR(RsServiceType) const uint16_t RS_SERVICE_GXS_TYPE_FORUMS_CONFIG = 0x0315;
RS_DEPRECATED_FOR(RsServiceType) const uint16_t RS_SERVICE_GXS_TYPE_POSTED_CONFIG = 0x0316;
RS_DEPRECATED_FOR(RsServiceType) const uint16_t RS_SERVICE_GXS_TYPE_CHANNELS_CONFIG = 0x0317;
RS_DEPRECATED_FOR(RsServiceType) const uint16_t RS_SERVICE_GXS_TYPE_CIRCLES_CONFIG = 0x0318;
// Experimental Services.
/* DSDV Testing at the moment - Service Only */

View File

@ -1632,15 +1632,16 @@ int RsServer::StartupRetroShare()
mConfigMgr->addConfiguration("gxs_trans.cfg" , mGxsTrans);
# endif // RS_GXS_TRANS
mConfigMgr->addConfiguration("p3identity.cfg" , mGxsIdService);
mConfigMgr->addConfiguration("identity.cfg" , gxsid_ns);
mConfigMgr->addConfiguration("gxsforums.cfg" , gxsforums_ns);
mConfigMgr->addConfiguration("gxsforums_srv.cfg", mGxsForums);
mConfigMgr->addConfiguration("gxschannels.cfg" , gxschannels_ns);
mConfigMgr->addConfiguration("p3identity.cfg" , mGxsIdService);
mConfigMgr->addConfiguration("identity.cfg" , gxsid_ns);
mConfigMgr->addConfiguration("gxsforums.cfg" , gxsforums_ns);
mConfigMgr->addConfiguration("gxsforums_srv.cfg" , mGxsForums);
mConfigMgr->addConfiguration("gxschannels.cfg" , gxschannels_ns);
mConfigMgr->addConfiguration("gxschannels_srv.cfg", mGxsChannels);
mConfigMgr->addConfiguration("gxscircles.cfg" , gxscircles_ns);
mConfigMgr->addConfiguration("posted.cfg" , posted_ns);
mConfigMgr->addConfiguration("gxsposted_srv.cfg", mPosted);
mConfigMgr->addConfiguration("gxscircles.cfg" , gxscircles_ns);
mConfigMgr->addConfiguration("gxscircles_srv.cfg" , mGxsCircles);
mConfigMgr->addConfiguration("posted.cfg" , posted_ns);
mConfigMgr->addConfiguration("gxsposted_srv.cfg" , mPosted);
#ifdef RS_USE_WIKI
mConfigMgr->addConfiguration("wiki.cfg", wiki_ns);
#endif

View File

@ -64,11 +64,13 @@
#define CHANNEL_PROCESS 0x0001
#define CHANNEL_TESTEVENT_DUMMYDATA 0x0002
#define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated.
#define DUMMYDATA_PERIOD 60 // Long enough for some RsIdentities to be generated.
#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7)
#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-)
#define CHANNEL_UNUSED_BY_FRIENDS_DELAY (3600*24*60) // Two months. Will be used to delete a channel if too old
#define CHANNEL_DELAY_FOR_CHECKING_AND_DELETING_OLD_GROUPS 300 // check for old channels every 30 mins. Far too often than above delay by RS needs to run it at least once per session
#define CHANNEL_DOWNLOAD_PERIOD (3600 * 24 * 7)
#define CHANNEL_MAX_AUTO_DL (8 * 1024 * 1024 * 1024ull) // 8 GB. Just a security ;-)
/********************************************************************************/
/******************* Startup / Tick ******************************************/
/********************************************************************************/
@ -209,7 +211,7 @@ bool p3GxsChannels::loadList(std::list<RsItem *>& loadList)
mKnownChannels.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)
if( now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE)
if(now < it->second + GXS_CHANNELS_CONFIG_MAX_TIME_NOTIFY_STORAGE)
mKnownChannels.insert(*it) ;
}
@ -332,28 +334,36 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mChannelGroupId = grpChange->mGroupId;
ev->mChannelEventCode = RsChannelEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
}
// also update channel usage. Statistics are updated when a friend sends some sync packets
RS_STACK_MUTEX(mKnownChannelsMutex);
mKnownChannels[grpChange->mGroupId] = time(NULL);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_PUBLISHED:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
/* group received */
/* group received or updated */
RS_STACK_MUTEX(mKnownChannelsMutex);
bool unknown ;
{
RS_STACK_MUTEX(mKnownChannelsMutex);
unknown = (mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end());
mKnownChannels[grpChange->mGroupId] = time(NULL);
IndicateConfigChanged();
}
#ifdef GXSCHANNEL_DEBUG
RsDbg() << " Type = Published/New " << std::endl;
#endif
if(mKnownChannels.find(grpChange->mGroupId) == mKnownChannels.end())
if(unknown)
{
#ifdef GXSCHANNEL_DEBUG
RsDbg() << " Status: unknown. Sending notification event." << std::endl;
#endif
mKnownChannels.insert(std::make_pair(grpChange->mGroupId,time(NULL))) ;
IndicateConfigChanged();
auto ev = std::make_shared<RsGxsChannelEvent>();
ev->mChannelGroupId = grpChange->mGroupId;
ev->mChannelEventCode = RsChannelEventCode::NEW_CHANNEL;
@ -366,7 +376,16 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
}
break;
case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
case RsGxsNotify::TYPE_GROUP_DELETED:
{
auto ev = std::make_shared<RsGxsChannelEvent>();
ev->mChannelGroupId = grpChange->mGroupId;
ev->mChannelEventCode = RsChannelEventCode::DELETED_CHANNEL;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
{
/* group received */
auto ev = std::make_shared<RsGxsChannelEvent>();
@ -431,6 +450,64 @@ void p3GxsChannels::service_tick()
}
}
bool p3GxsChannels::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << "p3gxsChannels: Checking unused channel: called by GxsCleaning." << std::endl;
#endif
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownChannelsMutex);
auto it = mKnownChannels.find(meta.mGroupId);
bool unknown_channel = it == mKnownChannels.end();
#ifdef GXSFORUMS_CHANNELS
std::cerr << " Channel " << meta.mGroupId ;
#endif
if(unknown_channel)
{
// This case should normally not happen. It does because this channel was never registered since it may
// arrived before this code was here
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
#endif
mKnownChannels[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + CHANNEL_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
#endif
if(!subscribed && !used_by_friends)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Scheduling for deletion" << std::endl;
#endif
return false;
}
else
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Keeping!" << std::endl;
#endif
return true;
}
}
}
bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups)
{
#ifdef GXSCHANNELS_DEBUG

View File

@ -62,6 +62,7 @@ public:
protected:
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override; // see RsGenExchange
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
@ -332,6 +333,7 @@ static uint32_t channelsAuthenPolicy();
void clearUnsubscribedGroup(const RsGxsGroupId &id);
bool setAutoDownload(const RsGxsGroupId &groupId, bool enabled);
bool autoDownloadEnabled(const RsGxsGroupId &groupId, bool &enabled);
bool checkForOldAndUnusedChannels();
// DUMMY DATA,
virtual bool generateDummyData();
@ -379,6 +381,7 @@ bool generateGroup(uint32_t &token, std::string groupName);
RsMutex mKnownChannelsMutex;
rstime_t mLastDistantSearchNotificationTS;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
#ifdef TO_REMOVE
/** Store search callbacks with timeout*/

View File

@ -40,7 +40,6 @@
/****
* #define DEBUG_CIRCLES 1
****/
#define DEBUG_CIRCLES 1
/*extern*/ RsGxsCircles* rsGxsCircles = nullptr;
@ -50,6 +49,8 @@
/*static*/ const std::string RsGxsCircles::CIRCLE_URL_ID_FIELD = "circleId";
/*static*/ const std::string RsGxsCircles::CIRCLE_URL_DATA_FIELD = "circleData";
static const uint32_t CIRCLES_UNUSED_BY_FRIENDS_DELAY = 60*86400 ; // 60 days ...O...
RsGxsCircles::~RsGxsCircles() = default;
RsGxsCircleMsg::~RsGxsCircleMsg() = default;
RsGxsCircleDetails::~RsGxsCircleDetails() = default;
@ -142,6 +143,7 @@ p3GxsCircles::p3GxsCircles( RsGeneralDataService *gds, RsNetworkExchangeService
RsGxsCircles(static_cast<RsGxsIface&>(*this)), GxsTokenQueue(this),
RsTickEvent(), mIdentities(identities), mPgpUtils(pgpUtils),
mCircleMtx("p3GxsCircles"),
mKnownCirclesMtx("p3GxsCircles"),
// mCircleCache(DEFAULT_MEM_CACHE_SIZE, "GxsCircleCache" ),
mShouldSendCacheUpdateNotification(false)
{
@ -656,8 +658,14 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
addCircleIdToList(RsGxsCircleId(*git), 0);
circles_to_reload.insert(RsGxsCircleId(*git));
}
break;
} // fallthrough
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
RS_STACK_MUTEX(mKnownCirclesMtx);
mKnownCircles[*git] = time(nullptr);
IndicateConfigChanged();
}
break;
default:
#ifdef DEBUG_CIRCLES
std::cerr << " Type: " << c->getType() << " is ignored" << std::endl;
@ -690,7 +698,7 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mCircleId = RsGxsCircleId(*git);
ev->mGxsId = gxs_id;
rsEvents->sendEvent(ev);
rsEvents->postEvent(ev);
}
}
else if(c->getType()==RsGxsNotify::TYPE_UPDATED)
@ -720,7 +728,7 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mCircleId = circle_id;
ev->mGxsId = gxs_id;
rsEvents->sendEvent(ev);
rsEvents->postEvent(ev);
}
for(auto& gxs_id: old_circle_grp_item->gxsIdSet.ids)
@ -732,10 +740,19 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
ev->mCircleId = circle_id;
ev->mGxsId = gxs_id;
rsEvents->sendEvent(ev);
rsEvents->postEvent(ev);
}
}
else if(c->getType()==RsGxsNotify::TYPE_GROUP_DELETED)
{
auto ev = std::make_shared<RsGxsCircleEvent>();
ev->mCircleEventType = RsGxsCircleEventCode::CIRCLE_DELETED;
ev->mCircleId = RsGxsCircleId(groupChange->mGroupId);
rsEvents->postEvent(ev);
}
}
}
@ -1714,6 +1731,62 @@ void p3GxsCircles::addCircleIdToList(const RsGxsCircleId &circleId, uint32_t cir
}
}
bool p3GxsCircles::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << "p3gxsChannels: Checking unused circles: called by GxsCleaning." << std::endl;
#endif
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownCirclesMtx);
auto it = mKnownCircles.find(meta.mGroupId);
bool unknown_posted = (it == mKnownCircles.end());
#ifdef GXSFORUMS_CHANNELS
std::cerr << " Circle " << meta.mGroupId ;
#endif
if(unknown_posted)
{
// This case should normally not happen. It does because this board was never registered since it may
// arrived before this code was here
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
#endif
mKnownCircles[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + CIRCLES_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
if(!subscribed && !used_by_friends)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Scheduling for deletion" << std::endl;
#endif
return false;
}
else
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Keeping!" << std::endl;
#endif
return true;
}
}
}
//====================================================================================//
// Event handling //
//====================================================================================//
@ -2074,6 +2147,100 @@ bool p3GxsCircles::processMembershipRequests(uint32_t token)
return true ;
}
//====================================================================================//
// p3Config methods //
//====================================================================================//
static const uint32_t GXS_FORUMS_CONFIG_MAX_TIME_NOTIFY_STORAGE = 86400*30*2 ; // ignore notifications for 2 months
static const uint8_t GXS_CIRCLES_CONFIG_SUBTYPE_NOTIFY_RECORD = 0x01 ;
struct RsGxsCirclesNotifyRecordsItem: public RsItem
{
RsGxsCirclesNotifyRecordsItem()
: RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_GXS_TYPE_CIRCLES_CONFIG,GXS_CIRCLES_CONFIG_SUBTYPE_NOTIFY_RECORD)
{}
virtual ~RsGxsCirclesNotifyRecordsItem() {}
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{ RS_SERIAL_PROCESS(records); }
void clear() {}
std::map<RsGxsGroupId,rstime_t> records;
};
class GxsCirclesConfigSerializer : public RsServiceSerializer
{
public:
GxsCirclesConfigSerializer() : RsServiceSerializer(RS_SERVICE_GXS_TYPE_CIRCLES_CONFIG) {}
virtual ~GxsCirclesConfigSerializer() {}
RsItem* create_item(uint16_t service_id, uint8_t item_sub_id) const
{
if(service_id != RS_SERVICE_GXS_TYPE_CIRCLES_CONFIG)
return NULL;
switch(item_sub_id)
{
case GXS_CIRCLES_CONFIG_SUBTYPE_NOTIFY_RECORD: return new RsGxsCirclesNotifyRecordsItem();
default:
return NULL;
}
}
};
bool p3GxsCircles::saveList(bool& cleanup, std::list<RsItem *>&saveList)
{
cleanup = true ;
RsGxsCirclesNotifyRecordsItem *item = new RsGxsCirclesNotifyRecordsItem ;
{
RS_STACK_MUTEX(mKnownCirclesMtx);
item->records = mKnownCircles ;
}
saveList.push_back(item) ;
return true;
}
bool p3GxsCircles::loadList(std::list<RsItem *>& loadList)
{
while(!loadList.empty())
{
RsItem *item = loadList.front();
loadList.pop_front();
rstime_t now = time(NULL);
RsGxsCirclesNotifyRecordsItem *fnr = dynamic_cast<RsGxsCirclesNotifyRecordsItem*>(item) ;
if(fnr != NULL)
{
RS_STACK_MUTEX(mKnownCirclesMtx);
mKnownCircles.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)
if( now < it->second + GXS_FORUMS_CONFIG_MAX_TIME_NOTIFY_STORAGE)
mKnownCircles.insert(*it) ;
}
delete item ;
}
return true;
}
RsSerialiser* p3GxsCircles::setupSerialiser()
{
RsSerialiser* rss = new RsSerialiser;
rss->addSerialType(new GxsCirclesConfigSerializer());
return rss;
}
//====================================================================================//
// DEBUG STUFF //

View File

@ -208,8 +208,7 @@ public:
}
};
class p3GxsCircles: public RsGxsCircleExchange, public RsGxsCircles,
public GxsTokenQueue, public RsTickEvent
class p3GxsCircles: public RsGxsCircleExchange, public RsGxsCircles, public GxsTokenQueue, public RsTickEvent, public p3Config
{
public:
p3GxsCircles(
@ -271,52 +270,58 @@ public:
std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string)
) override;
virtual bool getCircleDetails(const RsGxsCircleId &id, RsGxsCircleDetails &details);
virtual bool getCircleExternalIdList(std::list<RsGxsCircleId> &circleIds);
virtual bool getCircleDetails(const RsGxsCircleId &id, RsGxsCircleDetails &details) override;
virtual bool getCircleExternalIdList(std::list<RsGxsCircleId> &circleIds) override;
virtual bool isLoaded(const RsGxsCircleId &circleId);
virtual bool loadCircle(const RsGxsCircleId &circleId);
virtual bool isLoaded(const RsGxsCircleId &circleId) override;
virtual bool loadCircle(const RsGxsCircleId &circleId) override;
virtual int canSend(const RsGxsCircleId &circleId, const RsPgpId &id, bool &should_encrypt);
virtual int canReceive(const RsGxsCircleId &circleId, const RsPgpId &id);
virtual int canSend(const RsGxsCircleId &circleId, const RsPgpId &id, bool &should_encrypt) override;
virtual int canReceive(const RsGxsCircleId &circleId, const RsPgpId &id) override;
virtual bool recipients(const RsGxsCircleId &circleId, std::list<RsPgpId> &friendlist) ;
virtual bool recipients(const RsGxsCircleId &circleId, const RsGxsGroupId& dest_group, std::list<RsGxsId> &gxs_ids) ;
virtual bool isRecipient(const RsGxsCircleId &circleId, const RsGxsGroupId& destination_group, const RsGxsId& id) ;
virtual bool recipients(const RsGxsCircleId &circleId, std::list<RsPgpId> &friendlist) override;
virtual bool recipients(const RsGxsCircleId &circleId, const RsGxsGroupId& dest_group, std::list<RsGxsId> &gxs_ids) override;
virtual bool isRecipient(const RsGxsCircleId &circleId, const RsGxsGroupId& destination_group, const RsGxsId& id) override;
virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsCircleGroup> &groups);
virtual bool getMsgData(const uint32_t &token, std::vector<RsGxsCircleMsg> &msgs);
virtual void createGroup(uint32_t& token, RsGxsCircleGroup &group);
virtual void updateGroup(uint32_t &token, RsGxsCircleGroup &group);
virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsCircleGroup> &groups) override;
virtual bool getMsgData(const uint32_t &token, std::vector<RsGxsCircleMsg> &msgs) override;
virtual void createGroup(uint32_t& token, RsGxsCircleGroup &group) override;
virtual void updateGroup(uint32_t &token, RsGxsCircleGroup &group) override;
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override;
/* membership management for external circles */
virtual bool requestCircleMembership(const RsGxsId &own_gxsid, const RsGxsCircleId& circle_id) ;
virtual bool cancelCircleMembership(const RsGxsId &own_gxsid, const RsGxsCircleId& circle_id) ;
virtual bool requestCircleMembership(const RsGxsId &own_gxsid, const RsGxsCircleId& circle_id) override;
virtual bool cancelCircleMembership(const RsGxsId &own_gxsid, const RsGxsCircleId& circle_id) override;
/**********************************************/
// needed for background processing.
virtual void service_tick();
// needed for background processing.
virtual void service_tick() override;
protected:
protected:
bool pushCircleMembershipRequest(const RsGxsId& own_gxsid, const RsGxsCircleId& circle_id, RsGxsCircleSubscriptionType request_type) ;
// overloads p3Config
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override;
virtual bool loadList(std::list<RsItem *>& loadList) override;
virtual RsSerialiser *setupSerialiser() override;
bool pushCircleMembershipRequest(const RsGxsId& own_gxsid, const RsGxsCircleId& circle_id, RsGxsCircleSubscriptionType request_type) ;
static uint32_t circleAuthenPolicy();
/** Notifications **/
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) override;
/** Overloaded to add PgpIdHash to Group Definition **/
virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);
virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet) override;
// Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type);
virtual void handleResponse(uint32_t token, uint32_t req_type) override;
// Overloaded from RsTickEvent.
virtual void handle_event(uint32_t event_type, const std::string &elabel);
virtual void handle_event(uint32_t event_type, const std::string &elabel) override;
private:
@ -354,6 +359,9 @@ public:
void addCircleIdToList(const RsGxsCircleId& circleId, uint32_t circleType);
RsMutex mCircleMtx; /* Locked Below Here */
RsMutex mKnownCirclesMtx; /* Locked Below Here */
std::map<RsGxsGroupId,rstime_t> mKnownCircles;
std::list<RsGxsCircleId> mCircleExternalIdList;
std::list<RsGxsCircleId> mCirclePersonalIdList;

View File

@ -48,6 +48,7 @@ RsGxsForums *rsGxsForums = NULL;
#define FORUM_TESTEVENT_DUMMYDATA 0x0001
#define DUMMYDATA_PERIOD 60 // long enough for some RsIdentities to be generated.
#define FORUM_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // unused forums are deleted after 2 months
/********************************************************************************/
/******************* Startup / Tick ******************************************/
@ -145,7 +146,10 @@ bool p3GxsForums::saveList(bool &cleanup, std::list<RsItem *>&saveList)
RsGxsForumNotifyRecordsItem *item = new RsGxsForumNotifyRecordsItem ;
item->records = mKnownForums ;
{
RS_STACK_MUTEX(mKnownForumsMutex);
item->records = mKnownForums ;
}
saveList.push_back(item) ;
return true;
@ -164,6 +168,8 @@ bool p3GxsForums::loadList(std::list<RsItem *>& loadList)
if(fnr != NULL)
{
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)
@ -270,13 +276,16 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
{
/* group received */
RS_STACK_MUTEX(mKnownForumsMutex);
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(mKnownForums.find(grpChange->mGroupId) == mKnownForums.end())
if(unknown)
{
mKnownForums.insert( std::make_pair(grpChange->mGroupId, time(nullptr)));
IndicateConfigChanged();
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
@ -289,13 +298,26 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
}
break;
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
case RsGxsNotify::TYPE_GROUP_DELETED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
}
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_UPDATED:
@ -383,6 +405,63 @@ void p3GxsForums::service_tick()
return;
}
bool p3GxsForums::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
#ifdef GXSFORUMS_DEBUG
std::cerr << "p3gxsForums: Checking unused forums: called by GxsCleaning." << std::endl;
#endif
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownForumsMutex);
auto it = mKnownForums.find(meta.mGroupId);
bool unknown_forum = it == mKnownForums.end();
#ifdef GXSFORUMS_DEBUG
std::cerr << " Forum " << meta.mGroupId ;
#endif
if(unknown_forum)
{
// This case should normally not happen. It does because this forum was never registered since it may
// arrived before this code was here
#ifdef GXSFORUMS_DEBUG
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
#endif
mKnownForums[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + FORUM_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
#ifdef GXSFORUMS_DEBUG
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
#endif
if(!subscribed && !used_by_friends)
{
#ifdef GXSFORUMS_DEBUG
std::cerr << ". Scheduling for deletion" << std::endl;
#endif
return false;
}
else
{
#ifdef GXSFORUMS_DEBUG
std::cerr << ". Keeping!" << std::endl;
#endif
return true;
}
}
}
bool p3GxsForums::getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups)
{
std::vector<RsGxsGrpItem*> grpData;

View File

@ -40,18 +40,19 @@ public:
p3GxsForums(
RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs);
virtual RsServiceInfo getServiceInfo();
virtual void service_tick();
virtual RsServiceInfo getServiceInfo() override;
virtual void service_tick() override;
protected:
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) override;
/// Overloaded from RsTickEvent.
virtual void handle_event(uint32_t event_type, const std::string &elabel);
virtual void handle_event(uint32_t event_type, const std::string &elabel) override;
virtual RsSerialiser* setupSerialiser(); // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList); // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList); // @see p3Config::loadList(std::list<RsItem *>&)
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override;
public:
/// @see RsGxsForums::createForumV2
bool createForumV2(
@ -78,22 +79,22 @@ public:
/// @see RsGxsForums::createForum @deprecated
RS_DEPRECATED_FOR(createForumV2)
virtual bool createForum(RsGxsForumGroup& forum);
virtual bool createForum(RsGxsForumGroup& forum) override;
/// @see RsGxsForums::createMessage @deprecated
RS_DEPRECATED_FOR(createPost)
virtual bool createMessage(RsGxsForumMsg& message);
virtual bool createMessage(RsGxsForumMsg& message) override;
/// @see RsGxsForums::editForum
virtual bool editForum(RsGxsForumGroup& forum) override;
/// @see RsGxsForums::getForumsSummaries
virtual bool getForumsSummaries(std::list<RsGroupMetaData>& forums);
virtual bool getForumsSummaries(std::list<RsGroupMetaData>& forums) override;
/// @see RsGxsForums::getForumsInfo
virtual bool getForumsInfo(
const std::list<RsGxsGroupId>& forumIds,
std::vector<RsGxsForumGroup>& forumsInfo );
std::vector<RsGxsForumGroup>& forumsInfo ) override;
/// Implementation of @see RsGxsForums::getForumStatistics
bool getForumStatistics(const RsGxsGroupId& ForumId,GxsGroupStatistic& stat) override;
@ -102,20 +103,20 @@ public:
bool getForumServiceStatistics(GxsServiceStatistic& stat) override;
/// @see RsGxsForums::getForumMsgMetaData
virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector<RsMsgMetaData>& msg_metas) ;
virtual bool getForumMsgMetaData(const RsGxsGroupId& forumId, std::vector<RsMsgMetaData>& msg_metas) override;
/// @see RsGxsForums::getForumContent
virtual bool getForumContent(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& msgs_to_request,
std::vector<RsGxsForumMsg>& msgs );
std::vector<RsGxsForumMsg>& msgs ) override;
/// @see RsGxsForums::markRead
virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read);
virtual bool markRead(const RsGxsGrpMsgIdPair& messageId, bool read) override;
/// @see RsGxsForums::subscribeToForum
virtual bool subscribeToForum( const RsGxsGroupId& forumId,
bool subscribe );
bool subscribe ) override;
/// @see RsGxsForums
bool exportForumLink(
@ -140,7 +141,7 @@ public:
/// @see RsGxsForums
std::error_condition setPostKeepForever(
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever );
bool keepForever ) override;
/// implementation of rsGxsGorums
///

View File

@ -48,13 +48,18 @@
#define POSTBASE_UNPROCESSED_MSGS 0x0012
#define POSTBASE_ALL_MSGS 0x0013
#define POSTBASE_BG_POST_META 0x0014
#define POSTED_UNUSED_BY_FRIENDS_DELAY (2*30*86400) // delete unused posted groups after 2 months
/********************************************************************************/
/******************* Startup / Tick ******************************************/
/********************************************************************************/
p3PostBase::p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs,
RsSerialType* serviceSerialiser, uint16_t serviceType)
: RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(), mPostBaseMtx("PostBaseMtx")
: RsGenExchange(gds, nes, serviceSerialiser, serviceType, gixs, postBaseAuthenPolicy()), GxsTokenQueue(this), RsTickEvent(),
mPostBaseMtx("PostBaseMutex"),
mKnownPostedMutex("PostBaseKnownPostedMutex")
{
mBgProcessing = false;
@ -127,6 +132,7 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
}
break;
case RsGxsNotify::TYPE_PROCESSED:
{
auto ev = std::make_shared<RsGxsPostedEvent>();
@ -179,12 +185,26 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
}
break;
case RsGxsNotify::TYPE_GROUP_DELETED:
{
auto ev = std::make_shared<RsGxsPostedEvent>();
ev->mPostedGroupId = msgChange->mGroupId;
ev->mPostedEventCode = RsPostedEventCode::BOARD_DELETED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsPostedEvent>();
ev->mPostedGroupId = group_id;
ev->mPostedEventCode = RsPostedEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownPostedMutex);
mKnownPosted[group_id] = time(nullptr);
IndicateConfigChanged();
}
break;
@ -193,11 +213,16 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
{
/* group received */
if(mKnownPosted.find(group_id) == mKnownPosted.end())
bool unknown;
{
mKnownPosted.insert(std::make_pair(group_id, time(nullptr)));
IndicateConfigChanged();
RS_STACK_MUTEX(mKnownPostedMutex);
unknown = (mKnownPosted.find(grpChange->mGroupId) == mKnownPosted.end());
mKnownPosted[group_id] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsPostedEvent>();
ev->mPostedGroupId = group_id;
ev->mPostedEventCode = RsPostedEventCode::NEW_POSTED_GROUP;
@ -883,13 +908,72 @@ public:
}
};
bool p3PostBase::saveList(bool &cleanup, std::list<RsItem *>&saveList)
bool p3PostBase::service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << "p3gxsChannels: Checking unused board: called by GxsCleaning." << std::endl;
#endif
// request all group infos at once
rstime_t now = time(nullptr);
RS_STACK_MUTEX(mKnownPostedMutex);
auto it = mKnownPosted.find(meta.mGroupId);
bool unknown_posted = (it == mKnownPosted.end());
#ifdef GXSFORUMS_CHANNELS
std::cerr << " Board " << meta.mGroupId ;
#endif
if(unknown_posted)
{
// This case should normally not happen. It does because this board was never registered since it may
// arrived before this code was here
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Not known yet. Adding current time as new TS." << std::endl;
#endif
mKnownPosted[meta.mGroupId] = now;
IndicateConfigChanged();
return true;
}
else
{
bool used_by_friends = (now < it->second + POSTED_UNUSED_BY_FRIENDS_DELAY);
bool subscribed = static_cast<bool>(meta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED);
std::cerr << ". subscribed: " << subscribed << ", used_by_friends: " << used_by_friends << " last TS: " << now - it->second << " secs ago (" << (now-it->second)/86400 << " days)";
if(!subscribed && !used_by_friends)
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Scheduling for deletion" << std::endl;
#endif
return false;
}
else
{
#ifdef GXSFORUMS_CHANNELS
std::cerr << ". Keeping!" << std::endl;
#endif
return true;
}
}
}
bool p3PostBase::saveList(bool& cleanup, std::list<RsItem *>&saveList)
{
cleanup = true ;
RsGxsPostedNotifyRecordsItem *item = new RsGxsPostedNotifyRecordsItem ;
item->records = mKnownPosted ;
{
RS_STACK_MUTEX(mKnownPostedMutex);
item->records = mKnownPosted ;
}
saveList.push_back(item) ;
return true;
@ -908,6 +992,8 @@ bool p3PostBase::loadList(std::list<RsItem *>& loadList)
if(fnr != NULL)
{
RS_STACK_MUTEX(mKnownPostedMutex);
mKnownPosted.clear();
for(auto it(fnr->records.begin());it!=fnr->records.end();++it)

View File

@ -70,23 +70,24 @@ public:
p3PostBase(RsGeneralDataService *gds, RsNetworkExchangeService *nes, RsGixs* gixs,
RsSerialType* serviceSerialiser, uint16_t serviceType);
virtual void service_tick();
virtual void service_tick() override;
protected:
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) override;
// Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type);
virtual void handleResponse(uint32_t token, uint32_t req_type) override;
// Overloaded from RsTickEvent.
virtual void handle_event(uint32_t event_type, const std::string &elabel);
virtual void handle_event(uint32_t event_type, const std::string &elabel) override;
// overloads p3Config
virtual RsSerialiser* setupSerialiser() override; // @see p3Config::setupSerialiser()
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList) override; // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual bool service_checkIfGroupIsStillUsed(const RsGxsGrpMetaData& meta) override;
public:
//////////////////////////////////////////////////////////////////////////////
@ -122,6 +123,7 @@ private:
RsMutex mPostBaseMtx;
RsMutex mKnownPostedMutex;
bool mBgProcessing;
bool mBgIncremental;

View File

@ -49,6 +49,9 @@ const uint16_t GXS_POSTED_APP_MINOR_VERSION = 0;
const uint16_t GXS_POSTED_MIN_MAJOR_VERSION = 1;
const uint16_t GXS_POSTED_MIN_MINOR_VERSION = 0;
static const uint32_t GXS_POSTED_CONFIG_MAX_TIME_NOTIFY_STORAGE = 86400*30*2 ; // ignore notifications for 2 months
static const uint8_t GXS_POSTED_CONFIG_SUBTYPE_NOTIFY_RECORD = 0x01 ;
RsServiceInfo p3Posted::getServiceInfo()
{
return RsServiceInfo(RS_SERVICE_GXS_TYPE_POSTED,

View File

@ -92,19 +92,20 @@ virtual void receiveHelperChanges(std::vector<RsGxsNotify*>& changes)
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts, std::vector<RsGxsComment> &cmts, std::vector<RsGxsVote> &vots) override;
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts, std::vector<RsGxsComment> &cmts) override;
virtual bool getPostData(const uint32_t &token, std::vector<RsPostedPost> &posts) override;
//Not currently used
//virtual bool getRelatedPosts(const uint32_t &token, std::vector<RsPostedPost> &posts);
virtual bool createGroup(uint32_t &token, RsPostedGroup &group);
virtual bool createPost(uint32_t &token, RsPostedPost &post);
virtual bool createGroup(uint32_t &token, RsPostedGroup &group) override;
virtual bool createPost(uint32_t &token, RsPostedPost &post) override;
virtual bool updateGroup(uint32_t &token, RsPostedGroup &group);
virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set<RsPeerId>& peers);
virtual bool updateGroup(uint32_t &token, RsPostedGroup &group) override;
virtual bool groupShareKeys(const RsGxsGroupId &group, const std::set<RsPeerId>& peers) override;
//////////////////////////////////////////////////////////////////////////////
// WRAPPERS due to the separate Interface.
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read)
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) override
{
return p3PostBase::setMessageReadStatus(token, msgId, read);
}
@ -112,11 +113,11 @@ virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgI
/** Comment service - Provide RsGxsCommentService -
* redirect to p3GxsCommentService */
virtual bool getCommentData(uint32_t token, std::vector<RsGxsComment> &msgs)
virtual bool getCommentData(uint32_t token, std::vector<RsGxsComment> &msgs) override
{ return mCommentService->getGxsCommentData(token, msgs); }
virtual bool getRelatedComments( uint32_t token,
std::vector<RsGxsComment> &msgs )
std::vector<RsGxsComment> &msgs ) override
{ return mCommentService->getGxsRelatedComments(token, msgs); }
virtual bool createNewComment(uint32_t &token, const RsGxsComment &msg) override

View File

@ -439,7 +439,8 @@ void IdDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> event)
case RsGxsCircleEventCode::CIRCLE_MEMBERSHIP_LEAVE:
case RsGxsCircleEventCode::CIRCLE_MEMBERSHIP_ID_REMOVED_FROM_INVITEE_LIST:
case RsGxsCircleEventCode::NEW_CIRCLE:
case RsGxsCircleEventCode::CACHE_DATA_UPDATED:
case RsGxsCircleEventCode::CIRCLE_DELETED:
case RsGxsCircleEventCode::CACHE_DATA_UPDATED:
updateCircles();
default:

View File

@ -67,7 +67,8 @@ void PostedDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> event)
break;
case RsPostedEventCode::NEW_POSTED_GROUP: // [[fallthrough]];
case RsPostedEventCode::SUBSCRIBE_STATUS_CHANGED: // [[fallthrough]];
case RsPostedEventCode::BOARD_DELETED: // [[fallthrough]];
case RsPostedEventCode::SUBSCRIBE_STATUS_CHANGED: // [[fallthrough]];
updateDisplay(true);
break;

View File

@ -493,6 +493,7 @@ void PostedListWidgetWithModel::handleEvent_main_thread(std::shared_ptr<const Rs
case RsPostedEventCode::NEW_POSTED_GROUP: // [[fallthrough]];
case RsPostedEventCode::UPDATED_POSTED_GROUP: // [[fallthrough]];
case RsPostedEventCode::UPDATED_MESSAGE:
case RsPostedEventCode::BOARD_DELETED:
case RsPostedEventCode::SYNC_PARAMETERS_UPDATED:
{
if(e->mPostedGroupId == groupId())

View File

@ -75,6 +75,7 @@ void GxsChannelDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> ev
case RsChannelEventCode::RECEIVED_PUBLISH_KEY: // [[fallthrough]];
case RsChannelEventCode::NEW_CHANNEL: // [[fallthrough]];
case RsChannelEventCode::DELETED_CHANNEL: // [[fallthrough]];
case RsChannelEventCode::SUBSCRIBE_STATUS_CHANGED:// reloads group summary (calling GxsGroupFrameDialog parent method)
updateDisplay(true);
break;

View File

@ -750,6 +750,7 @@ void GxsChannelPostsWidgetWithModel::handleEvent_main_thread(std::shared_ptr<con
switch(e->mChannelEventCode)
{
case RsChannelEventCode::NEW_CHANNEL: // [[fallthrough]];
case RsChannelEventCode::DELETED_CHANNEL: // [[fallthrough]];
case RsChannelEventCode::NEW_COMMENT: // [[fallthrough]];
case RsChannelEventCode::NEW_VOTE: // [[fallthrough]];
case RsChannelEventCode::UPDATED_CHANNEL: // [[fallthrough]];

View File

@ -67,6 +67,7 @@ void GxsForumsDialog::handleEvent_main_thread(std::shared_ptr<const RsEvent> eve
break;
case RsForumEventCode::NEW_FORUM: // [[fallthrough]];
case RsForumEventCode::DELETED_FORUM: // [[fallthrough]];
case RsForumEventCode::SUBSCRIBE_STATUS_CHANGED:
updateDisplay(true);
break;