From 0dcef10ec2de01fbdf9100e44924e61ddb5d1b5f Mon Sep 17 00:00:00 2001 From: drbob Date: Wed, 2 Feb 2011 21:03:46 +0000 Subject: [PATCH] Fixed the FriendFeed backup of messages: * switched to one universal PendingCache list. * added mHistoricalCaches variable to p3distrib - to indicate when old caches have been loaded. * added calls to p3GroupDistrib::HistoricalCachesDone() in rsinit.cc * added "historical" parameter to lots of p3distrib functions. * updated child classes to only add FeedItems if (historical == false). * Switched Validate / Duplicate Msg checks to speed up historical data load. * corrected rsrandom function for OSX. * bugfix to rsloginhandler function. (compile error). git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4008 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/rsserver/rsinit.cc | 12 +- libretroshare/src/rsserver/rsloginhandler.cc | 4 +- libretroshare/src/services/p3blogs.cc | 23 +++- libretroshare/src/services/p3blogs.h | 6 +- libretroshare/src/services/p3channels.cc | 27 ++-- libretroshare/src/services/p3channels.h | 6 +- libretroshare/src/services/p3distrib.cc | 133 ++++++++++--------- libretroshare/src/services/p3distrib.h | 61 +++++++-- libretroshare/src/services/p3forums.cc | 24 +++- libretroshare/src/services/p3forums.h | 7 +- libretroshare/src/util/rsrandom.cc | 7 +- 11 files changed, 198 insertions(+), 112 deletions(-) diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 172cc32a7..e569d25ce 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2142,7 +2142,17 @@ int RsServer::StartupRetroShare() getPqiNotify()->ClearFeedItems(RS_FEED_ITEM_MESSAGE); //getPqiNotify()->ClearFeedItems(RS_FEED_ITEM_FILES_NEW); - + /* flag that the basic Caches are now in the pending Queues */ +#ifndef MINIMAL_LIBRS + mForums->HistoricalCachesDone(); + mChannels->HistoricalCachesDone(); + +#ifdef RS_USE_BLOGS + mBlogs->HistoricalCachesDone(); +#endif +#endif // MINIMAL_LIBRS + + /**************************************************************************/ /* Add AuthGPG services */ /**************************************************************************/ diff --git a/libretroshare/src/rsserver/rsloginhandler.cc b/libretroshare/src/rsserver/rsloginhandler.cc index 354648fb3..03b6763a6 100644 --- a/libretroshare/src/rsserver/rsloginhandler.cc +++ b/libretroshare/src/rsserver/rsloginhandler.cc @@ -540,7 +540,8 @@ bool RsLoginHandler::clearAutoLogin(const std::string& ssl_id) return false ; } #else - std::string passwdfile = getAutologinFileName() ; + + std::string passwdfile = getAutologinFileName(ssl_id) ; FILE *fp = fopen(passwdfile.c_str(), "wb"); @@ -658,6 +659,7 @@ bool RsLoginHandler::getSSLPasswdFromGPGFile(const std::string& ssl_id,std::stri } } + std::string RsLoginHandler::getSSLPasswdFileName(const std::string& ssl_id) { return RsInit::RsConfigKeysDirectory() + RsInit::dirSeperator() + "ssl_passphrase.pgp"; diff --git a/libretroshare/src/services/p3blogs.cc b/libretroshare/src/services/p3blogs.cc index 23ca85ace..fffa28fe3 100644 --- a/libretroshare/src/services/p3blogs.cc +++ b/libretroshare/src/services/p3blogs.cc @@ -292,14 +292,14 @@ const uint32_t DOWNLOAD_PERIOD = 7 * 24 * 3600; * */ -bool p3Blogs::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Blogs::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { return true; } #include "pqi/pqinotify.h" -bool p3Blogs::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Blogs::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { std::string grpId = msg->grpId; std::string msgId = msg->msgId; @@ -311,7 +311,10 @@ bool p3Blogs::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string std::cerr << " peerId: " << id; std::cerr << std::endl; - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_MSG, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_MSG, grpId, msgId, nullId); + } /* request the files * NB: This could result in duplicates. @@ -319,13 +322,13 @@ bool p3Blogs::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string * * this is exactly what DuplicateMsg does. * */ - return locked_eventDuplicateMsg(grp, msg, id); + return locked_eventDuplicateMsg(grp, msg, id, historical); } -void p3Blogs::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) +void p3Blogs::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags, bool historical) { std::string grpId = grp.grpId; std::string msgId; @@ -341,12 +344,18 @@ void p3Blogs::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) case GRP_NEW_UPDATE: std::cerr << "p3Blogs::locked_notifyGroupChanged() NEW UPDATE"; std::cerr << std::endl; - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_NEW, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_NEW, grpId, msgId, nullId); + } break; case GRP_UPDATE: std::cerr << "p3Blogs::locked_notifyGroupChanged() UPDATE"; std::cerr << std::endl; - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_UPDATE, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_BLOG_UPDATE, grpId, msgId, nullId); + } break; case GRP_LOAD_KEY: std::cerr << "p3Blogs::locked_notifyGroupChanged() LOAD_KEY"; diff --git a/libretroshare/src/services/p3blogs.h b/libretroshare/src/services/p3blogs.h index 07c6c4dc8..ec468a1ba 100644 --- a/libretroshare/src/services/p3blogs.h +++ b/libretroshare/src/services/p3blogs.h @@ -114,9 +114,9 @@ virtual bool isReply(BlogMsgInfo& info); /***************************************************************************************/ protected: -virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags); -virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string); -virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string); +virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags, bool historical); +virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); +virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); /****************************************/ diff --git a/libretroshare/src/services/p3channels.cc b/libretroshare/src/services/p3channels.cc index 7e0734af5..9369ddb3b 100644 --- a/libretroshare/src/services/p3channels.cc +++ b/libretroshare/src/services/p3channels.cc @@ -609,7 +609,7 @@ const uint32_t DOWNLOAD_PERIOD = 7 * 24 * 3600; * on a subscription to a channel.. */ -bool p3Channels::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Channels::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { std::string grpId = msg->grpId; std::string msgId = msg->msgId; @@ -714,24 +714,27 @@ bool p3Channels::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std #include "pqi/pqinotify.h" -bool p3Channels::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Channels::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { std::string grpId = msg->grpId; std::string msgId = msg->msgId; std::string nullId; - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_MSG, grpId, msgId, nullId); - + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_MSG, grpId, msgId, nullId); + } + /* request the files * NB: This could result in duplicates. * which must be handled by ft side. * * this is exactly what DuplicateMsg does. * */ - return locked_eventDuplicateMsg(grp, msg, id); + return locked_eventDuplicateMsg(grp, msg, id, historical); } -void p3Channels::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) +void p3Channels::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags, bool historical) { std::string grpId = grp.grpId; std::string msgId; @@ -744,14 +747,20 @@ void p3Channels::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) #ifdef CHANNEL_DEBUG std::cerr << "p3Channels::locked_notifyGroupChanged() NEW UPDATE" << std::endl; #endif - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_NEW, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_NEW, grpId, msgId, nullId); + } rsicontrol->getNotify().notifyListChange(NOTIFY_LIST_CHANNELLIST_LOCKED, NOTIFY_TYPE_ADD); break; case GRP_UPDATE: #ifdef CHANNEL_DEBUG std::cerr << "p3Channels::locked_notifyGroupChanged() UPDATE" << std::endl; #endif - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_UPDATE, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_CHAN_UPDATE, grpId, msgId, nullId); + } rsicontrol->getNotify().notifyListChange(NOTIFY_LIST_CHANNELLIST_LOCKED, NOTIFY_TYPE_MOD); break; case GRP_LOAD_KEY: @@ -800,7 +809,7 @@ void p3Channels::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) break; } - return p3GroupDistrib::locked_notifyGroupChanged(grp, flags); + return p3GroupDistrib::locked_notifyGroupChanged(grp, flags, historical); } diff --git a/libretroshare/src/services/p3channels.h b/libretroshare/src/services/p3channels.h index bf2589fc4..e4d446385 100644 --- a/libretroshare/src/services/p3channels.h +++ b/libretroshare/src/services/p3channels.h @@ -87,9 +87,9 @@ virtual void getPubKeysAvailableGrpIds(std::list& grpIds); /***************************************************************************************/ protected: -virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags); -virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string); -virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string); +virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags, bool historical); +virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); +virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); /****************************************/ diff --git a/libretroshare/src/services/p3distrib.cc b/libretroshare/src/services/p3distrib.cc index 2ccf560db..18d30b72e 100644 --- a/libretroshare/src/services/p3distrib.cc +++ b/libretroshare/src/services/p3distrib.cc @@ -69,6 +69,7 @@ p3GroupDistrib::p3GroupDistrib(uint16_t subtype, :CacheSource(subtype, true, cs, sourcedir), CacheStore(subtype, true, cs, cft, storedir), p3Config(configId), p3ThreadedService(subtype), + mHistoricalCaches(true), mStorePeriod(storePeriod), mPubPeriod(pubPeriod), mLastPublishTime(0), @@ -177,7 +178,7 @@ int p3GroupDistrib::loadCache(const CacheData &data) std::cerr << std::endl; #endif /* store the cache file for later processing */ - mPendingRemoteCache.push_back(data); + mPendingCaches.push_back(CacheDataPending(data, false, mHistoricalCaches)); } if (data.size > 0) @@ -206,7 +207,7 @@ bool p3GroupDistrib::loadLocalCache(const CacheData &data) #endif /* store the cache file for later processing */ - mPendingLocalCache.push_back(data); + mPendingCaches.push_back(CacheDataPending(data, true, mHistoricalCaches)); } if (data.size > 0) @@ -218,6 +219,18 @@ bool p3GroupDistrib::loadLocalCache(const CacheData &data) } +/* Handle the Cache Pending Setup */ +CacheDataPending::CacheDataPending(const CacheData &data, bool local, bool historical) + :mData(data), mLocal(local), mHistorical(historical) +{ + return; +} + +void p3GroupDistrib::HistoricalCachesDone() +{ + RsStackMutex stack(distribMtx); + mHistoricalCaches = false; // called when Stored Caches have been added to Pending List. +} /* From RsThread */ void p3GroupDistrib::run() /* called once the thread is started */ @@ -234,31 +247,22 @@ void p3GroupDistrib::run() /* called once the thread is started */ CacheData cache; bool validCache = false; bool isLocal = false; + bool isHistorical = false; { RsStackMutex stack(distribMtx); - if (mPendingLocalCache.size() > 0) + if (mPendingCaches.size() > 0) { - cache = mPendingLocalCache.front(); - mPendingLocalCache.pop_front(); + CacheDataPending &pendingCache = mPendingCaches.front(); + cache = pendingCache.mData; + isLocal = pendingCache.mLocal; + isHistorical = pendingCache.mHistorical; + validCache = true; - isLocal = true; + mPendingCaches.pop_front(); #ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::run() found pendingLocalCache"; - std::cerr << std::endl; -#endif - - } - else if (mPendingRemoteCache.size() > 0) - { - cache = mPendingRemoteCache.front(); - mPendingRemoteCache.pop_front(); - validCache = true; - isLocal = false; - -#ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::run() found pendingRemoteCache"; + std::cerr << "p3GroupDistrib::run() found pendingCache"; std::cerr << std::endl; #endif @@ -267,7 +271,7 @@ void p3GroupDistrib::run() /* called once the thread is started */ if (validCache) { - loadAnyCache(cache, isLocal); + loadAnyCache(cache, isLocal, isHistorical); #ifndef WINDOWS_SYS usleep(1000); @@ -288,7 +292,7 @@ void p3GroupDistrib::run() /* called once the thread is started */ -int p3GroupDistrib::loadAnyCache(const CacheData &data, bool local) +int p3GroupDistrib::loadAnyCache(const CacheData &data, bool local, bool historical) { /* if subtype = 1 -> FileGroup, else -> FileMsgs */ @@ -304,11 +308,11 @@ int p3GroupDistrib::loadAnyCache(const CacheData &data, bool local) if (data.cid.subid == 1) { - loadFileGroups(file, data.pid, local); + loadFileGroups(file, data.pid, local, historical); } else { - loadFileMsgs(file, data.cid.subid, data.pid, data.recvd, local); + loadFileMsgs(file, data.cid.subid, data.pid, data.recvd, local, historical); } return true; } @@ -323,7 +327,7 @@ int p3GroupDistrib::loadAnyCache(const CacheData &data, bool local) /* No need for special treatment for 'own' groups. * configuration should be loaded before cache files. */ -void p3GroupDistrib::loadFileGroups(const std::string &filename, const std::string &src, bool local) +void p3GroupDistrib::loadFileGroups(const std::string &filename, const std::string &src, bool local, bool historical) { #ifdef DISTRIB_DEBUG std::cerr << "p3GroupDistrib::loadFileGroups()"; @@ -354,11 +358,11 @@ void p3GroupDistrib::loadFileGroups(const std::string &filename, const std::stri newKey = dynamic_cast(item); if ((newGrp = dynamic_cast(item))) { - loadGroup(newGrp); + loadGroup(newGrp, historical); } else if ((newKey = dynamic_cast(item))) { - loadGroupKey(newKey); + loadGroupKey(newKey, historical); } else { @@ -384,7 +388,7 @@ void p3GroupDistrib::loadFileGroups(const std::string &filename, const std::stri } -void p3GroupDistrib::loadFileMsgs(const std::string &filename, uint16_t cacheSubId, const std::string &src, uint32_t ts, bool local) +void p3GroupDistrib::loadFileMsgs(const std::string &filename, uint16_t cacheSubId, const std::string &src, uint32_t ts, bool local, bool historical) { #ifdef DISTRIB_DEBUG @@ -418,7 +422,7 @@ void p3GroupDistrib::loadFileMsgs(const std::string &filename, uint16_t cacheSub if ((newMsg = dynamic_cast(item))) { - loadMsg(newMsg, src, local); + loadMsg(newMsg, src, local, historical); } else { @@ -475,7 +479,7 @@ void p3GroupDistrib::loadFileMsgs(const std::string &filename, uint16_t cacheSub /***************************************************************************************/ -void p3GroupDistrib::loadGroup(RsDistribGrp *newGrp) +void p3GroupDistrib::loadGroup(RsDistribGrp *newGrp, bool historical) { /* load groupInfo */ const std::string &gid = newGrp -> grpId; @@ -574,9 +578,9 @@ void p3GroupDistrib::loadGroup(RsDistribGrp *newGrp) /* Callback for any derived classes */ if (isNew) - locked_notifyGroupChanged(it->second, GRP_NEW_UPDATE); + locked_notifyGroupChanged(it->second, GRP_NEW_UPDATE, historical); else - locked_notifyGroupChanged(it->second, GRP_UPDATE); + locked_notifyGroupChanged(it->second, GRP_UPDATE, historical); } #ifdef DISTRIB_DEBUG @@ -586,7 +590,7 @@ void p3GroupDistrib::loadGroup(RsDistribGrp *newGrp) } -bool p3GroupDistrib::loadGroupKey(RsDistribGrpKey *newKey) +bool p3GroupDistrib::loadGroupKey(RsDistribGrpKey *newKey, bool historical) { /* load Key */ const std::string &gid = newKey -> grpId; @@ -659,7 +663,7 @@ bool p3GroupDistrib::loadGroupKey(RsDistribGrpKey *newKey) } if (updateOk) - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY); + locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, historical); @@ -676,7 +680,7 @@ bool p3GroupDistrib::loadGroupKey(RsDistribGrpKey *newKey) } -void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, bool local) +void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, bool local, bool historical) { /****************** check the msg ******************/ /* Do the most likely checks to fail first.... @@ -713,22 +717,12 @@ void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, return; } - /****************** check the msg ******************/ - if (!locked_validateDistribSignedMsg(git->second, newMsg)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() validate failed" << std::endl; - std::cerr << std::endl; -#endif - delete newMsg; - return; - } - /* check for duplicate message - * - * do this after validate - because we are calling - * duplicateMsg... only want to do if is good. + /****************** check the msg ******************/ + /* check for duplicate message, do this first to ensure minimal signature validations. + * therefore, duplicateMsg... could potentially be called on a dodgey msg (not a big problem!) */ + std::map::iterator mit; mit = (git->second).msgs.find(newMsg->msgId); if (mit != (git->second).msgs.end()) @@ -738,7 +732,18 @@ void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, std::cerr << std::endl; #endif /* if already there -> remove */ - locked_eventDuplicateMsg(&(git->second), mit->second, src); + locked_eventDuplicateMsg(&(git->second), mit->second, src, historical); + delete newMsg; + return; + } + + /* if unique (new) msg - do validation */ + if (!locked_validateDistribSignedMsg(git->second, newMsg)) + { +#ifdef DISTRIB_DEBUG + std::cerr << "p3GroupDistrib::loadMsg() validate failed" << std::endl; + std::cerr << std::endl; +#endif delete newMsg; return; } @@ -805,7 +810,7 @@ void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, #endif /* Callback for any derived classes to play with */ - locked_eventNewMsg(&(git->second), msg, src); + locked_eventNewMsg(&(git->second), msg, src, historical); /* else if group = subscribed | listener -> publish */ /* if it has come from us... then it has been published already */ @@ -836,7 +841,7 @@ void p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, #endif delete newMsg; } - locked_notifyGroupChanged(git->second, GRP_NEW_MSG); + locked_notifyGroupChanged(git->second, GRP_NEW_MSG, historical); } @@ -1330,7 +1335,7 @@ bool p3GroupDistrib::subscribeToGroup(const std::string &grpId, bool subscrib { git->second.flags |= RS_DISTRIB_SUBSCRIBED; - locked_notifyGroupChanged(git->second, GRP_SUBSCRIBED); + locked_notifyGroupChanged(git->second, GRP_SUBSCRIBED, false); mGroupsRepublish = true; /* reprocess groups messages .... so actions can be taken (by inherited) @@ -1350,7 +1355,7 @@ bool p3GroupDistrib::subscribeToGroup(const std::string &grpId, bool subscrib pit != git->second.sources.end(); pit++) { if(*pit != mOwnId) - locked_eventDuplicateMsg(&(git->second), mit->second, *pit); + locked_eventDuplicateMsg(&(git->second), mit->second, *pit, false); } } } @@ -1361,7 +1366,7 @@ bool p3GroupDistrib::subscribeToGroup(const std::string &grpId, bool subscrib { git->second.flags &= (~RS_DISTRIB_SUBSCRIBED); - locked_notifyGroupChanged(git->second, GRP_UNSUBSCRIBED); + locked_notifyGroupChanged(git->second, GRP_UNSUBSCRIBED, false); mGroupsRepublish = true; } } @@ -1402,7 +1407,7 @@ bool p3GroupDistrib::attemptPublishKeysRecvd() if(locked_updateGroupPublishKey(it->second, mit->second)){ - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY); + locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, false); } if(it->second.flags & RS_DISTRIB_SUBSCRIBED){ @@ -1646,7 +1651,7 @@ bool p3GroupDistrib::loadList(std::list& load) if ((newGrp = dynamic_cast(*lit))) { const std::string &gid = newGrp -> grpId; - loadGroup(newGrp); + loadGroup(newGrp, false); subscribeToGroup(gid, true); } @@ -1662,14 +1667,14 @@ bool p3GroupDistrib::loadList(std::list& load) continue; } - loadGroupKey(newKey); + loadGroupKey(newKey, false); } else if ((newMsg = dynamic_cast(*lit))) { newMsg->PeerId(mOwnId); - loadMsg(newMsg, mOwnId, false); /* false so it'll pushed to PendingPublish list */ + loadMsg(newMsg, mOwnId, false, false); /* false so it'll pushed to PendingPublish list */ } else if ((newChildConfig = dynamic_cast(*lit))) { @@ -1971,7 +1976,7 @@ std::string p3GroupDistrib::createGroup(std::wstring name, std::wstring desc, ui EVP_PKEY_free(key_admin); /******************* load up new Group *********************/ - loadGroup(newGrp); + loadGroup(newGrp, false); /* add Keys to GroupInfo */ RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ @@ -2094,7 +2099,7 @@ bool p3GroupDistrib::restoreGrpKeys(std::string grpId){ if(ok){ gi->flags |= RS_DISTRIB_SUBSCRIBED; - locked_notifyGroupChanged(*gi, GRP_SUBSCRIBED); + locked_notifyGroupChanged(*gi, GRP_SUBSCRIBED, false); IndicateConfigChanged(); mGroupsRepublish = true; } @@ -2274,7 +2279,7 @@ void p3GroupDistrib::receivePubKeys(){ if(locked_updateGroupPublishKey(it->second, key_item)){ mPubKeyAvailableGrpId.insert(key_item->grpId); - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY); + locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, false); // keep key if user not subscribed if(it->second.flags & RS_DISTRIB_SUBSCRIBED){ @@ -2483,7 +2488,7 @@ std::string p3GroupDistrib::publishMsg(RsDistribMsg *msg, bool personalSign) */ signedMsg->PeerId(mOwnId); - loadMsg(signedMsg, mOwnId, false); + loadMsg(signedMsg, mOwnId, false, false); /* done */ return msgId; @@ -3683,7 +3688,7 @@ std::ostream &operator<<(std::ostream &out, const GroupInfo &info) return out; } -void p3GroupDistrib::locked_notifyGroupChanged(GroupInfo &info, uint32_t flags) +void p3GroupDistrib::locked_notifyGroupChanged(GroupInfo &info, uint32_t flags, bool historical) { mGroupsChanged = true; info.grpChanged = true; diff --git a/libretroshare/src/services/p3distrib.h b/libretroshare/src/services/p3distrib.h index 0528c060d..1f59d2548 100644 --- a/libretroshare/src/services/p3distrib.h +++ b/libretroshare/src/services/p3distrib.h @@ -92,6 +92,22 @@ const uint32_t GROUP_KEY_DISTRIB_ADMIN = 0x0040; #endif +/* + * A data structure to store dummy (missing) msgs. + * They are added to the GroupInfo if there is a missing parent Msg of thread Msg + * Basic Logic is: + * + */ + +class RsDistribDummyMsg +{ + public: + std::string threadId; + std::string parentId; + std::string msgId; +}; + + //! for storing group keys to members of a group /*! * This key but be of many types, including private/public publish key, or admin prite key for group @@ -247,6 +263,24 @@ const uint32_t GRP_UNSUBSCRIBED = 0x0006; * Group id is the public admin keys id * */ + +/* + * To Handle Cache Data Loading.... we want to be able to seperate Historical + * from new data (primarily for the gui's benefit). + * to do this we have a mHistoricalCaches flag, which is automatically raised at startup, + * and a function is called to cancel it (HistoricalCachesDone()). + */ + +class CacheDataPending +{ + public: + + CacheDataPending(const CacheData &data, bool local, bool historical); + CacheData mData; + bool mLocal; + bool mHistorical; +}; + class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, public p3ThreadedService { public: @@ -272,18 +306,21 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu /* From RsThread */ virtual void run(); /* called once the thread is started */ + void HistoricalCachesDone(); // called when Stored Caches have been added to Pending List. + private: /* these lists are filled by the overloaded fns... then cleared by the thread */ - std::list mPendingLocalCache; - std::list mPendingRemoteCache; + bool mHistoricalCaches; // initially true.... falsified by HistoricalCachesDone() + std::list mPendingCaches; /* top level load */ - int loadAnyCache(const CacheData &data, bool local); + int loadAnyCache(const CacheData &data, bool local, bool historical); /* load cache files */ - void loadFileGroups(const std::string &filename, const std::string &src, bool local); - void loadFileMsgs(const std::string &filename, uint16_t cacheSubId, const std::string &src, uint32_t ts, bool local); + void loadFileGroups(const std::string &filename, const std::string &src, bool local, bool historical); + void loadFileMsgs(const std::string &filename, uint16_t cacheSubId, const std::string &src, uint32_t ts, bool local, bool historical); + bool backUpKeys(const std::list &keysToBackUp, std::string grpId); void locked_sharePubKey(); @@ -304,13 +341,13 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu * @param src src of msg (peer id) * @param local is this a local cache msg (your msg) */ - void loadMsg(RsDistribSignedMsg *msg, const std::string &src, bool local); + void loadMsg(RsDistribSignedMsg *msg, const std::string &src, bool local, bool historical); /*! * adds newgrp to grp set, GroupInfo type created and stored * @param newGrp grp to be added */ - void loadGroup(RsDistribGrp *newGrp); + void loadGroup(RsDistribGrp *newGrp, bool historical); /*! * Adds new keys dependent on whether it is an admin or publish key @@ -318,7 +355,7 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu * @param newKey key to be added * @return if key is loaded to group or stored return true */ - bool loadGroupKey(RsDistribGrpKey *newKey); + bool loadGroupKey(RsDistribGrpKey *newKey, bool historical); /***************************************************************************************/ @@ -411,7 +448,7 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu /*! * root version (p3Distrib::) of this function must be called */ - virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags); + virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags, bool historical); /*! * client (inheriting class) should use this to determing behaviour of @@ -419,18 +456,20 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu * @param group should be called when duplicate message loaded * @param the duplicate message * @param id + * @param historical: is this msg from an historical cache * @return successfully executed or not */ - virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string id) = 0; + virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string id, bool historical) = 0; /*! * Inheriting class should implement this as a response to a new msg arriving * @param * @param * @param id src of msg (peer id) + * @param historical: is this msg from an historical cache * @return */ - virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string id) = 0; + virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string id, bool historical) = 0; /***************************************************************************************/ /********************************* p3Config ********************************************/ diff --git a/libretroshare/src/services/p3forums.cc b/libretroshare/src/services/p3forums.cc index 58c5cd186..9d6ea11d6 100644 --- a/libretroshare/src/services/p3forums.cc +++ b/libretroshare/src/services/p3forums.cc @@ -542,7 +542,7 @@ bool p3Forums::getMessageCount(const std::string &fId, unsigned int &newCount, u #include "pqi/pqinotify.h" -void p3Forums::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) +void p3Forums::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags, bool historical) { const std::string &grpId = grp.grpId; std::string msgId; @@ -551,11 +551,17 @@ void p3Forums::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) switch(flags) { case GRP_NEW_UPDATE: - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_NEW, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_NEW, grpId, msgId, nullId); + } rsicontrol->getNotify().notifyListChange(NOTIFY_LIST_FORUMLIST_LOCKED, NOTIFY_TYPE_ADD); break; case GRP_UPDATE: - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_UPDATE, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_UPDATE, grpId, msgId, nullId); + } rsicontrol->getNotify().notifyListChange(NOTIFY_LIST_FORUMLIST_LOCKED, NOTIFY_TYPE_MOD); break; case GRP_LOAD_KEY: @@ -570,21 +576,25 @@ void p3Forums::locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags) rsicontrol->getNotify().notifyListChange(NOTIFY_LIST_FORUMLIST_LOCKED, NOTIFY_TYPE_DEL); break; } - return p3GroupDistrib::locked_notifyGroupChanged(grp, flags); + return p3GroupDistrib::locked_notifyGroupChanged(grp, flags, historical); } -bool p3Forums::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Forums::locked_eventDuplicateMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { return true; } -bool p3Forums::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id) +bool p3Forums::locked_eventNewMsg(GroupInfo *grp, RsDistribMsg *msg, std::string id, bool historical) { std::string grpId = msg->grpId; std::string msgId = msg->msgId; std::string nullId; - getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_MSG, grpId, msgId, nullId); + if (!historical) + { + getPqiNotify()->AddFeedItem(RS_FEED_ITEM_FORUM_MSG, grpId, msgId, nullId); + } + return true; } diff --git a/libretroshare/src/services/p3forums.h b/libretroshare/src/services/p3forums.h index c959a44a1..2dc21e28f 100644 --- a/libretroshare/src/services/p3forums.h +++ b/libretroshare/src/services/p3forums.h @@ -67,10 +67,9 @@ virtual bool getMessageCount(const std::string &fId, unsigned int &newCount, uns /****************** Event Feedback (Overloaded form p3distrib) *************************/ /***************************************************************************************/ -virtual void locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags); -virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string); -virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string); - +virtual void locked_notifyGroupChanged(GroupInfo &grp, uint32_t flags, bool historical); +virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); +virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, std::string, bool historical); /****************************************/ diff --git a/libretroshare/src/util/rsrandom.cc b/libretroshare/src/util/rsrandom.cc index 1a6adf2f3..bddccb997 100644 --- a/libretroshare/src/util/rsrandom.cc +++ b/libretroshare/src/util/rsrandom.cc @@ -10,9 +10,12 @@ RsMutex RSRandom::rndMtx ; #ifdef WINDOWS_SYS static bool auto_seed = RSRandom::seed( (time(NULL) + ((uint32_t) pthread_self().p)*0x1293fe)^0x18e34a12 ) ; #else -static bool auto_seed = RSRandom::seed( (time(NULL) + pthread_self()*0x1293fe + (getpid()^0x113ef76b))^0x18e34a12 ) ; + #ifdef __APPLE__ + static bool auto_seed = RSRandom::seed( (time(NULL) + pthread_mach_thread_np(pthread_self())*0x1293fe + (getpid()^0x113ef76b))^0x18e34a12 ) ; + #else + static bool auto_seed = RSRandom::seed( (time(NULL) + pthread_self()*0x1293fe + (getpid()^0x113ef76b))^0x18e34a12 ) ; + #endif #endif - bool RSRandom::seed(uint32_t s) { RsStackMutex mtx(rndMtx) ;