added auto-request of missing GXS ids for group posts authors and group authors

libretroshare/src/gxs/rsgenexchange.cc

@@ -77,10 +77,10 @@ RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService
   mAuthenPolicy(authenPolicy),
   MESSAGE_STORE_PERIOD(messageStorePeriod),
   mCleaning(false),
-  mLastClean(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD),	// this helps unsynchronising the checks for the different services
+  mLastClean((int)time(NULL) - (int)(RSRandom::random_u32() % MSG_CLEANUP_PERIOD)),	// this helps unsynchronising the checks for the different services
   mMsgCleanUp(NULL),
   mChecking(false),
-  mLastCheck(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD),	// this helps unsynchronising the checks for the different services
+  mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD)),	// this helps unsynchronising the checks for the different services
   mIntegrityCheck(NULL),
   CREATE_FAIL(0),
   CREATE_SUCCESS(1),

libretroshare/src/gxs/rsgxsutil.cc

@@ -27,9 +27,13 @@
 
 #include "rsgxsutil.h"
 #include "retroshare/rsgxsflags.h"
+#include "retroshare/rspeers.h"
 #include "pqi/pqihash.h"
 #include "gxs/rsgixs.h"
 
+static const uint32_t MAX_GXS_IDS_REQUESTS_NET   =  10 ; // max number of requests from cache/net (avoids killing the system!)
+
+//#define GXSUTIL_DEBUG 1
 
 RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, uint32_t messageStorePeriod, uint32_t chunkSize)
 : mDs(dataService), MESSAGE_STORE_PERIOD(messageStorePeriod), CHUNK_SIZE(chunkSize)
@@ -118,7 +122,6 @@ void RsGxsIntegrityCheck::run()
 
 bool RsGxsIntegrityCheck::check()
 {
-
     // first take out all the groups
     std::map<RsGxsGroupId, RsNxsGrp*> grp;
     mDs->retrieveNxsGrps(grp, true, true);
@@ -126,6 +129,7 @@ bool RsGxsIntegrityCheck::check()
     GxsMsgReq msgIds;
     GxsMsgReq grps;
 
+    std::set<RsGxsId> used_gxs_ids ;
     std::set<RsGxsGroupId> subscribed_groups ;
 
     // compute hash and compare to stored value, if it fails then simply add it
@@ -153,8 +157,11 @@ bool RsGxsIntegrityCheck::check()
 
 				    if(!grp->metaData->mAuthorId.isNull())
 				    {
-						std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->metaData->mAuthorId << std::endl;
+#ifdef GXSUTIL_DEBUG
-						mGixs->timeStampKey(grp->metaData->mAuthorId) ;
+					    std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl;
+#endif
+
+					    used_gxs_ids.insert(grp->metaData->mAuthorId) ;
 				    }
 			    }
 		    }
@@ -232,8 +239,10 @@ bool RsGxsIntegrityCheck::check()
 		    }
 		    else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end())
 		    {
+#ifdef GXSUTIL_DEBUG
 			    std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl;
-				mGixs->timeStampKey(msg->metaData->mAuthorId) ;
+#endif
+			    used_gxs_ids.insert(msg->metaData->mAuthorId) ;
 		    }
 
 		    delete msg;
@@ -252,6 +261,55 @@ bool RsGxsIntegrityCheck::check()
     }
     mDeletedMsgs = msgsToDel;
 
+#ifdef GXSUTIL_DEBUG
+    std::cerr << "At end of pass, this is the list used GXS ids: " << std::endl;
+    std::cerr << "  requesting them to GXS identity service to enforce loading." << std::endl;
+#endif
+
+    std::list<RsPeerId> connected_friends ;
+    rsPeers->getOnlineList(connected_friends) ;
+
+    std::vector<RsGxsId> gxs_ids ;
+
+    for(std::set<RsGxsId>::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it)
+    {
+	    gxs_ids.push_back(*it) ;
+#ifdef GXSUTIL_DEBUG
+	    std::cerr << "    " << *it <<  std::endl;
+#endif
+    }
+    int nb_requested_not_in_cache = 0;
+
+#ifdef GXSUTIL_DEBUG
+    std::cerr << "  issuing random get on friends for non existing IDs" << std::endl;
+#endif
+
+    for(;nb_requested_not_in_cache<MAX_GXS_IDS_REQUESTS_NET && gxs_ids.size()>0;)
+    {
+	    uint32_t n = RSRandom::random_u32() % gxs_ids.size() ;
+#ifdef GXSUTIL_DEBUG
+	    std::cerr << "    requesting ID " << gxs_ids[n] ;
+#endif
+
+	    if(!mGixs->haveKey(gxs_ids[n]))	// checks if we have it already in the cache (conservative way to ensure that we atually have it)
+	    { 
+		    mGixs->requestKey(gxs_ids[n],connected_friends);
+
+		    ++nb_requested_not_in_cache ;
+#ifdef GXSUTIL_DEBUG
+		    std::cerr << "  ... from cache/net" << std::endl;
+#endif
+	    }
+        else
+		    std::cerr << "  ... already in cache" << std::endl;
+
+	    gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ;
+	    gxs_ids.pop_back() ;
+    }
+#ifdef GXSUTIL_DEBUG
+        std::cerr << "  total actual cache requests: "<< nb_requested_not_in_cache << std::endl;
+#endif
+        
 	return true;
 }