Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-06-29 08:37:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

started to re-write the getMsgMeta and Ids in gxsdataaccess for better efficiency. Lastest msgs only net yet functional

Browse source
This commit is contained in:
csoler 2020-04-09 22:55:13 +02:00
parent b0e61376f1
commit 4e66455ac0
No known key found for this signature in database
GPG key ID: 7BCA522266C0804C
3 changed files with 126 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

2
libretroshare/src/gxs/rsgenexchange.cc
View file

@ -1193,7 +1193,7 @@ bool RsGenExchange::getGroupList(const uint32_t &token, std::list<RsGxsGroupId>
bool RsGenExchange::getMsgList(const uint32_t &token, bool RsGenExchange::getMsgList(const uint32_t &token,
GxsMsgIdResult &msgIds) GxsMsgIdResult &msgIds)
{ {
return mDataAccess->getMsgList(token, msgIds); return mDataAccess->getMsgIdList(token, msgIds);
} }
bool RsGenExchange::getMsgRelatedList(const uint32_t &token, MsgRelatedIdResult &msgIds) bool RsGenExchange::getMsgRelatedList(const uint32_t &token, MsgRelatedIdResult &msgIds)

212
libretroshare/src/gxs/rsgxsdataaccess.cc
View file

@ -603,7 +603,7 @@ bool RsGxsDataAccess::getMsgRelatedList(const uint32_t &token, MsgRelatedIdResul
return true; return true;
} }
bool RsGxsDataAccess::getMsgList(const uint32_t& token, GxsMsgIdResult& msgIds) bool RsGxsDataAccess::getMsgIdList(const uint32_t& token, GxsMsgIdResult& msgIds)
{ {
RsStackMutex stack(mDataMutex); RsStackMutex stack(mDataMutex);
@ -805,7 +805,7 @@ void RsGxsDataAccess::processRequests()
} }
else if((mir = dynamic_cast<MsgIdReq*>(req)) != NULL) else if((mir = dynamic_cast<MsgIdReq*>(req)) != NULL)
{ {
ok = getMsgList(mir); ok = getMsgIdList(mir);
} }
else if((mri = dynamic_cast<MsgRelatedInfoReq*>(req)) != NULL) else if((mri = dynamic_cast<MsgRelatedInfoReq*>(req)) != NULL)
{ {
@ -961,7 +961,7 @@ bool RsGxsDataAccess::getMsgData(MsgDataReq* req)
const RsTokReqOptions& opts(req->Options); const RsTokReqOptions& opts(req->Options);
// filter based on options // filter based on options
getMsgList(req->mMsgIds, opts, msgIdOut); getMsgIdList(req->mMsgIds, opts, msgIdOut);
// If the list is empty because of filtering do not retrieve from DB // If the list is empty because of filtering do not retrieve from DB
if((opts.mMsgFlagMask || opts.mStatusMask) && msgIdOut.empty()) if((opts.mMsgFlagMask || opts.mStatusMask) && msgIdOut.empty())
@ -979,24 +979,21 @@ bool RsGxsDataAccess::getMsgSummary(MsgMetaReq* req)
const RsTokReqOptions& opts(req->Options); const RsTokReqOptions& opts(req->Options);
// filter based on options // filter based on options
getMsgList(req->mMsgIds, opts, msgIdOut); getMsgMetaDataList(req->mMsgIds, opts, req->mMsgMetaData);
// If the list is empty because of filtering do not retrieve from DB // // If the list is empty because of filtering do not retrieve from DB
if((opts.mMsgFlagMask || opts.mStatusMask) && msgIdOut.empty()) // if((opts.mMsgFlagMask || opts.mStatusMask) && msgIdOut.empty())
return true; // return true;
//
mDataStore->retrieveGxsMsgMetaData(msgIdOut, req->mMsgMetaData); // mDataStore->retrieveGxsMsgMetaData(msgIdOut, req->mMsgMetaData);
return true; return true;
} }
bool RsGxsDataAccess::getMsgMetaDataList( const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgMetaResult& result )
bool RsGxsDataAccess::getMsgList(
const GxsMsgReq& msgIds, const RsTokReqOptions& opts,
GxsMsgReq& msgIdsOut )
{ {
GxsMsgMetaResult result; // First get all message metas, then filter out the ones we want to keep.
result.clear();
mDataStore->retrieveGxsMsgMetaData(msgIds, result); mDataStore->retrieveGxsMsgMetaData(msgIds, result);
/* CASEs this handles. /* CASEs this handles.
@ -1041,123 +1038,136 @@ bool RsGxsDataAccess::getMsgList(
} }
GxsMsgMetaResult::iterator meta_it; GxsMsgMetaResult::iterator meta_it;
MsgMetaFilter metaFilter; //MsgMetaFilter metaFilter;
for(meta_it = result.begin(); meta_it != result.end(); ++meta_it) for(meta_it = result.begin(); meta_it != result.end(); ++meta_it)
{ {
const RsGxsGroupId& grpId = meta_it->first; const RsGxsGroupId& grpId = meta_it->first;
metaFilter[grpId] = std::map<RsGxsMessageId, RsGxsMsgMetaData*>(); //auto& filter( metaFilter[grpId] ); // does the initialization of metaFilter[grpId] and avoids further O(log(n)) calls
std::vector<RsGxsMsgMetaData*>& metaV = meta_it->second;
const std::vector<RsGxsMsgMetaData*>& metaV = meta_it->second;
if (onlyLatestMsgs) // THIS ONE IS HARD -> LOTS OF COMP. if (onlyLatestMsgs) // THIS ONE IS HARD -> LOTS OF COMP.
{ {
std::vector<RsGxsMsgMetaData*>::const_iterator vit = metaV.begin(); #ifdef TODO
// RUN THROUGH ALL MSGS... in map origId -> TS. // RUN THROUGH ALL MSGS... in map origId -> TS.
std::map<RsGxsMessageId, std::pair<RsGxsMessageId, rstime_t> > origMsgTs; std::map<RsGxsMessageId, std::pair<RsGxsMessageId, rstime_t> > origMsgTs;
std::map<RsGxsMessageId, std::pair<RsGxsMessageId, rstime_t> >::iterator oit;
for(; vit != metaV.end(); ++vit) for(uint32_t i=0;i<metaV.size();++i)
{ {
RsGxsMsgMetaData* msgMeta = *vit; RsGxsMsgMetaData* msgMeta = metaV[i];
/* if we are grabbing thread Head... then parentId == empty. */ /* if we are grabbing thread Head... then parentId == empty. */
if (onlyThreadHeadMsgs) if (onlyThreadHeadMsgs && !(msgMeta->mParentId.isNull()))
{ {
if (!(msgMeta->mParentId.isNull())) delete msgMeta;
metaV[i] = nullptr;
continue;
}
auto oit = origMsgTs.find(msgMeta->mOrigMsgId);
bool addMsg = false;
if (oit != origMsgTs.end())
{
if(oit->second.second > msgMeta->mPublishTs)
{ {
continue; #ifdef DATA_DEBUG
std::cerr << "RsGxsDataAccess::getMsgList() Found New OrigMsgId: ";
std::cerr << msgMeta->mOrigMsgId;
std::cerr << " MsgId: " << msgMeta->mMsgId;
std::cerr << " TS: " << msgMeta->mPublishTs;
std::cerr << std::endl;
#endif
origMsgTs[msgMeta->mOrigMsgId] = std::make_pair(msgMeta->mMsgId, msgMeta->mPublishTs); // add as latest. (overwriting if necessary)
} }
} }
else
oit = origMsgTs.find(msgMeta->mOrigMsgId);
bool addMsg = false;
if (oit == origMsgTs.end())
{ {
#ifdef DATA_DEBUG delete msgMeta;
std::cerr << "RsGxsDataAccess::getMsgList() Found New OrigMsgId: "; metaV[i] = nullptr;
std::cerr << msgMeta->mOrigMsgId; continue;
std::cerr << " MsgId: " << msgMeta->mMsgId;
std::cerr << " TS: " << msgMeta->mPublishTs;
std::cerr << std::endl;
#endif
addMsg = true;
}
// check timestamps.
else if (oit->second.second < msgMeta->mPublishTs)
{
#ifdef DATA_DEBUG
std::cerr << "RsGxsDataAccess::getMsgList() Found Later Msg. OrigMsgId: ";
std::cerr << msgMeta->mOrigMsgId;
std::cerr << " MsgId: " << msgMeta->mMsgId;
std::cerr << " TS: " << msgMeta->mPublishTs;
#endif
addMsg = true;
} }
if (addMsg)
{
// add as latest. (overwriting if necessary)
origMsgTs[msgMeta->mOrigMsgId] = std::make_pair(msgMeta->mMsgId, msgMeta->mPublishTs);
metaFilter[grpId].insert(std::make_pair(msgMeta->mMsgId, msgMeta));
}
} }
// Add the discovered Latest Msgs. // Add the discovered Latest Msgs.
for(oit = origMsgTs.begin(); oit != origMsgTs.end(); ++oit) for(auto oit = origMsgTs.begin(); oit != origMsgTs.end(); ++oit)
{ {
msgIdsOut[grpId].insert(oit->second.first); msgIdsOut[grpId].insert(oit->second.first);
} }
#endif
} }
else // ALL OTHER CASES. else // ALL OTHER CASES.
{ {
std::vector<RsGxsMsgMetaData*>::const_iterator vit = metaV.begin(); for(uint32_t i=0;i<metaV.size();++i)
for(; vit != metaV.end(); ++vit)
{ {
RsGxsMsgMetaData* msgMeta = *vit; RsGxsMsgMetaData* msgMeta = metaV[i];
bool add = false; bool add = false;
/* if we are grabbing thread Head... then parentId == empty. */ /* if we are grabbing thread Head... then parentId == empty. */
if (onlyThreadHeadMsgs) if (onlyThreadHeadMsgs && !msgMeta->mParentId.isNull())
{ {
if (!(msgMeta->mParentId.isNull())) delete msgMeta;
{ metaV[i] = nullptr;
continue; continue;
}
} }
if (onlyOrigMsgs && !msgMeta->mOrigMsgId.isNull() && msgMeta->mMsgId != msgMeta->mOrigMsgId)
if (onlyOrigMsgs) {
{ delete msgMeta;
if (msgMeta->mMsgId == msgMeta->mOrigMsgId) metaV[i] = nullptr;
{ continue;
add = true;
}
} }
else
{
add = true;
}
if (add)
{
msgIdsOut[grpId].insert(msgMeta->mMsgId);
metaFilter[grpId].insert(std::make_pair(msgMeta->mMsgId, msgMeta));
}
} }
} }
} }
filterMsgList(msgIdsOut, opts, metaFilter); // collapse results while keeping the order, eliminating empty slots
metaFilter.clear(); for(auto it(result.begin());it!=result.end();++it)
{
uint32_t j=0; // j is the end of the cleaned-up tab, at the first available place
for(uint32_t i=0;i<it->second.size();++i) // i is the index in the tab possibly containing nullptr's
if(it->second[i] != nullptr)
{
it->second[j] = it->second[i]; // move the pointer to the first available place
++j;
}
it->second.resize(j); // normally all pointers have been moved forward so there is nothing to delete here.
}
// filterMsgIdList(msgIdsOut, opts, metaFilter); // this call is absurd: we already have in metaFilter the content we want.
//metaFilter.clear();
// delete meta data
//cleanseMsgMetaMap(result);
return true;
}
bool RsGxsDataAccess::getMsgIdList( const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgReq& msgIdsOut )
{
GxsMsgMetaResult result;
getMsgMetaDataList( msgIdsOut, opts, result );
// extract MessageIds
msgIdsOut.clear();
for(auto it(result.begin());it!=result.end();++it)
{
auto& id_set(msgIdsOut[it->first]);
for(uint32_t i=0;i<it->second.size();++i)
id_set.insert(it->second[i]->mMsgId);
}
// delete meta data // delete meta data
cleanseMsgMetaMap(result); cleanseMsgMetaMap(result);
@ -1441,7 +1451,7 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req)
GxsMsgIdResult filteredOutMsgIds; GxsMsgIdResult filteredOutMsgIds;
filteredOutMsgIds[grpId] = outMsgIds; filteredOutMsgIds[grpId] = outMsgIds;
filterMsgList(filteredOutMsgIds, opts, filterMap); filterMsgIdList(filteredOutMsgIds, opts, filterMap);
if(!filteredOutMsgIds[grpId].empty()) if(!filteredOutMsgIds[grpId].empty())
{ {
@ -1571,7 +1581,7 @@ bool RsGxsDataAccess::getServiceStatistic(ServiceStatisticRequest *req)
return true; return true;
} }
bool RsGxsDataAccess::getMsgList(MsgIdReq* req) bool RsGxsDataAccess::getMsgIdList(MsgIdReq* req)
{ {
GxsMsgMetaResult result; GxsMsgMetaResult result;
@ -1599,7 +1609,7 @@ bool RsGxsDataAccess::getMsgList(MsgIdReq* req)
GxsMsgReq msgIdOut; GxsMsgReq msgIdOut;
// filter based on options // filter based on options
getMsgList(req->mMsgIdResult, req->Options, msgIdOut); getMsgIdList(req->mMsgIdResult, req->Options, msgIdOut);
req->mMsgIdResult = msgIdOut; req->mMsgIdResult = msgIdOut;
return true; return true;
@ -1624,12 +1634,9 @@ void RsGxsDataAccess::cleanseMsgMetaMap(GxsMsgMetaResult& result)
return; return;
} }
void RsGxsDataAccess::filterMsgList( void RsGxsDataAccess::filterMsgIdList( GxsMsgIdResult& resultsMap, const RsTokReqOptions& opts, const MsgMetaFilter& msgMetas ) const
GxsMsgIdResult& resultsMap, const RsTokReqOptions& opts,
const MsgMetaFilter& msgMetas ) const
{ {
for( GxsMsgIdResult::iterator grpIt = resultsMap.begin(); for( GxsMsgIdResult::iterator grpIt = resultsMap.begin(); grpIt != resultsMap.end(); ++grpIt )
grpIt != resultsMap.end(); ++grpIt )
{ {
const RsGxsGroupId& groupId(grpIt->first); const RsGxsGroupId& groupId(grpIt->first);
std::set<RsGxsMessageId>& msgsIdSet(grpIt->second); std::set<RsGxsMessageId>& msgsIdSet(grpIt->second);
@ -1642,8 +1649,7 @@ void RsGxsDataAccess::filterMsgList(
<< std::endl; << std::endl;
#endif #endif
for( std::set<RsGxsMessageId>::iterator msgIdIt = msgsIdSet.begin(); for( std::set<RsGxsMessageId>::iterator msgIdIt = msgsIdSet.begin(); msgIdIt != msgsIdSet.end(); )
msgIdIt != msgsIdSet.end(); )
{ {
const RsGxsMessageId& msgId(*msgIdIt); const RsGxsMessageId& msgId(*msgIdIt);
const std::map<RsGxsMessageId, RsGxsMsgMetaData*>& msgsMetaMap = const std::map<RsGxsMessageId, RsGxsMsgMetaData*>& msgsMetaMap =
@ -1657,8 +1663,10 @@ void RsGxsDataAccess::filterMsgList(
keep = checkMsgFilter(opts, msgsMetaMapIt->second); keep = checkMsgFilter(opts, msgsMetaMapIt->second);
} }
if(keep) ++msgIdIt; if(keep)
else msgIdIt = msgsIdSet.erase(msgIdIt); ++msgIdIt;
else
msgIdIt = msgsIdSet.erase(msgIdIt);
} }
#ifdef DATA_DEBUG #ifdef DATA_DEBUG

19
libretroshare/src/gxs/rsgxsdataaccess.h
View file

@ -204,7 +204,8 @@ public:
* @param token request token to be redeemed * @param token request token to be redeemed
* @param msgIds * @param msgIds
*/ */
bool getMsgList(const uint32_t &token, GxsMsgIdResult &msgIds); bool getMsgIdList(const uint32_t &token, GxsMsgIdResult &msgIds);
/*! /*!
* Retrieve msg list for a given token for message related info * Retrieve msg list for a given token for message related info
@ -382,8 +383,18 @@ private:
* @param req * @param req
* @return false if unsuccessful, true otherwise * @return false if unsuccessful, true otherwise
*/ */
bool getMsgList(MsgIdReq* req); bool getMsgIdList(MsgIdReq* req);
/*!
* Attempts to retrieve msg Meta list from data store
* Computationally/CPU-Bandwidth expensive
*
* @param msgIds List of message Ids for the Message Metas to retrieve
* @param opts GxsRequest options
* @param result Map of Meta information for messages
*
*/
bool getMsgMetaDataList( const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgMetaResult& result );
/*! /*!
* Attempts to retrieve group meta data from data store * Attempts to retrieve group meta data from data store
@ -449,7 +460,7 @@ private:
* @param opts the request options set by user * @param opts the request options set by user
* @param meta The accompanying meta information for msg, ids * @param meta The accompanying meta information for msg, ids
*/ */
void filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const; void filterMsgIdList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const;
/*! /*!
* This filter msgs based of options supplied (at the moment just status masks) * This filter msgs based of options supplied (at the moment just status masks)
@ -486,7 +497,7 @@ private:
* @param opts the options used to parameterise the id filter * @param opts the options used to parameterise the id filter
* @param msgIdsOut the left overs ids after filter is applied to msgIds * @param msgIdsOut the left overs ids after filter is applied to msgIds
*/ */
bool getMsgList(const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgReq& msgIdsOut); bool getMsgIdList(const GxsMsgReq& msgIds, const RsTokReqOptions& opts, GxsMsgReq& msgIdsOut);
private: private:
bool locked_clearRequest(const uint32_t &token); bool locked_clearRequest(const uint32_t &token);