mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-01-16 09:57:19 -05:00
Properly plug deep search in GXS search
Some modifications breaks retrocompatibility of GXS search:
remove horrible templated RsTypeSerializer::serial_process for
RsGxsGroupSummary with hardcoded member names
RsGxsGroupSummary doesn't use old TLV serialization format anymore
RsGxsGroupSummary remove unused description member
RsGxsGroupSummary derive from RsSerializable and use serialization
helper macro
Add autor id and signature flags to the index so there is no need to
retrive them from GXS, thus improving performances
RsGroupNetworkStats initialize members properly
RsGxsGroupSummary rename members to follow usual mMemberName convention
This commit is contained in:
Gioacchino Mazzurco
parent
6982ae6cd5
commit
f97dc8a125
@ -118,6 +118,11 @@ struct DeepSearch
|
||||
|
||||
chanUrl.setQueryKV("publishDate", date);
|
||||
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
|
||||
if(!chan.mMeta.mAuthorId.isNull())
|
||||
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
|
||||
if(chan.mMeta.mSignFlags)
|
||||
chanUrl.setQueryKV( "signFlags",
|
||||
std::to_string(chan.mMeta.mSignFlags) );
|
||||
std::string rsLink(chanUrl.toString());
|
||||
|
||||
// store the RS link so we are able to retrive it on matching search
|
||||
|
||||
@ -74,23 +74,21 @@ public:
|
||||
};
|
||||
|
||||
/*!
|
||||
* This is used to query network statistics for a given group. This is useful to e.g. show group
|
||||
* popularity, or number of visible messages for unsubscribed group.
|
||||
* This is used to query network statistics for a given group. This is useful
|
||||
* to e.g. show group popularity, or number of visible messages for unsubscribed
|
||||
* group.
|
||||
*/
|
||||
|
||||
class RsGroupNetworkStats
|
||||
struct RsGroupNetworkStats
|
||||
{
|
||||
public:
|
||||
RsGroupNetworkStats()
|
||||
{
|
||||
mMaxVisibleCount = 0 ;
|
||||
}
|
||||
RsGroupNetworkStats() :
|
||||
mSuppliers(0), mMaxVisibleCount(0), mGrpAutoSync(false),
|
||||
mAllowMsgSync(false), mLastGroupModificationTS(0) {}
|
||||
|
||||
uint32_t mSuppliers ;
|
||||
uint32_t mMaxVisibleCount ;
|
||||
bool mGrpAutoSync ;
|
||||
bool mAllowMsgSync;
|
||||
time_t mLastGroupModificationTS ;
|
||||
uint32_t mSuppliers;
|
||||
uint32_t mMaxVisibleCount;
|
||||
bool mGrpAutoSync;
|
||||
bool mAllowMsgSync;
|
||||
time_t mLastGroupModificationTS;
|
||||
};
|
||||
|
||||
typedef std::map<RsGxsGroupId, std::vector<RsNxsMsg*> > NxsMsgDataResult;
|
||||
|
||||
@ -243,6 +243,7 @@
|
||||
#include <math.h>
|
||||
#include <sstream>
|
||||
#include <typeinfo>
|
||||
#include <iomanip>
|
||||
|
||||
#include "rsgxsnetservice.h"
|
||||
#include "gxssecurity.h"
|
||||
@ -5187,8 +5188,8 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std:
|
||||
std::map<RsGxsGroupId,RsGxsGroupSummary>& search_results_map(mDistantSearchResults[req]) ;
|
||||
|
||||
for(auto it(group_infos.begin());it!=group_infos.end();++it)
|
||||
if(search_results_map.find((*it).group_id) == search_results_map.end())
|
||||
grpMeta[(*it).group_id] = NULL;
|
||||
if(search_results_map.find((*it).mGroupId) == search_results_map.end())
|
||||
grpMeta[(*it).mGroupId] = NULL;
|
||||
|
||||
mDataStore->retrieveGxsGrpMetaData(grpMeta);
|
||||
|
||||
@ -5197,26 +5198,26 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std:
|
||||
// only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure
|
||||
|
||||
for(auto it(group_infos.begin());it!=group_infos.end();++it)
|
||||
if(grpMeta[(*it).group_id] == NULL)
|
||||
if(grpMeta[(*it).mGroupId] == NULL)
|
||||
{
|
||||
filtered_results.push_back(*it) ;
|
||||
|
||||
auto it2 = search_results_map.find((*it).group_id) ;
|
||||
auto it2 = search_results_map.find((*it).mGroupId) ;
|
||||
|
||||
if(it2 != search_results_map.end())
|
||||
{
|
||||
// update existing data
|
||||
|
||||
it2->second.popularity++ ;
|
||||
it2->second.number_of_messages = std::max(it2->second.number_of_messages,(*it).number_of_messages) ;
|
||||
it2->second.mPopularity++ ;
|
||||
it2->second.mNumberOfMessages = std::max(it2->second.mNumberOfMessages,(*it).mNumberOfMessages) ;
|
||||
}
|
||||
else
|
||||
{
|
||||
search_results_map[(*it).group_id] = *it;
|
||||
search_results_map[(*it).group_id].popularity = 1; // number of results so far
|
||||
search_results_map[(*it).mGroupId] = *it;
|
||||
search_results_map[(*it).mGroupId].mPopularity = 1; // number of results so far
|
||||
}
|
||||
|
||||
mObserver->receiveDistantSearchResults(req,(*it).group_id) ;
|
||||
mObserver->receiveDistantSearchResults(req,(*it).mGroupId) ;
|
||||
}
|
||||
}
|
||||
|
||||
@ -5277,12 +5278,6 @@ bool RsGxsNetService::search( const std::string& substring,
|
||||
{
|
||||
group_infos.clear();
|
||||
|
||||
RsGxsGrpMetaTemporaryMap grpMetaMap;
|
||||
{
|
||||
RS_STACK_MUTEX(mNxsMutex) ;
|
||||
mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
|
||||
}
|
||||
|
||||
#ifdef RS_DEEP_SEARCH
|
||||
std::vector<DeepSearch::SearchResult> results;
|
||||
DeepSearch::search(substring, results, 0);
|
||||
@ -5290,33 +5285,48 @@ bool RsGxsNetService::search( const std::string& substring,
|
||||
for(auto dsr : results)
|
||||
{
|
||||
RsUrl rUrl(dsr.mUrl);
|
||||
auto rit = rUrl.query().find("id");
|
||||
const auto& uQ(rUrl.query());
|
||||
auto rit = uQ.find("id");
|
||||
if(rit != rUrl.query().end())
|
||||
{
|
||||
RsGroupNetworkStats stats;
|
||||
RsGxsGroupId grpId(rit->second);
|
||||
RsGxsGrpMetaTemporaryMap::iterator mIt;
|
||||
if( !grpId.isNull() &&
|
||||
(mIt = grpMetaMap.find(grpId)) != grpMetaMap.end() &&
|
||||
getGroupNetworkStats(grpId, stats) )
|
||||
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
|
||||
{
|
||||
RsGxsGrpMetaData& gMeta(*mIt->second);
|
||||
RsGxsGroupSummary s;
|
||||
s.group_id = grpId;
|
||||
s.group_name = gMeta.mGroupName;
|
||||
s.search_context = dsr.mSnippet;
|
||||
s.sign_flags = gMeta.mSignFlags;
|
||||
s.publish_ts = gMeta.mSignFlags;
|
||||
s.author_id = gMeta.mAuthorId;
|
||||
s.number_of_messages = stats.mMaxVisibleCount;
|
||||
s.last_message_ts = stats.mLastGroupModificationTS;
|
||||
s.popularity = gMeta.mPop;
|
||||
|
||||
s.mGroupId = grpId;
|
||||
|
||||
if((rit = uQ.find("name")) != uQ.end())
|
||||
s.mGroupName = rit->second;
|
||||
if((rit = uQ.find("signFlags")) != uQ.end())
|
||||
s.mSignFlags = std::stoul(rit->second);
|
||||
if((rit = uQ.find("publishDate")) != uQ.end())
|
||||
{
|
||||
std::istringstream ss(rit->second);
|
||||
std::tm tm;
|
||||
ss >> std::get_time(&tm, "%Y%m%d");
|
||||
s.mPublishTs = mktime(&tm);
|
||||
}
|
||||
if((rit = uQ.find("authorId")) != uQ.end())
|
||||
s.mAuthorId = RsGxsId(rit->second);
|
||||
|
||||
s.mSearchContext = dsr.mSnippet;
|
||||
|
||||
s.mNumberOfMessages = stats.mMaxVisibleCount;
|
||||
s.mLastMessageTs = stats.mLastGroupModificationTS;
|
||||
s.mPopularity = stats.mSuppliers;
|
||||
|
||||
group_infos.push_back(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
#else // RS_DEEP_SEARCH
|
||||
RsGxsGrpMetaTemporaryMap grpMetaMap;
|
||||
{
|
||||
RS_STACK_MUTEX(mNxsMutex) ;
|
||||
mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
|
||||
}
|
||||
RsGroupNetworkStats stats ;
|
||||
for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it)
|
||||
if(termSearch(it->second->mGroupName,substring))
|
||||
|
||||
@ -216,22 +216,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, RsGxsGroupSummary& gs, const std::string& member_name )
|
||||
{
|
||||
RsTypeSerializer::serial_process (j,ctx,gs.group_id ,member_name+"-group_id") ; // RsGxsGroupId group_id ;
|
||||
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ;
|
||||
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_COMMENT ,gs.group_description,member_name+"-group_description") ; // std::string group_description ;
|
||||
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ;
|
||||
RsTypeSerializer::serial_process (j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ;
|
||||
RsTypeSerializer::serial_process (j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ;
|
||||
RsTypeSerializer::serial_process (j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ;
|
||||
RsTypeSerializer::serial_process<time_t> (j,ctx,gs.last_message_ts ,member_name+"-last_message_ts") ; // time_t last_message_ts ;
|
||||
RsTypeSerializer::serial_process<uint32_t>(j,ctx,gs.sign_flags ,member_name+"-sign_flags") ; // uint32_t sign_flags ;
|
||||
RsTypeSerializer::serial_process<uint32_t>(j,ctx,gs.popularity ,member_name+"-popularity") ; // uint32_t popularity ;
|
||||
}
|
||||
|
||||
|
||||
//===========================================================================================================================================//
|
||||
// Interface with rest of the software //
|
||||
//===========================================================================================================================================//
|
||||
@ -1102,7 +1086,7 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id
|
||||
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
|
||||
|
||||
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
|
||||
std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl;
|
||||
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
|
||||
|
||||
auto it = mSearchableServices.find(result_gs->service) ;
|
||||
|
||||
|
||||
@ -34,26 +34,42 @@
|
||||
#include "util/rsdeprecate.h"
|
||||
|
||||
/*!
|
||||
* \brief The RsGxsGroupSymmary struct
|
||||
* This structure is used to transport group summary information when a GXS service is searched. It contains the group information
|
||||
* as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make
|
||||
* search responses as light as possible.
|
||||
* This structure is used to transport group summary information when a GXS
|
||||
* service is searched. It contains the group information as well as a context
|
||||
* string to tell where the information was found. It is more compact than a
|
||||
* GroupMeta object, so as to make search responses as light as possible.
|
||||
*/
|
||||
struct RsGxsGroupSummary
|
||||
struct RsGxsGroupSummary : RsSerializable
|
||||
{
|
||||
RsGxsGroupSummary() : publish_ts(0), number_of_messages(0),last_message_ts(0),sign_flags(0),popularity(0) {}
|
||||
RsGxsGroupSummary() :
|
||||
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
|
||||
mSignFlags(0),mPopularity(0) {}
|
||||
|
||||
RsGxsGroupId group_id ;
|
||||
RsGxsGroupId mGroupId;
|
||||
std::string mGroupName;
|
||||
RsGxsId mAuthorId;
|
||||
time_t mPublishTs;
|
||||
uint32_t mNumberOfMessages;
|
||||
time_t mLastMessageTs;
|
||||
uint32_t mSignFlags;
|
||||
uint32_t mPopularity;
|
||||
|
||||
std::string group_name ;
|
||||
RS_DEPRECATED std::string group_description;
|
||||
std::string search_context ;
|
||||
RsGxsId author_id ;
|
||||
time_t publish_ts ;
|
||||
uint32_t number_of_messages ;
|
||||
time_t last_message_ts ;
|
||||
uint32_t sign_flags ;
|
||||
uint32_t popularity ;
|
||||
std::string mSearchContext;
|
||||
|
||||
/// @see RsSerializable::serial_process
|
||||
void serial_process( RsGenericSerializer::SerializeJob j,
|
||||
RsGenericSerializer::SerializeContext& ctx )
|
||||
{
|
||||
RS_SERIAL_PROCESS(mGroupId);
|
||||
RS_SERIAL_PROCESS(mGroupName);
|
||||
RS_SERIAL_PROCESS(mAuthorId);
|
||||
RS_SERIAL_PROCESS(mPublishTs);
|
||||
RS_SERIAL_PROCESS(mNumberOfMessages);
|
||||
RS_SERIAL_PROCESS(mLastMessageTs);
|
||||
RS_SERIAL_PROCESS(mSignFlags);
|
||||
RS_SERIAL_PROCESS(mPopularity);
|
||||
RS_SERIAL_PROCESS(mSearchContext);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -1703,21 +1703,18 @@ bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<Rs
|
||||
|
||||
bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
|
||||
{
|
||||
RsGxsGroupSummary gs ;
|
||||
RsGxsGroupSummary gs;
|
||||
|
||||
if(netService()->retrieveDistantGroupSummary(group_id,gs))
|
||||
{
|
||||
// This is a placeholder information by the time we receive the full group meta data.
|
||||
|
||||
distant_group.mDescription = gs.group_description;
|
||||
|
||||
distant_group.mMeta.mGroupId = gs.group_id ;
|
||||
distant_group.mMeta.mGroupName = gs.group_name;
|
||||
distant_group.mMeta.mGroupId = gs.mGroupId ;
|
||||
distant_group.mMeta.mGroupName = gs.mGroupName;
|
||||
distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ;
|
||||
distant_group.mMeta.mSignFlags = gs.sign_flags;
|
||||
distant_group.mMeta.mSignFlags = gs.mSignFlags;
|
||||
|
||||
distant_group.mMeta.mPublishTs = gs.publish_ts;
|
||||
distant_group.mMeta.mAuthorId = gs.author_id;
|
||||
distant_group.mMeta.mPublishTs = gs.mPublishTs;
|
||||
distant_group.mMeta.mAuthorId = gs.mAuthorId;
|
||||
|
||||
distant_group.mMeta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able.
|
||||
|
||||
@ -1726,9 +1723,9 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann
|
||||
|
||||
distant_group.mMeta.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ;
|
||||
|
||||
distant_group.mMeta.mPop = gs.popularity; // Popularity = number of friend subscribers
|
||||
distant_group.mMeta.mVisibleMsgCount = gs.number_of_messages; // Max messages reported by friends
|
||||
distant_group.mMeta.mLastPost = gs.last_message_ts; // Timestamp for last message. Not used yet.
|
||||
distant_group.mMeta.mPop = gs.mPopularity; // Popularity = number of friend subscribers
|
||||
distant_group.mMeta.mVisibleMsgCount = gs.mNumberOfMessages; // Max messages reported by friends
|
||||
distant_group.mMeta.mLastPost = gs.mLastMessageTs; // Timestamp for last message. Not used yet.
|
||||
|
||||
return true ;
|
||||
}
|
||||
|
||||
@ -279,21 +279,22 @@ void GxsGroupFrameDialog::updateSearchResults()
|
||||
|
||||
QList<GroupItemInfo> group_items ;
|
||||
|
||||
for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
|
||||
if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end())
|
||||
for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
|
||||
if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end())
|
||||
{
|
||||
std::cerr << " adding new group " << it3->first << " " << it3->second.group_id << " \"" << it3->second.group_name << "\"" << std::endl;
|
||||
std::cerr << " adding new group " << it3->first << " "
|
||||
<< it3->second.mGroupId << " \""
|
||||
<< it3->second.mGroupName << "\"" << std::endl;
|
||||
|
||||
GroupItemInfo i ;
|
||||
i.id = QString(it3->second.group_id.toStdString().c_str()) ;
|
||||
i.name = QString::fromUtf8(it3->second.group_name.c_str()) ;
|
||||
i.description = QString::fromUtf8(it3->second.group_description.c_str()) ;
|
||||
GroupItemInfo i;
|
||||
i.id = QString(it3->second.mGroupId.toStdString().c_str());
|
||||
i.name = QString::fromUtf8(it3->second.mGroupName.c_str());
|
||||
i.popularity = 0; // could be set to the number of hits
|
||||
i.lastpost = QDateTime::fromTime_t(it3->second.last_message_ts);
|
||||
i.lastpost = QDateTime::fromTime_t(it3->second.mLastMessageTs);
|
||||
i.subscribeFlags = 0; // irrelevant here
|
||||
i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags) ;
|
||||
i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags) ;
|
||||
i.max_visible_posts = it3->second.number_of_messages ;
|
||||
i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags);
|
||||
i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags);
|
||||
i.max_visible_posts = it3->second.mNumberOfMessages;
|
||||
|
||||
group_items.push_back(i);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user