Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-06-21 04:44:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Properly plug deep search in GXS search

Browse source 
Some modifications breaks retrocompatibility of GXS search:
  remove horrible templated RsTypeSerializer::serial_process for
    RsGxsGroupSummary with hardcoded member names
  RsGxsGroupSummary doesn't use old TLV serialization format anymore
  RsGxsGroupSummary remove unused description member
  RsGxsGroupSummary derive from RsSerializable and use serialization
    helper macro
Add autor id and signature flags to the index so there is no need to
  retrive them from GXS, thus improving performances
RsGroupNetworkStats initialize members properly
RsGxsGroupSummary rename members to follow usual mMemberName convention
This commit is contained in:
Gioacchino Mazzurco 2018-07-22 21:33:40 +02:00
parent 6982ae6cd5
commit f97dc8a125
No known key found for this signature in database
GPG key ID: A1FBCA3872E87051
7 changed files with 111 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

5
libretroshare/src/deep_search/deep_search.h
View file

@ -118,6 +118,11 @@ struct DeepSearch
chanUrl.setQueryKV("publishDate", date); chanUrl.setQueryKV("publishDate", date);
chanUrl.setQueryKV("name", chan.mMeta.mGroupName); chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString()); std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search // store the RS link so we are able to retrive it on matching search

24
libretroshare/src/gxs/rsgds.h
View file

@ -74,23 +74,21 @@ public:
}; };
/*! /*!
* This is used to query network statistics for a given group. This is useful to e.g. show group * This is used to query network statistics for a given group. This is useful
* popularity, or number of visible messages for unsubscribed group. * to e.g. show group popularity, or number of visible messages for unsubscribed
* group.
*/ */
struct RsGroupNetworkStats
class RsGroupNetworkStats
{ {
public: RsGroupNetworkStats() :
RsGroupNetworkStats() mSuppliers(0), mMaxVisibleCount(0), mGrpAutoSync(false),
{ mAllowMsgSync(false), mLastGroupModificationTS(0) {}
mMaxVisibleCount = 0 ;
}
uint32_t mSuppliers ; uint32_t mSuppliers;
uint32_t mMaxVisibleCount ; uint32_t mMaxVisibleCount;
bool mGrpAutoSync ; bool mGrpAutoSync;
bool mAllowMsgSync; bool mAllowMsgSync;
time_t mLastGroupModificationTS ; time_t mLastGroupModificationTS;
}; };
typedef std::map<RsGxsGroupId, std::vector<RsNxsMsg*> > NxsMsgDataResult; typedef std::map<RsGxsGroupId, std::vector<RsNxsMsg*> > NxsMsgDataResult;

70
libretroshare/src/gxs/rsgxsnetservice.cc
View file

@ -243,6 +243,7 @@
#include <math.h> #include <math.h>
#include <sstream> #include <sstream>
#include <typeinfo> #include <typeinfo>
#include <iomanip>
#include "rsgxsnetservice.h" #include "rsgxsnetservice.h"
#include "gxssecurity.h" #include "gxssecurity.h"
@ -5187,8 +5188,8 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std:
std::map<RsGxsGroupId,RsGxsGroupSummary>& search_results_map(mDistantSearchResults[req]) ; std::map<RsGxsGroupId,RsGxsGroupSummary>& search_results_map(mDistantSearchResults[req]) ;
for(auto it(group_infos.begin());it!=group_infos.end();++it) for(auto it(group_infos.begin());it!=group_infos.end();++it)
if(search_results_map.find((*it).group_id) == search_results_map.end()) if(search_results_map.find((*it).mGroupId) == search_results_map.end())
grpMeta[(*it).group_id] = NULL; grpMeta[(*it).mGroupId] = NULL;
mDataStore->retrieveGxsGrpMetaData(grpMeta); mDataStore->retrieveGxsGrpMetaData(grpMeta);
@ -5197,26 +5198,26 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std:
// only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure // only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure
for(auto it(group_infos.begin());it!=group_infos.end();++it) for(auto it(group_infos.begin());it!=group_infos.end();++it)
if(grpMeta[(*it).group_id] == NULL) if(grpMeta[(*it).mGroupId] == NULL)
{ {
filtered_results.push_back(*it) ; filtered_results.push_back(*it) ;
auto it2 = search_results_map.find((*it).group_id) ; auto it2 = search_results_map.find((*it).mGroupId) ;
if(it2 != search_results_map.end()) if(it2 != search_results_map.end())
{ {
// update existing data // update existing data
it2->second.popularity++ ; it2->second.mPopularity++ ;
it2->second.number_of_messages = std::max(it2->second.number_of_messages,(*it).number_of_messages) ; it2->second.mNumberOfMessages = std::max(it2->second.mNumberOfMessages,(*it).mNumberOfMessages) ;
} }
else else
{ {
search_results_map[(*it).group_id] = *it; search_results_map[(*it).mGroupId] = *it;
search_results_map[(*it).group_id].popularity = 1; // number of results so far search_results_map[(*it).mGroupId].mPopularity = 1; // number of results so far
} }
mObserver->receiveDistantSearchResults(req,(*it).group_id) ; mObserver->receiveDistantSearchResults(req,(*it).mGroupId) ;
} }
} }
@ -5277,12 +5278,6 @@ bool RsGxsNetService::search( const std::string& substring,
{ {
group_infos.clear(); group_infos.clear();
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
}
#ifdef RS_DEEP_SEARCH #ifdef RS_DEEP_SEARCH
std::vector<DeepSearch::SearchResult> results; std::vector<DeepSearch::SearchResult> results;
DeepSearch::search(substring, results, 0); DeepSearch::search(substring, results, 0);
@ -5290,33 +5285,48 @@ bool RsGxsNetService::search( const std::string& substring,
for(auto dsr : results) for(auto dsr : results)
{ {
RsUrl rUrl(dsr.mUrl); RsUrl rUrl(dsr.mUrl);
auto rit = rUrl.query().find("id"); const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end()) if(rit != rUrl.query().end())
{ {
RsGroupNetworkStats stats; RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second); RsGxsGroupId grpId(rit->second);
RsGxsGrpMetaTemporaryMap::iterator mIt; if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
if( !grpId.isNull() &&
(mIt = grpMetaMap.find(grpId)) != grpMetaMap.end() &&
getGroupNetworkStats(grpId, stats) )
{ {
RsGxsGrpMetaData& gMeta(*mIt->second);
RsGxsGroupSummary s; RsGxsGroupSummary s;
s.group_id = grpId;
s.group_name = gMeta.mGroupName; s.mGroupId = grpId;
s.search_context = dsr.mSnippet;
s.sign_flags = gMeta.mSignFlags; if((rit = uQ.find("name")) != uQ.end())
s.publish_ts = gMeta.mSignFlags; s.mGroupName = rit->second;
s.author_id = gMeta.mAuthorId; if((rit = uQ.find("signFlags")) != uQ.end())
s.number_of_messages = stats.mMaxVisibleCount; s.mSignFlags = std::stoul(rit->second);
s.last_message_ts = stats.mLastGroupModificationTS; if((rit = uQ.find("publishDate")) != uQ.end())
s.popularity = gMeta.mPop; {
std::istringstream ss(rit->second);
std::tm tm;
ss >> std::get_time(&tm, "%Y%m%d");
s.mPublishTs = mktime(&tm);
}
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s); group_infos.push_back(s);
} }
} }
} }
#else // RS_DEEP_SEARCH #else // RS_DEEP_SEARCH
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
}
RsGroupNetworkStats stats ; RsGroupNetworkStats stats ;
for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it) for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it)
if(termSearch(it->second->mGroupName,substring)) if(termSearch(it->second->mGroupName,substring))

18
libretroshare/src/gxs/rsgxsnettunnel.cc
View file

@ -216,22 +216,6 @@ public:
} }
}; };
template<>
void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, RsGxsGroupSummary& gs, const std::string& member_name )
{
RsTypeSerializer::serial_process (j,ctx,gs.group_id ,member_name+"-group_id") ; // RsGxsGroupId group_id ;
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ;
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_COMMENT ,gs.group_description,member_name+"-group_description") ; // std::string group_description ;
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ;
RsTypeSerializer::serial_process (j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ;
RsTypeSerializer::serial_process (j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ;
RsTypeSerializer::serial_process (j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ;
RsTypeSerializer::serial_process<time_t> (j,ctx,gs.last_message_ts ,member_name+"-last_message_ts") ; // time_t last_message_ts ;
RsTypeSerializer::serial_process<uint32_t>(j,ctx,gs.sign_flags ,member_name+"-sign_flags") ; // uint32_t sign_flags ;
RsTypeSerializer::serial_process<uint32_t>(j,ctx,gs.popularity ,member_name+"-popularity") ; // uint32_t popularity ;
}
//===========================================================================================================================================// //===========================================================================================================================================//
// Interface with rest of the software // // Interface with rest of the software //
//===========================================================================================================================================// //===========================================================================================================================================//
@ -1102,7 +1086,7 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
auto it = mSearchableServices.find(result_gs->service) ; auto it = mSearchableServices.find(result_gs->service) ;

48
libretroshare/src/retroshare/rsgxsiface.h
View file

@ -34,26 +34,42 @@
#include "util/rsdeprecate.h" #include "util/rsdeprecate.h"
/*! /*!
* \brief The RsGxsGroupSymmary struct * This structure is used to transport group summary information when a GXS
* This structure is used to transport group summary information when a GXS service is searched. It contains the group information * service is searched. It contains the group information as well as a context
* as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make * string to tell where the information was found. It is more compact than a
* search responses as light as possible. * GroupMeta object, so as to make search responses as light as possible.
*/ */
struct RsGxsGroupSummary struct RsGxsGroupSummary : RsSerializable
{ {
RsGxsGroupSummary() : publish_ts(0), number_of_messages(0),last_message_ts(0),sign_flags(0),popularity(0) {} RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
mSignFlags(0),mPopularity(0) {}
RsGxsGroupId group_id ; RsGxsGroupId mGroupId;
std::string mGroupName;
RsGxsId mAuthorId;
time_t mPublishTs;
uint32_t mNumberOfMessages;
time_t mLastMessageTs;
uint32_t mSignFlags;
uint32_t mPopularity;
std::string group_name ; std::string mSearchContext;
RS_DEPRECATED std::string group_description;
std::string search_context ; /// @see RsSerializable::serial_process
RsGxsId author_id ; void serial_process( RsGenericSerializer::SerializeJob j,
time_t publish_ts ; RsGenericSerializer::SerializeContext& ctx )
uint32_t number_of_messages ; {
time_t last_message_ts ; RS_SERIAL_PROCESS(mGroupId);
uint32_t sign_flags ; RS_SERIAL_PROCESS(mGroupName);
uint32_t popularity ; RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mNumberOfMessages);
RS_SERIAL_PROCESS(mLastMessageTs);
RS_SERIAL_PROCESS(mSignFlags);
RS_SERIAL_PROCESS(mPopularity);
RS_SERIAL_PROCESS(mSearchContext);
}
}; };

21
libretroshare/src/services/p3gxschannels.cc
View file

@ -1703,21 +1703,18 @@ bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<Rs
bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
{ {
RsGxsGroupSummary gs ; RsGxsGroupSummary gs;
if(netService()->retrieveDistantGroupSummary(group_id,gs)) if(netService()->retrieveDistantGroupSummary(group_id,gs))
{ {
// This is a placeholder information by the time we receive the full group meta data. // This is a placeholder information by the time we receive the full group meta data.
distant_group.mMeta.mGroupId = gs.mGroupId ;
distant_group.mDescription = gs.group_description; distant_group.mMeta.mGroupName = gs.mGroupName;
distant_group.mMeta.mGroupId = gs.group_id ;
distant_group.mMeta.mGroupName = gs.group_name;
distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ; distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ;
distant_group.mMeta.mSignFlags = gs.sign_flags; distant_group.mMeta.mSignFlags = gs.mSignFlags;
distant_group.mMeta.mPublishTs = gs.publish_ts; distant_group.mMeta.mPublishTs = gs.mPublishTs;
distant_group.mMeta.mAuthorId = gs.author_id; distant_group.mMeta.mAuthorId = gs.mAuthorId;
distant_group.mMeta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able. distant_group.mMeta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able.
@ -1726,9 +1723,9 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann
distant_group.mMeta.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ; distant_group.mMeta.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ;
distant_group.mMeta.mPop = gs.popularity; // Popularity = number of friend subscribers distant_group.mMeta.mPop = gs.mPopularity; // Popularity = number of friend subscribers
distant_group.mMeta.mVisibleMsgCount = gs.number_of_messages; // Max messages reported by friends distant_group.mMeta.mVisibleMsgCount = gs.mNumberOfMessages; // Max messages reported by friends
distant_group.mMeta.mLastPost = gs.last_message_ts; // Timestamp for last message. Not used yet. distant_group.mMeta.mLastPost = gs.mLastMessageTs; // Timestamp for last message. Not used yet.
return true ; return true ;
} }

19
retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp
View file

@ -282,18 +282,19 @@ void GxsGroupFrameDialog::updateSearchResults()
for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) for(auto it3(group_infos.begin());it3!=group_infos.end();++it3)
if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end()) if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end())
{ {
std::cerr << " adding new group " << it3->first << " " << it3->second.group_id << " \"" << it3->second.group_name << "\"" << std::endl; std::cerr << " adding new group " << it3->first << " "
<< it3->second.mGroupId << " \""
<< it3->second.mGroupName << "\"" << std::endl;
GroupItemInfo i ; GroupItemInfo i;
i.id = QString(it3->second.group_id.toStdString().c_str()) ; i.id = QString(it3->second.mGroupId.toStdString().c_str());
i.name = QString::fromUtf8(it3->second.group_name.c_str()) ; i.name = QString::fromUtf8(it3->second.mGroupName.c_str());
i.description = QString::fromUtf8(it3->second.group_description.c_str()) ;
i.popularity = 0; // could be set to the number of hits i.popularity = 0; // could be set to the number of hits
i.lastpost = QDateTime::fromTime_t(it3->second.last_message_ts); i.lastpost = QDateTime::fromTime_t(it3->second.mLastMessageTs);
i.subscribeFlags = 0; // irrelevant here i.subscribeFlags = 0; // irrelevant here
i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags) ; i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags);
i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags) ; i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags);
i.max_visible_posts = it3->second.number_of_messages ; i.max_visible_posts = it3->second.mNumberOfMessages;
group_items.push_back(i); group_items.push_back(i);
} }