Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-01-13 00:19:30 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Epurate this branch from channels deep search changes

Browse Source
This commit is contained in:
Gioacchino Mazzurco 2021-02-25 11:18:26 +01:00
parent 9c38eed648
commit dcb2bee8cc
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
6 changed files with 218 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
libretroshare/src/deep_search/channelsindex.cpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -20,23 +20,16 @@
#include "deep_search/channelsindex.hpp" #include "deep_search/channelsindex.hpp"
#include "deep_search/commonutils.hpp" #include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "util/rsdebuglevel3.h"
/*static*/ std::string DeepChannelsIndex::dbDefaultPath() uint32_t DeepChannelsIndex::search(
{ return RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; }
std::error_condition DeepChannelsIndex::search(
const std::string& queryStr, const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults ) std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults )
{ {
RS_DBG3(queryStr);
results.clear(); results.clear();
std::unique_ptr<Xapian::Database> dbPtr( std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(mDbPath) ); DeepSearch::openReadOnlyDatabase(dbPath()) );
if(!dbPtr) return std::errc::bad_file_descriptor; if(!dbPtr) return 0;
Xapian::Database& db(*dbPtr); Xapian::Database& db(*dbPtr);
@ -70,13 +63,17 @@ std::error_condition DeepChannelsIndex::search(
results.push_back(s); results.push_back(s);
} }
return std::error_condition(); return static_cast<uint32_t>(results.size());
} }
std::error_condition DeepChannelsIndex::indexChannelGroup( void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
const RsGxsChannelGroup& chan )
{ {
RS_DBG4(chan); std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing. // Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator; Xapian::TermGenerator termgenerator;
@ -97,8 +94,21 @@ std::error_condition DeepChannelsIndex::indexChannelGroup(
termgenerator.increase_termpos(); termgenerator.increase_termpos();
termgenerator.index_text(chan.mDescription); termgenerator.index_text(chan.mDescription);
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", chan.mMeta.mGroupId.toStdString());
const std::string idTerm("Q" + chanUrl.toString());
chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs));
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search // store the RS link so we are able to retrive it on matching search
const std::string rsLink(channelIndexId(chan.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink); doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes. // Store some fields for display purposes.
@ -107,32 +117,35 @@ std::error_condition DeepChannelsIndex::indexChannelGroup(
// We use the identifier to ensure each object ends up in the // We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the // database only once no matter how many times we run the
// indexer. "Q" prefix is a Xapian convention for unique id term. // indexer. "Q" prefix is a Xapian convention for unique id term.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm); doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
} }
std::error_condition DeepChannelsIndex::removeChannelFromIndex( void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId)
const RsGxsGroupId& grpId )
{ {
RS_DBG3(grpId);
// "Q" prefix is a Xapian convention for unique id term. // "Q" prefix is a Xapian convention for unique id term.
const std::string idTerm("Q" + channelIndexId(grpId)); RsUrl chanUrl; chanUrl
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) .setScheme("retroshare").setPath("/channel")
{ db.delete_document(idTerm); } ); .setQueryKV("id", grpId.toStdString());
std::string idTerm("Q" + chanUrl.toString());
return std::error_condition(); std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
} }
std::error_condition DeepChannelsIndex::indexChannelPost( void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
const RsGxsChannelPost& post )
{ {
RS_DBG4(post); std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing. // Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator; Xapian::TermGenerator termgenerator;
@ -147,16 +160,21 @@ std::error_condition DeepChannelsIndex::indexChannelPost(
termgenerator.index_text( termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// Avoid indexing RetroShare-gui HTML tags // TODO: we should strip out HTML tags instead of skipping indexing
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg); // Avoid indexing HTML
termgenerator.index_text( bool isPlainMsg =
DeepSearch::simpleTextHtmlExtract(post.mMsg), 1, "XD" ); post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>';
if(isPlainMsg)
termgenerator.index_text(post.mMsg, 1, "XD");
// Index fields without prefixes for general search. // Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName); termgenerator.index_text(post.mMeta.mMsgName);
if(isPlainMsg)
{
termgenerator.increase_termpos(); termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg); termgenerator.index_text(post.mMsg);
}
for(const RsGxsFile& attachment : post.mFiles) for(const RsGxsFile& attachment : post.mFiles)
{ {
@ -166,50 +184,47 @@ std::error_condition DeepChannelsIndex::indexChannelPost(
termgenerator.index_text(attachment.mName); termgenerator.index_text(attachment.mName);
} }
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(post.mMeta.mGroupId, post.mMeta.mMsgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the // We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the // database only once no matter how many times we run the
// indexer. // indexer.
const std::string idTerm("Q" + rsLink); RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", post.mMeta.mGroupId.toStdString())
.setQueryKV("msgid", post.mMeta.mMsgId.toStdString());
std::string idTerm("Q" + postUrl.toString());
postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs));
postUrl.setQueryKV("name", post.mMeta.mMsgName);
postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString());
std::string rsLink(postUrl.toString());
// store the RS link so we are able to retrive it on matching search
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
if(isPlainMsg)
doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg);
else doc.set_data(post.mMeta.mMsgName);
doc.add_boolean_term(idTerm); doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
} }
std::error_condition DeepChannelsIndex::removeChannelPostFromIndex( void DeepChannelsIndex::removeChannelPostFromIndex(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RS_DBG3(grpId, msgId);
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepChannelsIndex::channelIndexId(RsGxsGroupId grpId)
{
RsUrl channelIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL);
channelIndexId.setQueryKV(
RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString() );
return channelIndexId.toString();
}
/*static*/ std::string DeepChannelsIndex::postIndexId(
RsGxsGroupId grpId, RsGxsMessageId msgId ) RsGxsGroupId grpId, RsGxsMessageId msgId )
{ {
RsUrl postIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL); RsUrl postUrl; postUrl
postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString()); .setScheme("retroshare").setPath("/channel")
postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_MSG_ID_FIELD, msgId.toStdString()); .setQueryKV("id", grpId.toStdString())
return postIndexId.toString(); .setQueryKV("msgid", msgId.toStdString());
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
} }

35
libretroshare/src/deep_search/channelsindex.hpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -24,8 +24,8 @@
#include "util/rstime.h" #include "util/rstime.h"
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "retroshare/rsinit.h"
#include "util/rsurl.h" #include "util/rsurl.h"
#include "deep_search/commonutils.hpp"
struct DeepChannelsSearchResult struct DeepChannelsSearchResult
{ {
@ -36,34 +36,28 @@ struct DeepChannelsSearchResult
struct DeepChannelsIndex struct DeepChannelsIndex
{ {
explicit DeepChannelsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/** /**
* @brief Search indexed GXS groups and messages * @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for * @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits * no limits
* @return search results count * @return search results count
*/ */
std::error_condition search( static uint32_t search( const std::string& queryStr,
const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results, std::vector<DeepChannelsSearchResult>& results,
uint32_t maxResults = 100 ); uint32_t maxResults = 100 );
std::error_condition indexChannelGroup(const RsGxsChannelGroup& chan); static void indexChannelGroup(const RsGxsChannelGroup& chan);
std::error_condition removeChannelFromIndex(const RsGxsGroupId& grpId); static void removeChannelFromIndex(RsGxsGroupId grpId);
std::error_condition indexChannelPost(const RsGxsChannelPost& post); static void indexChannelPost(const RsGxsChannelPost& post);
std::error_condition removeChannelPostFromIndex( static void removeChannelPostFromIndex(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ); RsGxsGroupId grpId, RsGxsMessageId msgId );
static std::string dbDefaultPath(); static uint32_t indexFile(const std::string& path);
private: private:
static std::string channelIndexId(RsGxsGroupId grpId);
static std::string postIndexId(RsGxsGroupId grpId, RsGxsMessageId msgId);
enum : Xapian::valueno enum : Xapian::valueno
{ {
@ -74,7 +68,10 @@ private:
BAD_VALUENO = Xapian::BAD_VALUENO BAD_VALUENO = Xapian::BAD_VALUENO
}; };
const std::string mDbPath; static const std::string& dbPath()
{
DeepSearch::StubbornWriteOpQueue mWriteQueue; static const std::string dbDir =
RsAccounts::AccountDirectory() + "/deep_channels_xapian_db";
return dbDir;
}
}; };

125
libretroshare/src/retroshare/rsgxschannels.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -46,11 +46,6 @@ class RsGxsChannels;
extern RsGxsChannels* rsGxsChannels; extern RsGxsChannels* rsGxsChannels;
/* TODO: At this abstraction level that turtle is used for distant searches
* should be an hidden implementation detail. As of today a bit of those
* implementation details leaks around because we use TurtleRequestId in this
* interface */
struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData
{ {
RsGxsChannelGroup() : mAutoDownload(false) {} RsGxsChannelGroup() : mAutoDownload(false) {}
@ -113,33 +108,24 @@ struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData
enum class RsChannelEventCode: uint8_t enum class RsChannelEventCode: uint8_t
{ {
UNKNOWN = 0x00, UNKNOWN = 0x00,
NEW_CHANNEL = 0x01, /// emitted when new channel is received NEW_CHANNEL = 0x01, // emitted when new channel is received
UPDATED_CHANNEL = 0x02, /// emitted when existing channel is updated UPDATED_CHANNEL = 0x02, // emitted when existing channel is updated
NEW_MESSAGE = 0x03, /// new message reeived in a particular channel (group and msg id) NEW_MESSAGE = 0x03, // new message reeived in a particular channel (group and msg id)
UPDATED_MESSAGE = 0x04, /// existing message has been updated in a particular channel UPDATED_MESSAGE = 0x04, // existing message has been updated in a particular channel
RECEIVED_PUBLISH_KEY = 0x05, /// publish key for this channel has been received RECEIVED_PUBLISH_KEY = 0x05, // publish key for this channel has been received
SUBSCRIBE_STATUS_CHANGED = 0x06, /// subscription for channel mChannelGroupId changed. SUBSCRIBE_STATUS_CHANGED = 0x06, // subscription for channel mChannelGroupId changed.
READ_STATUS_CHANGED = 0x07, /// existing message has been read or set to unread READ_STATUS_CHANGED = 0x07, // existing message has been read or set to unread
RECEIVED_DISTANT_SEARCH_RESULT = 0x08, // result for the given group id available for the given turtle request id
/** Result for the given group id available for the given turtle request id STATISTICS_CHANGED = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed
* @deprecated kept for retrocompatibility with old search system new code SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed
* should use @see DISTANT_SEARCH_RESULT instead */ NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message
RECEIVED_TURTLE_SEARCH_RESULT = 0x08, NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system
STATISTICS_CHANGED = 0x09, /// stats (nb of supplier friends, how many msgs they have etc) has changed
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
NEW_COMMENT = 0x0b, /// new comment arrived/published. mChannelThreadId gives the ID of the commented message
NEW_VOTE = 0x0c, /// new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, /// channel was deleted by auto-cleaning system
DELETED_POST = 0x0e, /// Post deleted (usually by cleaning)
DISTANT_SEARCH_RESULT = 0x0f /// Distant search result received
}; };
struct RsGxsChannelEvent: RsEvent struct RsGxsChannelEvent: RsEvent
{ {
RsGxsChannelEvent(): RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
RsChannelEventCode mChannelEventCode; RsChannelEventCode mChannelEventCode;
RsGxsGroupId mChannelGroupId; RsGxsGroupId mChannelGroupId;
@ -157,14 +143,13 @@ struct RsGxsChannelEvent: RsEvent
} }
}; };
/** This event is used to factor multiple search results notifications in a // This event is used to factor multiple search results notifications in a single event.
* single event.*/
struct RS_DEPRECATED_FOR(RsGxsChannelDistantSearchResultEvent) struct RsGxsChannelSearchResultEvent: RsEvent
RsGxsChannelSearchResultEvent: RsEvent
{ {
RsGxsChannelSearchResultEvent(): RsGxsChannelSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS), RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::RECEIVED_TURTLE_SEARCH_RESULT) {} mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode; RsChannelEventCode mChannelEventCode;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap; std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap;
@ -179,29 +164,6 @@ RsGxsChannelSearchResultEvent: RsEvent
} }
}; };
/** This event is fired once distant search results are received */
struct RsGxsChannelDistantSearchResultEvent: RsEvent
{
RsGxsChannelDistantSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::DISTANT_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mChannelEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
{ {
public: public:
@ -424,14 +386,10 @@ public:
* @brief Get channel content summaries * @brief Get channel content summaries
* @jsonapi{development} * @jsonapi{development}
* @param[in] channelId id of the channel of which the content is requested * @param[in] channelId id of the channel of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries * @param[out] summaries storage for summaries
* @return success or error details if something failed * @return false if something failed, true otherwhise
*/ */
virtual std::error_condition getContentSummaries( virtual bool getContentSummaries( const RsGxsGroupId& channelId,
const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0; std::vector<RsMsgMetaData>& summaries ) = 0;
/** /**
@ -465,22 +423,21 @@ public:
bool subscribe ) = 0; bool subscribe ) = 0;
/** /**
* @brief Retrieve statistics about the channel service * \brief Retrieve statistics about the channel service
* @jsonapi{development} * @jsonapi{development}
* @param[out] stat storage for statistics * \param[out] stat Statistics structure
* @return true on success false otherwise * \return
*/ */
virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0;
/** /**
* @brief Retrieve statistics about the given channel * \brief Retrieve statistics about the given channel
* @jsonapi{development} * @jsonapi{development}
* @param[in] channelId Id of the channel group * \param[in] channelId Id of the channel group
* @param[out] stat storage for statistics * \param[out] stat Statistics structure
* @return true on success false otherwise * \return
*/ */
virtual bool getChannelStatistics( virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0;
const RsGxsGroupId& channelId, GxsGroupStatistic& stat ) =0;
/// default base URL used for channels links @see exportChannelLink /// default base URL used for channels links @see exportChannelLink
static const std::string DEFAULT_CHANNEL_BASE_URL; static const std::string DEFAULT_CHANNEL_BASE_URL;
@ -539,18 +496,14 @@ public:
std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
/** /**
* @brief Search the whole reachable network for matching channels and * @brief Search the turtle reachable network for matching channels
* contents
* @jsonapi{development} * @jsonapi{development}
* An @see RsGxsChannelSearchResultEvent is emitted when matching results * An @see RsGxsChannelSearchResultEvent is emitted when matching channels
* arrives from the network * arrives from the network
* @param[in] matchString string to search into the channels * @param[in] matchString string to search into the channels
* @param[out] searchId storage for search id, useful to track search events * @return search id
* and retrieve search results
* @return success or error details
*/ */
virtual std::error_condition distantSearchRequest( virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0;
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/** /**
* @brief Retrieve available search results * @brief Retrieve available search results
@ -580,16 +533,14 @@ public:
* @param[out] distantGroup storage for group data * @param[out] distantGroup storage for group data
* @return false on error, true otherwise * @return false on error, true otherwise
*/ */
virtual bool getDistantSearchResultGroupData( virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
/** /**
* @brief Get the status of ongoing search * @brief getDistantSearchStatus
* @return unknown (probably not even searched), known as a search result, * Returns the status of ongoing search: unknown (probably not even searched), known as a search result,
* data request ongoing and data available * data request ongoing and data available
*/ */
virtual DistantSearchGroupStatus getDistantSearchStatus( virtual DistantSearchGroupStatus getDistantSearchStatus(const RsGxsGroupId& group_id) =0;
const RsGxsGroupId& group_id ) =0;
/** /**
* @brief Clear accumulated search results * @brief Clear accumulated search results

65
libretroshare/src/rsitems/rsgxschannelitems.h
View File

@ -3,9 +3,7 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> * * Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -21,7 +19,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* * * *
*******************************************************************************/ *******************************************************************************/
#pragma once #ifndef RS_GXS_CHANNEL_ITEMS_H
#define RS_GXS_CHANNEL_ITEMS_H
#include <map> #include <map>
@ -31,23 +30,14 @@
#include "serialiser/rstlvfileitem.h" #include "serialiser/rstlvfileitem.h"
#include "serialiser/rstlvimage.h" #include "serialiser/rstlvimage.h"
#include "serialiser/rsserializable.h"
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "serialiser/rsserializer.h" #include "serialiser/rsserializer.h"
#include "util/rsdir.h" #include "util/rsdir.h"
enum class RsGxsChannelItems : uint8_t
{
GROUP_ITEM = 0x02,
POST_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02; const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03; const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03;
class RsGxsChannelGroupItem : public RsGxsGrpItem class RsGxsChannelGroupItem : public RsGxsGrpItem
@ -89,47 +79,6 @@ public:
RsTlvImage mThumbnail; RsTlvImage mThumbnail;
}; };
struct RsGxsChannelsSearchRequest : RsSerializable
{
RsGxsChannelsSearchRequest() : mType(RsGxsChannelItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsChannelsSearchRequest() override = default;
};
struct RsGxsChannelsSearchReply : RsSerializable
{
RsGxsChannelsSearchReply() : mType(RsGxsChannelItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsChannelsSearchReply() override = default;
};
class RsGxsChannelSerialiser : public RsGxsCommentSerialiser class RsGxsChannelSerialiser : public RsGxsCommentSerialiser
{ {
@ -140,3 +89,5 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ; virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
}; };
#endif /* RS_GXS_CHANNEL_ITEMS_H */

74
libretroshare/src/services/p3gxschannels.h
View File

@ -4,8 +4,7 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -23,8 +22,6 @@
*******************************************************************************/ *******************************************************************************/
#pragma once #pragma once
#include <map>
#include <string>
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "services/p3gxscommon.h" #include "services/p3gxscommon.h"
@ -33,7 +30,9 @@
#include "util/rsmemory.h" #include "util/rsmemory.h"
#include "util/rsdebug.h" #include "util/rsdebug.h"
#include "util/rstickevent.h" #include "util/rstickevent.h"
#include "deep_search/channelsindex.hpp"
#include <map>
#include <string>
// This class is only a helper to parse the channel group service string. // This class is only a helper to parse the channel group service string.
@ -57,11 +56,6 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels,
public: public:
p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes, p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes,
RsGixs* gixs ); RsGixs* gixs );
/// @see RsGxsChannels
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
virtual RsServiceInfo getServiceInfo() override; virtual RsServiceInfo getServiceInfo() override;
virtual void service_tick() override; virtual void service_tick() override;
@ -75,7 +69,7 @@ protected:
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&) virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override; virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) override; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) override;
virtual bool clearDistantSearchResults(TurtleRequestId req) override; virtual bool clearDistantSearchResults(TurtleRequestId req) override;
virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override; virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override;
@ -118,6 +112,24 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled)
virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override; virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override;
virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override; virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override;
#ifdef TO_REMOVE
/// @see RsGxsChannels::turtleSearchRequest
virtual bool turtleSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::turtleChannelRequest
virtual bool turtleChannelRequest(
const RsGxsGroupId& channelId,
const std::function<void (const RsGxsChannelGroup& result)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::localSearchRequest
virtual bool localSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
rstime_t maxWait = 30 ) override;
#endif
/** /**
* Receive results from turtle search @see RsGenExchange @see RsNxsObserver * Receive results from turtle search @see RsGenExchange @see RsNxsObserver
* @see RsGxsNetService::receiveTurtleSearchResults * @see RsGxsNetService::receiveTurtleSearchResults
@ -203,10 +215,9 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
const std::set<RsGxsMessageId> &contentIds, const std::set<RsGxsMessageId> &contentIds,
std::vector<RsGxsComment> &comments) override; std::vector<RsGxsComment> &comments) override;
/// @see RsGxsChannels /// Implementation of @see RsGxsChannels::getContentSummaries
std::error_condition getContentSummaries( bool getContentSummaries(
const RsGxsGroupId& channelId, const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override; std::vector<RsMsgMetaData>& summaries ) override;
/// Implementation of @see RsGxsChannels::getChannelStatistics /// Implementation of @see RsGxsChannels::getChannelStatistics
@ -286,17 +297,6 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
virtual bool shareChannelKeys( virtual bool shareChannelKeys(
const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) override; const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) override;
#ifdef RS_DEEP_CHANNEL_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// Implementation of @see RsGxsChannels::createChannel /// Implementation of @see RsGxsChannels::createChannel
RS_DEPRECATED_FOR(createChannelV2) RS_DEPRECATED_FOR(createChannelV2)
bool createChannel(RsGxsChannelGroup& channel) override; bool createChannel(RsGxsChannelGroup& channel) override;
@ -313,6 +313,7 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
RS_DEPRECATED_FOR(createVoteV2) RS_DEPRECATED_FOR(createVoteV2)
bool createVote(RsGxsVote& vote) override; bool createVote(RsGxsVote& vote) override;
protected: protected:
// Overloaded from GxsTokenQueue for Request callbacks. // Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type virtual void handleResponse(uint32_t token, uint32_t req_type
@ -328,6 +329,7 @@ static uint32_t channelsAuthenPolicy();
void request_SpecificSubscribedGroups(const std::list<RsGxsGroupId> &groups); void request_SpecificSubscribedGroups(const std::list<RsGxsGroupId> &groups);
void load_SubscribedGroups(const uint32_t &token); void load_SubscribedGroups(const uint32_t &token);
void request_SpecificUnprocessedPosts(std::list<std::pair<RsGxsGroupId, RsGxsMessageId> > &ids);
void request_GroupUnprocessedPosts(const std::list<RsGxsGroupId> &grouplist); void request_GroupUnprocessedPosts(const std::list<RsGxsGroupId> &grouplist);
void load_unprocessedPosts(uint32_t token); void load_unprocessedPosts(uint32_t token);
@ -388,8 +390,26 @@ bool generateGroup(uint32_t &token, std::string groupName);
rstime_t mLastDistantSearchNotificationTS; rstime_t mLastDistantSearchNotificationTS;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify; std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
#ifdef TO_REMOVE
/** Store search callbacks with timeout*/
std::map<
TurtleRequestId,
std::pair<
std::function<void (const RsGxsGroupSummary&)>,
std::chrono::system_clock::time_point >
> mSearchCallbacksMap;
RsMutex mSearchCallbacksMapMutex;
#ifdef RS_DEEP_CHANNEL_INDEX /** Store distant channels requests callbacks with timeout*/
DeepChannelsIndex mDeepIndex; std::map<
TurtleRequestId,
std::pair<
std::function<void (const RsGxsChannelGroup&)>,
std::chrono::system_clock::time_point >
> mDistantChannelsCallbacksMap;
RsMutex mDistantChannelsCallbacksMapMutex;
/// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap
void cleanTimedOutCallbacks();
#endif #endif
}; };

4
retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp
View File

@ -411,9 +411,7 @@ void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id)
TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string)
{ {
TurtleRequestId searchId; return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ;
rsGxsChannels->distantSearchRequest(search_string.toStdString(), searchId);
return searchId;
} }
bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos) bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)