Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

First working prototype of GXS service search

Browse Source 
Channels are now able to take advantage of the new deep search
  goodies

Rebase on top of master 2021/10/19
This commit is contained in:
Gioacchino Mazzurco 2021-02-08 17:04:04 +01:00
parent ab349a8157
commit 1b551d809f
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
30 changed files with 1455 additions and 874 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
libretroshare/src/deep_search/channelsindex.cpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -20,16 +20,23 @@
#include "deep_search/channelsindex.hpp" #include "deep_search/channelsindex.hpp"
#include "deep_search/commonutils.hpp" #include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "util/rsdebuglevel3.h"
uint32_t DeepChannelsIndex::search( /*static*/ std::string DeepChannelsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; }
std::error_condition DeepChannelsIndex::search(
const std::string& queryStr, const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults ) std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults )
{ {
RS_DBG3(queryStr);
results.clear(); results.clear();
std::unique_ptr<Xapian::Database> dbPtr( std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(dbPath()) ); DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return 0; if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr); Xapian::Database& db(*dbPtr);
@ -63,17 +70,13 @@ uint32_t DeepChannelsIndex::search(
results.push_back(s); results.push_back(s);
} }
return static_cast<uint32_t>(results.size()); return std::error_condition();
} }
void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) std::error_condition DeepChannelsIndex::indexChannelGroup(
const RsGxsChannelGroup& chan )
{ {
std::unique_ptr<Xapian::WritableDatabase> dbPtr( RS_DBG4(chan);
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing. // Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator; Xapian::TermGenerator termgenerator;
@ -94,21 +97,8 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
termgenerator.increase_termpos(); termgenerator.increase_termpos();
termgenerator.index_text(chan.mDescription); termgenerator.index_text(chan.mDescription);
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", chan.mMeta.mGroupId.toStdString());
const std::string idTerm("Q" + chanUrl.toString());
chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs));
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search // store the RS link so we are able to retrive it on matching search
const std::string rsLink(channelIndexId(chan.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink); doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes. // Store some fields for display purposes.
@ -117,35 +107,32 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
// We use the identifier to ensure each object ends up in the // We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the // database only once no matter how many times we run the
// indexer. "Q" prefix is a Xapian convention for unique id term. // indexer. "Q" prefix is a Xapian convention for unique id term.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm); doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
} }
void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId) std::error_condition DeepChannelsIndex::removeChannelFromIndex(
const RsGxsGroupId& grpId )
{ {
RS_DBG3(grpId);
// "Q" prefix is a Xapian convention for unique id term. // "Q" prefix is a Xapian convention for unique id term.
RsUrl chanUrl; chanUrl const std::string idTerm("Q" + channelIndexId(grpId));
.setScheme("retroshare").setPath("/channel") mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
.setQueryKV("id", grpId.toStdString()); { db.delete_document(idTerm); } );
std::string idTerm("Q" + chanUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr( return std::error_condition();
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
} }
void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) std::error_condition DeepChannelsIndex::indexChannelPost(
const RsGxsChannelPost& post )
{ {
std::unique_ptr<Xapian::WritableDatabase> dbPtr( RS_DBG4(post);
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing. // Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator; Xapian::TermGenerator termgenerator;
@ -160,21 +147,16 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
termgenerator.index_text( termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// TODO: we should strip out HTML tags instead of skipping indexing // Avoid indexing RetroShare-gui HTML tags
// Avoid indexing HTML const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
bool isPlainMsg = termgenerator.index_text(
post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>'; DeepSearch::simpleTextHtmlExtract(post.mMsg), 1, "XD" );
if(isPlainMsg)
termgenerator.index_text(post.mMsg, 1, "XD");
// Index fields without prefixes for general search. // Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName); termgenerator.index_text(post.mMeta.mMsgName);
if(isPlainMsg)
{ termgenerator.increase_termpos();
termgenerator.increase_termpos(); termgenerator.index_text(cleanMsg);
termgenerator.index_text(post.mMsg);
}
for(const RsGxsFile& attachment : post.mFiles) for(const RsGxsFile& attachment : post.mFiles)
{ {
@ -184,47 +166,50 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
termgenerator.index_text(attachment.mName); termgenerator.index_text(attachment.mName);
} }
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", post.mMeta.mGroupId.toStdString())
.setQueryKV("msgid", post.mMeta.mMsgId.toStdString());
std::string idTerm("Q" + postUrl.toString());
postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs));
postUrl.setQueryKV("name", post.mMeta.mMsgName);
postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString());
std::string rsLink(postUrl.toString());
// store the RS link so we are able to retrive it on matching search // store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(post.mMeta.mGroupId, post.mMeta.mMsgId));
doc.add_value(URL_VALUENO, rsLink); doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes. // Store some fields for display purposes.
if(isPlainMsg) doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg);
else doc.set_data(post.mMeta.mMsgName);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm); doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
} }
void DeepChannelsIndex::removeChannelPostFromIndex( std::error_condition DeepChannelsIndex::removeChannelPostFromIndex(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RS_DBG3(grpId, msgId);
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepChannelsIndex::channelIndexId(RsGxsGroupId grpId)
{
RsUrl channelIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL);
channelIndexId.setQueryKV(
RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString() );
return channelIndexId.toString();
}
/*static*/ std::string DeepChannelsIndex::postIndexId(
RsGxsGroupId grpId, RsGxsMessageId msgId ) RsGxsGroupId grpId, RsGxsMessageId msgId )
{ {
RsUrl postUrl; postUrl RsUrl postIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL);
.setScheme("retroshare").setPath("/channel") postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString());
.setQueryKV("id", grpId.toStdString()) postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_MSG_ID_FIELD, msgId.toStdString());
.setQueryKV("msgid", msgId.toStdString()); return postIndexId.toString();
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
} }

39
libretroshare/src/deep_search/channelsindex.hpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -24,8 +24,8 @@
#include "util/rstime.h" #include "util/rstime.h"
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "retroshare/rsinit.h"
#include "util/rsurl.h" #include "util/rsurl.h"
#include "deep_search/commonutils.hpp"
struct DeepChannelsSearchResult struct DeepChannelsSearchResult
{ {
@ -36,28 +36,34 @@ struct DeepChannelsSearchResult
struct DeepChannelsIndex struct DeepChannelsIndex
{ {
explicit DeepChannelsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/** /**
* @brief Search indexed GXS groups and messages * @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for * @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits * no limits
* @return search results count * @return search results count
*/ */
static uint32_t search( const std::string& queryStr, std::error_condition search(
std::vector<DeepChannelsSearchResult>& results, const std::string& queryStr,
uint32_t maxResults = 100 ); std::vector<DeepChannelsSearchResult>& results,
uint32_t maxResults = 100 );
static void indexChannelGroup(const RsGxsChannelGroup& chan); std::error_condition indexChannelGroup(const RsGxsChannelGroup& chan);
static void removeChannelFromIndex(RsGxsGroupId grpId); std::error_condition removeChannelFromIndex(const RsGxsGroupId& grpId);
static void indexChannelPost(const RsGxsChannelPost& post); std::error_condition indexChannelPost(const RsGxsChannelPost& post);
static void removeChannelPostFromIndex( std::error_condition removeChannelPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId ); const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
static uint32_t indexFile(const std::string& path); static std::string dbDefaultPath();
private: private:
static std::string channelIndexId(RsGxsGroupId grpId);
static std::string postIndexId(RsGxsGroupId grpId, RsGxsMessageId msgId);
enum : Xapian::valueno enum : Xapian::valueno
{ {
@ -68,10 +74,7 @@ private:
BAD_VALUENO = Xapian::BAD_VALUENO BAD_VALUENO = Xapian::BAD_VALUENO
}; };
static const std::string& dbPath() const std::string mDbPath;
{
static const std::string dbDir = DeepSearch::StubbornWriteOpQueue mWriteQueue;
RsAccounts::AccountDirectory() + "/deep_channels_xapian_db";
return dbDir;
}
}; };

154
libretroshare/src/deep_search/commonutils.cpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -18,39 +18,17 @@
* * * *
*******************************************************************************/ *******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp" #include "deep_search/commonutils.hpp"
#include "util/stacktrace.h" #include "util/stacktrace.h"
#include "util/rsdebug.h" #include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
namespace DeepSearch namespace DeepSearch
{ {
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase( std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags ) const std::string& path, int flags )
{ {
@ -60,12 +38,12 @@ std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
new Xapian::Database(path, flags) ); new Xapian::Database(path, flags) );
return dbPtr; return dbPtr;
} }
catch(Xapian::DatabaseOpeningError e) catch(Xapian::DatabaseOpeningError& e)
{ {
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg() RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl; << ", probably nothing has been indexed yet." << std::endl;
} }
catch(Xapian::DatabaseLockError) catch(Xapian::DatabaseLockError&)
{ {
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock " RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl; << path << std::endl;
@ -90,4 +68,116 @@ std::string timetToXapianDate(const rstime_t& time)
return date; return date;
} }
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(50l, maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
std::string::size_type oPos;
std::string::size_type cPos;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
}
return retVal;
}
} }

34
libretroshare/src/deep_search/commonutils.hpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h> #include <xapian.h>
#include <memory> #include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h" #include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch namespace DeepSearch
{ {
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
std::unique_ptr<Xapian::Database> openReadOnlyDatabase( std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 ); const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time); std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
} }

83
libretroshare/src/deep_search/filesindex.cpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -18,47 +18,47 @@
* * * *
*******************************************************************************/ *******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility> #include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType> /*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {}; DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile( std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name, const std::string& path, const std::string& name,
const RsFileHash& hash ) const RsFileHash& hash )
{ {
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString(); const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString); const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc; auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
Xapian::PostingIterator pIt = db.postlist_begin(idTerm); if(db)
if( pIt != db.postlist_end(idTerm) )
{ {
oldDoc = db.get_document(*pIt); Xapian::Document oldDoc;
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) == Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
RS_HUMAN_READABLE_VERSION && if( pIt != db->postlist_end(idTerm) )
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{ {
/* Looks like this file has already been indexed by this RetroShare oldDoc = db->get_document(*pIt);
* exact version, so we can skip it. If the version was different it if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
* made sense to reindex it as better indexers might be available RS_HUMAN_READABLE_VERSION &&
* since last time it was indexed */ std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: " indexersRegister.size() )
<< hash << " " << name << std::endl; {
return true; /* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
} }
db.reset(); // Release DB read lock ASAP
} }
Xapian::Document doc; Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value( doc.add_value(
INDEXERS_COUNT_VALUENO, INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) ); std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true; mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
} }
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash) std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{ {
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: " RS_DBG3(hash);
<< hash << std::endl;
std::unique_ptr<Xapian::WritableDatabase> db = mWriteQueue.push([hash](Xapian::WritableDatabase& db)
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN); { db.delete_document("Q" + hash.toStdString()); });
if(!db) return false;
db->delete_document("Q" + hash.toStdString()); return std::error_condition();
return true;
} }
/*static*/ std::string DeepFilesIndex::dbDefaultPath() /*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer( /*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun ) int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{ {
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl; RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun)); indexersRegister.insert(std::make_pair(order, indexerFun));
return true; return true;
} }
uint32_t DeepFilesIndex::search( std::error_condition DeepFilesIndex::search(
const std::string& queryStr, const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults ) std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{ {
results.clear(); results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath); auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0; if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr); Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes. // Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s); results.push_back(s);
} }
return static_cast<uint32_t>(results.size()); return std::error_condition();
} }

23
libretroshare/src/deep_search/filesindex.hpp
View File

@ -1,8 +1,8 @@
/******************************************************************************* /*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian * * RetroShare full text indexing and search implementation based on Xapian *
* * * *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as * * it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/ *******************************************************************************/
#pragma once #pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string> #include <string>
#include <cstdint> #include <cstdint>
#include <vector> #include <vector>
@ -29,6 +26,9 @@
#include <map> #include <map>
#include <functional> #include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult struct DeepFilesSearchResult
{ {
DeepFilesSearchResult() : mWeight(0) {} DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex class DeepFilesIndex
{ {
public: public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {} explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/** /**
* @brief Search indexed files * @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits * no limits
* @return search results count * @return search results count
*/ */
uint32_t search( const std::string& queryStr, std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 ); uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or * @return false if file could not be indexed because of error or
* unsupported type, true otherwise. * unsupported type, true otherwise.
*/ */
bool indexFile( std::error_condition indexFile(
const std::string& path, const std::string& name, const std::string& path, const std::string& name,
const RsFileHash& hash ); const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database * @brief Remove file entry from database
* @return false on error, true otherwise. * @return false on error, true otherwise.
*/ */
bool removeFileFromIndex(const RsFileHash& hash); std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath(); static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath; const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */ /** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister; static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
}; };

6
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> * * Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -549,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH ) fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{ {
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash); ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
} }
#endif // def RS_DEEP_FILES_INDEX #endif // def RS_DEEP_FILES_INDEX

4
libretroshare/src/ft/ftserver.cc
View File

@ -5,7 +5,7 @@
* * * *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -2037,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes; std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0) if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{ {
RsFileSearchResultItem resIt; RsFileSearchResultItem resIt;

42
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -5,6 +5,7 @@
* * * *
* Copyright (C) 2012 Christopher Evi-Parker * * Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60) VALIDATE_MAX_WAITING_TIME(60)
{ {
mDataAccess = new RsGxsDataAccess(gds); mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
} }
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns) void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)

134
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2012-2012 by Christopher Evi-Parker * * Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ; return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
} }
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring) static bool termSearch(const std::string& src, const std::string& substring)
{ {
/* always ignore case */ /* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
} }
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos) bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{ {
@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ; return true ;
} }
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos ) void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{ {
std::set<RsGxsGroupId> groupsToNotifyResults; std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos) for (const RsGxsGroupSummary& gps : group_infos)
{ {
#ifndef RS_DEEP_CHANNEL_INDEX #ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are /* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure. * not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important * mDataStore may in some situations allocate an empty group meta
* to test that the group meta is both non null and actually corresponds to the group id we seek. */ * data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId); const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId); groupsToNotifyResults.insert(grpId);
@ -5332,18 +5330,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId); mObserver->receiveDistantSearchResults(req, grpId);
} }
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len) void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{ {
#ifdef NXS_NET_DEBUG_8 #ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl; GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif #endif
auto it = mSearchRequests.find(req); auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
if(mSearchRequests.end() == it) {
{ RS_WARN("Received search results for unknown request: ", req);
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ; return;
return; }
}
RsGxsGroupId grpId = it->second; RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32]; uint8_t encryption_master_key[32];
@ -5417,56 +5416,36 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId); mObserver->receiveDistantSearchResults(req, grpId);
} }
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring, bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos ) std::list<RsGxsGroupSummary>& group_infos )
{ {
group_infos.clear(); group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap; RsGxsGrpMetaTemporaryMap grpMetaMap;
{ {
RS_STACK_MUTEX(mNxsMutex) ; RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,12 +5471,11 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s); group_infos.push_back(s);
} }
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8 #ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl; GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif #endif
return !group_infos.empty(); return !group_infos.empty();
} }
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len) bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)

62
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2012-2012 by Christopher Evi-Parker * * Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* * * *
*******************************************************************************/ *******************************************************************************/
#ifndef RSGXSNETSERVICE_H #pragma once
#define RSGXSNETSERVICE_H
#include <list> #include <list>
#include <queue> #include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; } virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; } virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ; /// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ; virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ; virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ; virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -629,5 +665,3 @@ private:
bool mUseMetaCache; bool mUseMetaCache;
}; };
#endif // RSGXSNETSERVICE_H

607
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> * * Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h" #include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h" #include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1 //#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1; static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100; static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{ {
mRandomBias.clear(); mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services /* adds some variance in order to avoid doing all this tasks at once across
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20); * services */
mLastDump = time(NULL) + (RSRandom::random_u32()%20); auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
} }
//===========================================================================================================================================// //============================================================================//
// Transport Items // // Transport Items //
//===========================================================================================================================================// //============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem struct RsGxsNetTunnelItem: RsItem
{ {
public: public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype) explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{ {
// no priority. All items are encapsulated into generic Turtle items anyway. // no priority. All items are encapsulated into generic Turtle items anyway.
} }
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
}; };
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
}; };
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{ {
public: public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {} explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ; RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
} }
}; };
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{ {
public: public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer class RsGxsNetTunnelSerializer: public RsServiceSerializer
{ {
public: public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {} RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{ {
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL) if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{ {
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl; RS_ERR( "received item with wrong service ID ", service);
return NULL ; print_stacktrace();
return nullptr;
} }
switch(item_subtype) switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{ {
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ; case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; return new RsGxsNetTunnelVirtualPeerItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ; case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem; return new RsGxsNetTunnelKeepAliveItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem; case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem; return new RsGxsNetTunnelRandomBiasItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem; case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default: default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl; RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return NULL ; return nullptr;
} }
} }
}; };
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ; return mTurtle->turtleSearch(mem,size,this) ;
} }
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{ {
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl; GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ; search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ; uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL) if(mem == NULL)
return 0 ; return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ; return mTurtle->turtleSearch(mem,size,this) ;
} }
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits) std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{ {
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ; RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL) uint32_t size = tSerializer.size(&searchItem);
{ uint8_t* buf = rs_malloc<uint8_t>(size);
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ; tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ; requestId = mTurtle->turtleSearch(buf, size, this);
RsNetworkExchangeService *service = nullptr; if(!requestId) return std::errc::result_out_of_range;
{ return std::error_condition();
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service ;
search_result_item.group_infos = results ;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
return true ;
}
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
search_result_item.service = substring_gr->service ;
search_result_item.encrypted_group_data = encrypted_group_data ;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
return true ;
}
}
delete item;
return false ;
} }
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len) rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{ {
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl; bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ; RS_DBG3("");
if(result_gs != NULL) RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{ {
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
#ifdef DEBUG_RSGXSNETTUNNEL print_stacktrace();
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) return false;
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
delete item;
return ;
} }
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ; switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
if(result_gd != NULL) case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{ {
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl; if(!search_result_data)
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
{ {
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl; RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
delete item; "results!" );
return ; print_stacktrace();
break;
} }
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ; auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
result_gd->encrypted_group_data = NULL ; // prevents deletion max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
delete item; std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
return ; tSerializer.serialise(
} &search_result_item, search_result_data,
&search_result_data_size );
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl; return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
return false;
} }
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RS_DBG3(request_id);
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
auto castFailedWarn = [](const uint8_t subtype)
{
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
return it->second;
};
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{ {
groups = mGroups ; groups = mGroups ;
virtual_peers = mVirtualPeers ; virtual_peers = mVirtualPeers ;

62
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> * * Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once #pragma once
#include <map> #include <map>
#include <system_error>
#include "turtle/p3turtle.h" #include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h" #include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen. // and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
// //
class RsGxsNetTunnelItem ; struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ; class RsNetworkExchangeService ;
class RsGxsNetTunnelService: class RsGxsNetTunnelService:
@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
public RsGxsDistSync public RsGxsDistSync
{ {
public: public:
RsGxsNetTunnelService() ; RsGxsNetTunnelService();
virtual ~RsGxsNetTunnelService() ; ~RsGxsNetTunnelService() override;
/*! /*!
* \brief registerSearchableService * \brief registerSearchableService
@ -181,24 +184,38 @@ public:
*/ */
void dump() const; void dump() const;
/*! /*!
* \brief connectToTurtleRouter * Should be called after allocating a RsGxsNetTunnelService
* Should be called after allocating a RsGxsNetTunnelService * \param tr turtle router object
* \param tr turtle router object */
*/ void connectToTurtleRouter(p3turtle *tr) override;
virtual void connectToTurtleRouter(p3turtle *tr) ;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ; /** Gxs services (channels, forums...) are supposed to use this to request
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ; * searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*! ///@see RsTurtleClientService
* \brief receiveSearchRequest bool receiveSearchRequest(
* See RsTurtleClientService::@ unsigned char* search_request_data,
*/ uint32_t search_request_data_len,
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits); unsigned char*& search_result_data,
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len); uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread ///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config // Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid, std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const; Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected: protected:
// interaction with turtle router // interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ; void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ; void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items(); void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

145
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -3,8 +3,8 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2013-2013 by Christopher Evi-Parker * * Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h" #include "pqi/pqihash.h"
#include "gxs/rsgixs.h" #include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly // The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't // re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure. // happen anyway, but we still conduct these test as an extra safety measure.
@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
} }
RsGxsIntegrityCheck::RsGxsIntegrityCheck( RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex, RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType& RsSerialType&, RsGixs* gixs )
, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex), : mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {} mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true; return true;
} }
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds bool RsGxsSinglePassIntegrityCheck::check(
#ifdef RS_DEEP_CHANNEL_INDEX uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
{ {
#ifdef DEBUG_GXSUTIL #ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl; GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif #endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups // first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp; std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
} }
else else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list. msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
} }
else
{
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
}
delete grp; delete grp;
} }
@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(nxsMsg) if(nxsMsg)
nxsMsgS.insert(nxsMsg->msgId); nxsMsgS.insert(nxsMsg->msgId);
for (auto& msgId:msgIdV) for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end()) if(nxsMsgS.find(msgId) == nxsMsgS.end())
{ msgsToDel[grpId].insert(msgId);
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
} }
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit) for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash) if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{ {
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl; RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId); msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
} }
delete msg; delete msg;

36
libretroshare/src/gxs/rsgxsutil.h
View File

@ -3,8 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2013-2013 by Christopher Evi-Parker * * Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking }; enum CheckState { CheckStart, CheckChecking };
public: public:
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
/*! RsGixs* gixs );
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType&,
RsGixs *gixs);
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds); static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
bool isDone(); bool isDone();
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck class RsGxsSinglePassIntegrityCheck
{ {
public: public:
static bool check(
/*! uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
* std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
}; };
class GroupUpdate class GroupUpdate

62
libretroshare/src/gxs/rsnxs.h
View File

@ -3,8 +3,10 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> * * Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker * * Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* * * *
*******************************************************************************/ *******************************************************************************/
#pragma once
#ifndef RSGNP_H
#define RSGNP_H
#include <set> #include <set>
#include <string> #include <string>
#include "util/rstime.h"
#include <stdlib.h> #include <stdlib.h>
#include <list> #include <list>
#include <map> #include <map>
#include "util/rstime.h"
#include "services/p3service.h" #include "services/p3service.h"
#include "retroshare/rsreputations.h" #include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h" #include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService class RsNetworkExchangeService
{ {
public: public:
RsNetworkExchangeService() = default;
RsNetworkExchangeService(){ return;} virtual ~RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() {}
virtual uint16_t serviceType() const =0; virtual uint16_t serviceType() const =0;
/*! /*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0; virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0; virtual bool grpAutoSync() const =0;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS /// /// DISTANT SEARCH FUNCTIONS ///
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*! /*!
* \brief turtleGroupRequest * \brief turtleGroupRequest
@ -115,13 +129,17 @@ public:
*/ */
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0; virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*! /*!
* \brief receiveTurtleSearchResults * \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received * Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result * \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data * \param encrypted_group_data Group data
*/ */
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0; RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*! /*!
* \brief retrieveTurtleSearchResults * \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0; virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0; RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0; virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*! /*!
@ -306,5 +326,3 @@ public:
} }
} }
}; };
#endif // RSGNP_H

89
libretroshare/src/util/rsmemory.cc → libretroshare/src/gxs/rsnxsobserver.cpp
View File

@ -1,53 +1,36 @@
/******************************************************************************* /*******************************************************************************
* libretroshare/src/util: rsmemory.cc * * RetroShare General eXchange System *
* * * *
* libretroshare: retroshare core library * * Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* * * Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net> * * *
* * * This program is free software: you can redistribute it and/or modify *
* This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as *
* it under the terms of the GNU Lesser General Public License as * * published by the Free Software Foundation, either version 3 of the *
* published by the Free Software Foundation, either version 3 of the * * License, or (at your option) any later version. *
* License, or (at your option) any later version. * * *
* * * This program is distributed in the hope that it will be useful, *
* This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of *
* but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU Lesser General Public License for more details. *
* GNU Lesser General Public License for more details. * * *
* * * You should have received a copy of the GNU Lesser General Public License *
* You should have received a copy of the GNU Lesser General Public License * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * *
* * *******************************************************************************/
*******************************************************************************/
#include "util/rsmemory.h" #include "gxs/rsnxsobserver.h"
void *rs_malloc(size_t size) const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
{
static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything! std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
if(size == 0) {
{ switch(static_cast<RsNxsObserverErrorNum>(ev))
std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl; {
print_stacktrace() ; case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return NULL ; return std::errc::operation_not_supported;
} default:
return std::error_condition(ev, *this);
if(size > SAFE_MEMALLOC_THRESHOLD) }
{ }
std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
void *mem = malloc(size) ;
if(mem == NULL)
{
std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
return mem ;
}

104
libretroshare/src/gxs/rsnxsobserver.h
View File

@ -3,7 +3,10 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher * * Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* * * *
*******************************************************************************/ *******************************************************************************/
#ifndef RSNXSOBSERVER_H #pragma once
#define RSNXSOBSERVER_H
#include <set> #include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h" #include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId ; typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver class RsNxsObserver
{ {
public:
RsNxsObserver() {}
public: public:
/*! /*!
@ -56,6 +99,42 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl; std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
} }
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*! /*!
* @param grpId group id * @param grpId group id
*/ */
@ -70,6 +149,7 @@ public:
* @param grpId group id * @param grpId group id
*/ */
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0; virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

12
libretroshare/src/libretroshare.pro
View File

@ -633,7 +633,6 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \ util/rsexpr.cc \
util/smallobject.cc \ util/smallobject.cc \
util/rsdir.cc \ util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \ util/rsdiscspace.cc \
util/rsnet.cc \ util/rsnet.cc \
util/rsnet_ss.cc \ util/rsnet_ss.cc \
@ -712,7 +711,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \ gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \ gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \ gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# gxs tunnels # gxs tunnels
HEADERS += gxstunnel/p3gxstunnel.h \ HEADERS += gxstunnel/p3gxstunnel.h \
@ -936,6 +936,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp SOURCES += jsonapi/jsonapi.cpp
} }
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index { rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp SOURCES *= deep_search/commonutils.cpp

147
libretroshare/src/retroshare/rsgxschannels.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -46,6 +46,11 @@ class RsGxsChannels;
extern RsGxsChannels* rsGxsChannels; extern RsGxsChannels* rsGxsChannels;
/* TODO: At this abstraction level that turtle is used for distant searches
* should be an hidden implementation detail. As of today a bit of those
* implementation details leaks around because we use TurtleRequestId in this
* interface */
struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData
{ {
RsGxsChannelGroup() : mAutoDownload(false) {} RsGxsChannelGroup() : mAutoDownload(false) {}
@ -108,29 +113,38 @@ struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData
enum class RsChannelEventCode: uint8_t enum class RsChannelEventCode: uint8_t
{ {
UNKNOWN = 0x00, UNKNOWN = 0x00,
NEW_CHANNEL = 0x01, // emitted when new channel is received NEW_CHANNEL = 0x01, /// emitted when new channel is received
UPDATED_CHANNEL = 0x02, // emitted when existing channel is updated UPDATED_CHANNEL = 0x02, /// emitted when existing channel is updated
NEW_MESSAGE = 0x03, // new message reeived in a particular channel (group and msg id) NEW_MESSAGE = 0x03, /// new message reeived in a particular channel (group and msg id)
UPDATED_MESSAGE = 0x04, // existing message has been updated in a particular channel UPDATED_MESSAGE = 0x04, /// existing message has been updated in a particular channel
RECEIVED_PUBLISH_KEY = 0x05, // publish key for this channel has been received RECEIVED_PUBLISH_KEY = 0x05, /// publish key for this channel has been received
SUBSCRIBE_STATUS_CHANGED = 0x06, // subscription for channel mChannelGroupId changed. SUBSCRIBE_STATUS_CHANGED = 0x06, /// subscription for channel mChannelGroupId changed.
READ_STATUS_CHANGED = 0x07, // existing message has been read or set to unread READ_STATUS_CHANGED = 0x07, /// existing message has been read or set to unread
RECEIVED_DISTANT_SEARCH_RESULT = 0x08, // result for the given group id available for the given turtle request id
STATISTICS_CHANGED = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed /** Result for the given group id available for the given turtle request id
SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed * @deprecated kept for retrocompatibility with old search system new code
NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message * should use @see DISTANT_SEARCH_RESULT instead */
NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment RECEIVED_TURTLE_SEARCH_RESULT = 0x08,
DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system
STATISTICS_CHANGED = 0x09, /// stats (nb of supplier friends, how many msgs they have etc) has changed
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
NEW_COMMENT = 0x0b, /// new comment arrived/published. mChannelThreadId gives the ID of the commented message
NEW_VOTE = 0x0c, /// new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, /// channel was deleted by auto-cleaning system
DELETED_POST = 0x0e, /// Post deleted (usually by cleaning)
DISTANT_SEARCH_RESULT = 0x0f /// Distant search result received
}; };
struct RsGxsChannelEvent: RsEvent struct RsGxsChannelEvent: RsEvent
{ {
RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {} RsGxsChannelEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
RsChannelEventCode mChannelEventCode; RsChannelEventCode mChannelEventCode;
RsGxsGroupId mChannelGroupId; RsGxsGroupId mChannelGroupId;
RsGxsMessageId mChannelMsgId; RsGxsMessageId mChannelMsgId;
RsGxsMessageId mChannelThreadId; RsGxsMessageId mChannelThreadId;
///* @see RsEvent @see RsSerializable ///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
@ -143,13 +157,14 @@ struct RsGxsChannelEvent: RsEvent
} }
}; };
// This event is used to factor multiple search results notifications in a single event. /** This event is used to factor multiple search results notifications in a
* single event.*/
struct RsGxsChannelSearchResultEvent: RsEvent struct RS_DEPRECATED_FOR(RsGxsChannelDistantSearchResultEvent)
RsGxsChannelSearchResultEvent: RsEvent
{ {
RsGxsChannelSearchResultEvent(): RsGxsChannelSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS), RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {} mChannelEventCode(RsChannelEventCode::RECEIVED_TURTLE_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode; RsChannelEventCode mChannelEventCode;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap; std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap;
@ -164,6 +179,29 @@ struct RsGxsChannelSearchResultEvent: RsEvent
} }
}; };
/** This event is fired once distant search results are received */
struct RsGxsChannelDistantSearchResultEvent: RsEvent
{
RsGxsChannelDistantSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::DISTANT_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mChannelEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
{ {
public: public:
@ -386,11 +424,15 @@ public:
* @brief Get channel content summaries * @brief Get channel content summaries
* @jsonapi{development} * @jsonapi{development}
* @param[in] channelId id of the channel of which the content is requested * @param[in] channelId id of the channel of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries * @param[out] summaries storage for summaries
* @return false if something failed, true otherwhise * @return false if something failed, true otherwhise
*/ */
virtual bool getContentSummaries( const RsGxsGroupId& channelId, virtual std::error_condition getContentSummaries(
std::vector<RsMsgMetaData>& summaries ) = 0; const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/** /**
* @brief Toggle post read status. Blocking API. * @brief Toggle post read status. Blocking API.
@ -422,22 +464,23 @@ public:
virtual bool subscribeToChannel( const RsGxsGroupId& channelId, virtual bool subscribeToChannel( const RsGxsGroupId& channelId,
bool subscribe ) = 0; bool subscribe ) = 0;
/** /**
* \brief Retrieve statistics about the channel service * @brief Retrieve statistics about the channel service
* @jsonapi{development} * @jsonapi{development}
* \param[out] stat Statistics structure * @param[out] stat storage for statistics
* \return * @return true on success false otherwise
*/ */
virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0;
/** /**
* \brief Retrieve statistics about the given channel * @brief Retrieve statistics about the given channel
* @jsonapi{development} * @jsonapi{development}
* \param[in] channelId Id of the channel group * @param[in] channelId Id of the channel group
* \param[out] stat Statistics structure * @param[out] stat storage for statistics
* \return * @return true on success false otherwise
*/ */
virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0; virtual bool getChannelStatistics(
const RsGxsGroupId& channelId, GxsGroupStatistic& stat ) =0;
/// default base URL used for channels links @see exportChannelLink /// default base URL used for channels links @see exportChannelLink
static const std::string DEFAULT_CHANNEL_BASE_URL; static const std::string DEFAULT_CHANNEL_BASE_URL;
@ -496,14 +539,18 @@ public:
std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
/** /**
* @brief Search the turtle reachable network for matching channels * @brief Search the whole reachable network for matching channels and
* contents
* @jsonapi{development} * @jsonapi{development}
* An @see RsGxsChannelSearchResultEvent is emitted when matching channels * An @see RsGxsChannelSearchResultEvent is emitted when matching results
* arrives from the network * arrives from the network
* @param[in] matchString string to search into the channels * @param[in] matchString string to search into the channels
* @return search id * @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/ */
virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0; virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/** /**
* @brief Retrieve available search results * @brief Retrieve available search results
@ -533,16 +580,18 @@ public:
* @param[out] distantGroup storage for group data * @param[out] distantGroup storage for group data
* @return false on error, true otherwise * @return false on error, true otherwise
*/ */
virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; virtual bool getDistantSearchResultGroupData(
const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
/** /**
* @brief getDistantSearchStatus * @brief Get the status of ongoing search
* Returns the status of ongoing search: unknown (probably not even searched), known as a search result, * @return unknown (probably not even searched), known as a search result,
* data request ongoing and data available * data request ongoing and data available
*/ */
virtual DistantSearchGroupStatus getDistantSearchStatus(const RsGxsGroupId& group_id) =0; virtual DistantSearchGroupStatus getDistantSearchStatus(
const RsGxsGroupId& group_id ) =0;
/** /**
* @brief Clear accumulated search results * @brief Clear accumulated search results
* @jsonapi{development} * @jsonapi{development}
* @param[in] reqId search id * @param[in] reqId search id

62
libretroshare/src/retroshare/rsgxsiface.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Christopher Evi-Parker * * Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h" #include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h" #include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*! /*!
* This structure is used to transport group summary information when a GXS * This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context * service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a * string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible. * GroupMeta object, so as to make search responses as light as possible.
*/ */
struct RsGxsGroupSummary : RsSerializable struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{ {
RsGxsGroupSummary() : RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a * strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible. * GroupMeta object, so as to make search responses as light as possible.
*/ */
struct RsGxsGroupSearchResults : RsSerializable struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{ {
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults() RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0) : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{} {}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default; virtual ~RsGxsGroupSearchResults() = default;
}; };
/*! /*!
* Stores ids of changed gxs groups and messages. * Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes. * It is used to notify about GXS changes.

2
libretroshare/src/retroshare/rsgxsifacetypes.h
View File

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0), mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {} mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {} virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta); void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); } RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

65
libretroshare/src/rsitems/rsgxschannelitems.h
View File

@ -3,7 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * * along with this program. If not, see <https://www.gnu.org/licenses/>. *
* * * *
*******************************************************************************/ *******************************************************************************/
#ifndef RS_GXS_CHANNEL_ITEMS_H #pragma once
#define RS_GXS_CHANNEL_ITEMS_H
#include <map> #include <map>
@ -30,14 +31,23 @@
#include "serialiser/rstlvfileitem.h" #include "serialiser/rstlvfileitem.h"
#include "serialiser/rstlvimage.h" #include "serialiser/rstlvimage.h"
#include "serialiser/rsserializable.h"
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "serialiser/rsserializer.h" #include "serialiser/rsserializer.h"
#include "util/rsdir.h" #include "util/rsdir.h"
enum class RsGxsChannelItems : uint8_t
{
GROUP_ITEM = 0x02,
POST_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02; const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03; const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03;
class RsGxsChannelGroupItem : public RsGxsGrpItem class RsGxsChannelGroupItem : public RsGxsGrpItem
@ -79,6 +89,47 @@ public:
RsTlvImage mThumbnail; RsTlvImage mThumbnail;
}; };
struct RsGxsChannelsSearchRequest : RsSerializable
{
RsGxsChannelsSearchRequest() : mType(RsGxsChannelItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsChannelsSearchRequest() override = default;
};
struct RsGxsChannelsSearchReply : RsSerializable
{
RsGxsChannelsSearchReply() : mType(RsGxsChannelItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsChannelsSearchReply() override = default;
};
class RsGxsChannelSerialiser : public RsGxsCommentSerialiser class RsGxsChannelSerialiser : public RsGxsCommentSerialiser
{ {
@ -89,5 +140,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ; virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
}; };
#endif /* RS_GXS_CHANNEL_ITEMS_H */

5
libretroshare/src/rsitems/rsitem.h
View File

@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob, virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0; RsGenericSerializer::SerializeContext&)// = 0;
{ {
std::cerr << "(EE) RsItem::serial_process(...) called by an item using" RS_ERR( "called by an item using new serialization system without "
<< "new serialization classes, but not derived! Class is " "overriding Class is: ", typeid(*this).name() );
<< typeid(*this).name() << std::endl;
print_stacktrace(); print_stacktrace();
} }

14
libretroshare/src/rsitems/rsserviceids.h
View File

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t enum class RsServiceType : uint16_t
{ {
NONE = 0, /// To detect non-initialized reads /// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011, GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012, CHAT = 0x0012,
MSG = 0x0013, MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025, GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026, SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027, DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028, GXS_TUNNEL = 0x0028,
BANLIST = 0x0101, BANLIST = 0x0101,
STATUS = 0x0102, STATUS = 0x0102,
NXS = 0x0200, NXS = 0x0200,
@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216, POSTED = 0x0216,
CHANNELS = 0x0217, CHANNELS = 0x0217,
GXSCIRCLE = 0x0218, GXSCIRCLE = 0x0218,
/// not gxs, but used with identities. /// not gxs, but used with identities.
REPUTATION = 0x0219, REPUTATION = 0x0219,
GXS_RECOGN = 0x0220, GXS_RECOGN = 0x0220,
@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317, CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001, PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002, PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003, PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes. /// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB, PACKET_SLICING_PROBE = 0xAABB,

74
libretroshare/src/services/p3gxschannels.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -22,6 +23,8 @@
*******************************************************************************/ *******************************************************************************/
#pragma once #pragma once
#include <map>
#include <string>
#include "retroshare/rsgxschannels.h" #include "retroshare/rsgxschannels.h"
#include "services/p3gxscommon.h" #include "services/p3gxscommon.h"
@ -30,9 +33,7 @@
#include "util/rsmemory.h" #include "util/rsmemory.h"
#include "util/rsdebug.h" #include "util/rsdebug.h"
#include "util/rstickevent.h" #include "util/rstickevent.h"
#include "deep_search/channelsindex.hpp"
#include <map>
#include <string>
// This class is only a helper to parse the channel group service string. // This class is only a helper to parse the channel group service string.
@ -56,6 +57,11 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels,
public: public:
p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes, p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes,
RsGixs* gixs ); RsGixs* gixs );
/// @see RsGxsChannels
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
virtual RsServiceInfo getServiceInfo() override; virtual RsServiceInfo getServiceInfo() override;
virtual void service_tick() override; virtual void service_tick() override;
@ -69,7 +75,7 @@ protected:
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&) virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override; virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) override; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) override;
virtual bool clearDistantSearchResults(TurtleRequestId req) override; virtual bool clearDistantSearchResults(TurtleRequestId req) override;
virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override; virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override;
@ -112,24 +118,6 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled)
virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override; virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override;
virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override; virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override;
#ifdef TO_REMOVE
/// @see RsGxsChannels::turtleSearchRequest
virtual bool turtleSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::turtleChannelRequest
virtual bool turtleChannelRequest(
const RsGxsGroupId& channelId,
const std::function<void (const RsGxsChannelGroup& result)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::localSearchRequest
virtual bool localSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
rstime_t maxWait = 30 ) override;
#endif
/** /**
* Receive results from turtle search @see RsGenExchange @see RsNxsObserver * Receive results from turtle search @see RsGenExchange @see RsNxsObserver
* @see RsGxsNetService::receiveTurtleSearchResults * @see RsGxsNetService::receiveTurtleSearchResults
@ -215,9 +203,10 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
const std::set<RsGxsMessageId> &contentIds, const std::set<RsGxsMessageId> &contentIds,
std::vector<RsGxsComment> &comments) override; std::vector<RsGxsComment> &comments) override;
/// Implementation of @see RsGxsChannels::getContentSummaries /// @see RsGxsChannels
bool getContentSummaries( std::error_condition getContentSummaries(
const RsGxsGroupId& channelId, const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override; std::vector<RsMsgMetaData>& summaries ) override;
/// Implementation of @see RsGxsChannels::getChannelStatistics /// Implementation of @see RsGxsChannels::getChannelStatistics
@ -297,6 +286,17 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
virtual bool shareChannelKeys( virtual bool shareChannelKeys(
const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) override; const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) override;
#ifdef RS_DEEP_CHANNEL_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// Implementation of @see RsGxsChannels::createChannel /// Implementation of @see RsGxsChannels::createChannel
RS_DEPRECATED_FOR(createChannelV2) RS_DEPRECATED_FOR(createChannelV2)
bool createChannel(RsGxsChannelGroup& channel) override; bool createChannel(RsGxsChannelGroup& channel) override;
@ -313,7 +313,6 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
RS_DEPRECATED_FOR(createVoteV2) RS_DEPRECATED_FOR(createVoteV2)
bool createVote(RsGxsVote& vote) override; bool createVote(RsGxsVote& vote) override;
protected: protected:
// Overloaded from GxsTokenQueue for Request callbacks. // Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type virtual void handleResponse(uint32_t token, uint32_t req_type
@ -329,7 +328,6 @@ static uint32_t channelsAuthenPolicy();
void request_SpecificSubscribedGroups(const std::list<RsGxsGroupId> &groups); void request_SpecificSubscribedGroups(const std::list<RsGxsGroupId> &groups);
void load_SubscribedGroups(const uint32_t &token); void load_SubscribedGroups(const uint32_t &token);
void request_SpecificUnprocessedPosts(std::list<std::pair<RsGxsGroupId, RsGxsMessageId> > &ids);
void request_GroupUnprocessedPosts(const std::list<RsGxsGroupId> &grouplist); void request_GroupUnprocessedPosts(const std::list<RsGxsGroupId> &grouplist);
void load_unprocessedPosts(uint32_t token); void load_unprocessedPosts(uint32_t token);
@ -390,26 +388,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
rstime_t mLastDistantSearchNotificationTS; rstime_t mLastDistantSearchNotificationTS;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify; std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
#ifdef TO_REMOVE
/** Store search callbacks with timeout*/
std::map<
TurtleRequestId,
std::pair<
std::function<void (const RsGxsGroupSummary&)>,
std::chrono::system_clock::time_point >
> mSearchCallbacksMap;
RsMutex mSearchCallbacksMapMutex;
/** Store distant channels requests callbacks with timeout*/ #ifdef RS_DEEP_CHANNEL_INDEX
std::map< DeepChannelsIndex mDeepIndex;
TurtleRequestId,
std::pair<
std::function<void (const RsGxsChannelGroup&)>,
std::chrono::system_clock::time_point >
> mDistantChannelsCallbacksMap;
RsMutex mDistantChannelsCallbacksMapMutex;
/// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap
void cleanTimedOutCallbacks();
#endif #endif
}; };

9
libretroshare/src/util/rsdebug.cc
View File

@ -3,9 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -36,6 +36,9 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
" not available in category: " + categoryName; " not available in category: " + categoryName;
} }
std::error_condition rs_errno_to_condition(int errno_code)
{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////

10
libretroshare/src/util/rsdebug.h
View File

@ -2,8 +2,8 @@
* RetroShare debugging utilities * * RetroShare debugging utilities *
* * * *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> * * Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> * * Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> * * Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
* the message around */ * the message around */
std::string rsErrorNotInCategory(int errNum, const std::string& categoryName); std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
/** Convert C errno codes to modern C++11 std::error_condition, this is quite
* useful to use toghether with C functions used around the code like `malloc`,
* `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
*/
std::error_condition rs_errno_to_condition(int errno_code);
template <RsLoggerCategories CATEGORY> template <RsLoggerCategories CATEGORY>
struct t_RsLogger : std::ostringstream struct t_RsLogger : std::ostringstream

70
libretroshare/src/util/rsmemory.h
View File

@ -3,8 +3,9 @@
* * * *
* libretroshare: retroshare core library * * libretroshare: retroshare core library *
* * * *
* Copyright 2012 Cyril Soler <csoler@users.sourceforge.net> * * Copyright (C) 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net> * * Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* * * *
* This program is free software: you can redistribute it and/or modify * * This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as * * it under the terms of the GNU Lesser General Public License as *
@ -25,8 +26,10 @@
#include <cstdlib> #include <cstdlib>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <system_error>
#include "util/stacktrace.h" #include "util/stacktrace.h"
#include "util/rsdebug.h"
/** /**
* @brief Shorthand macro to declare optional functions output parameters * @brief Shorthand macro to declare optional functions output parameters
@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
* @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
template<typename T> using rs_owner_ptr = T*; template<typename T> using rs_owner_ptr = T*;
void *rs_malloc(size_t size) ;
/// 1Gb should be enough for everything!
static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
/** Comfortable templated safer malloc, just use it specifing the type of the
* pointer to be returned without need of ugly casting the returned pointer
* `uint8_t* ptr = rs_malloc<uint8_t>(40);`
* @param[in] size number of bytes to allocate
* @param[out] ec optional storage for error details. Value is meaningful only
* whem nullptr is returned.
* @return nullptr on error, pointer to the allocated chuck of memory on success
*/
template<typename T = void> rs_owner_ptr<T> rs_malloc(
size_t size,
rs_view_ptr<std::error_condition> ec = nullptr )
{
if(size == 0)
{
if(!ec)
{
RS_ERR("A chunk of size 0 was requested");
print_stacktrace();
exit(static_cast<int>(std::errc::invalid_argument));
}
*ec = std::errc::invalid_argument;
return nullptr;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
if(!ec)
{
RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
" was requested" );
exit(static_cast<int>(std::errc::argument_out_of_domain));
}
*ec = std::errc::argument_out_of_domain;
return nullptr;
}
void* mem = malloc(size);
if(!mem)
{
if(!ec)
{
RS_ERR( "Allocation failed for a chunk of ", size,
" bytes with: ", errno);
print_stacktrace();
exit(errno);
}
*ec = rs_errno_to_condition(errno);
return nullptr;
}
return static_cast<rs_owner_ptr<T>>(mem);
}
/** @deprecated use std::unique_ptr instead /** @deprecated use std::unique_ptr instead
// This is a scope guard to release the memory block when going of of the current scope. // This is a scope guard to release the memory block when going of of the current scope.
@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
// //
// } // mem gets freed automatically // } // mem gets freed automatically
*/ */
class RsTemporaryMemory class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
{ {
public: public:
explicit RsTemporaryMemory(size_t s) explicit RsTemporaryMemory(size_t s)

4
retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp
View File

@ -411,7 +411,9 @@ void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id)
TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string)
{ {
return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; TurtleRequestId searchId;
rsGxsChannels->distantSearchRequest(search_string.toStdString(), searchId);
return searchId;
} }
bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos) bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)