Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

First working prototype of GXS service search

Browse Source 
Channels are now able to take advantage of the new deep search
  goodies

Rebase on top of master 2021/10/19
This commit is contained in:
Gioacchino Mazzurco 2021-02-08 17:04:04 +01:00
parent ab349a8157
commit 1b551d809f
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
30 changed files with 1455 additions and 874 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
libretroshare/src/deep_search/channelsindex.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -20,16 +20,23 @@
#include "deep_search/channelsindex.hpp"
#include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "util/rsdebuglevel3.h"
uint32_t DeepChannelsIndex::search(
/*static*/ std::string DeepChannelsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; }
std::error_condition DeepChannelsIndex::search(
const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults )
{
RS_DBG3(queryStr);
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(dbPath()) );
if(!dbPtr) return 0;
DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
@ -63,17 +70,13 @@ uint32_t DeepChannelsIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}
void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
std::error_condition DeepChannelsIndex::indexChannelGroup(
const RsGxsChannelGroup& chan )
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
RS_DBG4(chan);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
@ -94,21 +97,8 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
termgenerator.increase_termpos();
termgenerator.index_text(chan.mDescription);
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", chan.mMeta.mGroupId.toStdString());
const std::string idTerm("Q" + chanUrl.toString());
chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs));
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(channelIndexId(chan.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
@ -117,35 +107,32 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer. "Q" prefix is a Xapian convention for unique id term.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId)
std::error_condition DeepChannelsIndex::removeChannelFromIndex(
const RsGxsGroupId& grpId )
{
RS_DBG3(grpId);
// "Q" prefix is a Xapian convention for unique id term.
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString());
std::string idTerm("Q" + chanUrl.toString());
const std::string idTerm("Q" + channelIndexId(grpId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
return std::error_condition();
}
void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
std::error_condition DeepChannelsIndex::indexChannelPost(
const RsGxsChannelPost& post )
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
RS_DBG4(post);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
@ -160,21 +147,16 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// TODO: we should strip out HTML tags instead of skipping indexing
// Avoid indexing HTML
bool isPlainMsg =
post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>';
if(isPlainMsg)
termgenerator.index_text(post.mMsg, 1, "XD");
// Avoid indexing RetroShare-gui HTML tags
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
termgenerator.index_text(
DeepSearch::simpleTextHtmlExtract(post.mMsg), 1, "XD" );
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
if(isPlainMsg)
{
termgenerator.increase_termpos();
termgenerator.index_text(post.mMsg);
}
termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg);
for(const RsGxsFile& attachment : post.mFiles)
{
@ -184,47 +166,50 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
termgenerator.index_text(attachment.mName);
}
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", post.mMeta.mGroupId.toStdString())
.setQueryKV("msgid", post.mMeta.mMsgId.toStdString());
std::string idTerm("Q" + postUrl.toString());
postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs));
postUrl.setQueryKV("name", post.mMeta.mMsgName);
postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString());
std::string rsLink(postUrl.toString());
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(post.mMeta.mGroupId, post.mMeta.mMsgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
if(isPlainMsg)
doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg);
else doc.set_data(post.mMeta.mMsgName);
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
void DeepChannelsIndex::removeChannelPostFromIndex(
std::error_condition DeepChannelsIndex::removeChannelPostFromIndex(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RS_DBG3(grpId, msgId);
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepChannelsIndex::channelIndexId(RsGxsGroupId grpId)
{
RsUrl channelIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL);
channelIndexId.setQueryKV(
RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString() );
return channelIndexId.toString();
}
/*static*/ std::string DeepChannelsIndex::postIndexId(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString())
.setQueryKV("msgid", msgId.toStdString());
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
RsUrl postIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL);
postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString());
postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_MSG_ID_FIELD, msgId.toStdString());
return postIndexId.toString();
}

39
libretroshare/src/deep_search/channelsindex.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -24,8 +24,8 @@
#include "util/rstime.h"
#include "retroshare/rsgxschannels.h"
#include "retroshare/rsinit.h"
#include "util/rsurl.h"
#include "deep_search/commonutils.hpp"
struct DeepChannelsSearchResult
{
@ -36,28 +36,34 @@ struct DeepChannelsSearchResult
struct DeepChannelsIndex
{
explicit DeepChannelsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
static uint32_t search( const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results,
uint32_t maxResults = 100 );
std::error_condition search(
const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results,
uint32_t maxResults = 100 );
static void indexChannelGroup(const RsGxsChannelGroup& chan);
std::error_condition indexChannelGroup(const RsGxsChannelGroup& chan);
static void removeChannelFromIndex(RsGxsGroupId grpId);
std::error_condition removeChannelFromIndex(const RsGxsGroupId& grpId);
static void indexChannelPost(const RsGxsChannelPost& post);
std::error_condition indexChannelPost(const RsGxsChannelPost& post);
static void removeChannelPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
std::error_condition removeChannelPostFromIndex(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
static uint32_t indexFile(const std::string& path);
static std::string dbDefaultPath();
private:
static std::string channelIndexId(RsGxsGroupId grpId);
static std::string postIndexId(RsGxsGroupId grpId, RsGxsMessageId msgId);
enum : Xapian::valueno
{
@ -68,10 +74,7 @@ private:
BAD_VALUENO = Xapian::BAD_VALUENO
};
static const std::string& dbPath()
{
static const std::string dbDir =
RsAccounts::AccountDirectory() + "/deep_channels_xapian_db";
return dbDir;
}
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
};

154
libretroshare/src/deep_search/commonutils.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,39 +18,17 @@
* *
*******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
@ -60,12 +38,12 @@ std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
new Xapian::Database(path, flags) );
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
catch(Xapian::DatabaseOpeningError& e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
catch(Xapian::DatabaseLockError&)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
@ -90,4 +68,116 @@ std::string timetToXapianDate(const rstime_t& time)
return date;
}
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(50l, maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
std::string::size_type oPos;
std::string::size_type cPos;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
}
return retVal;
}
}

34
libretroshare/src/deep_search/commonutils.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h>
#include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
}

83
libretroshare/src/deep_search/filesindex.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,47 +18,47 @@
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
if(db)
{
oldDoc = db.get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
if( pIt != db->postlist_end(idTerm) )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
oldDoc = db->get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
}
db.reset(); // Release DB read lock ASAP
}
Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
RS_DBG3(hash);
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
mWriteQueue.push([hash](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + hash.toStdString()); });
db->delete_document("Q" + hash.toStdString());
return true;
return std::error_condition();
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
std::error_condition DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}

23
libretroshare/src/deep_search/filesindex.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
@ -29,6 +26,9 @@
#include <map>
#include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
std::error_condition indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

6
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -549,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX

4
libretroshare/src/ft/ftserver.cc
View File

@ -5,7 +5,7 @@
* *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -2037,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{
RsFileSearchResultItem resIt;

42
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -5,6 +5,7 @@
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60)
{
mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
}
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)

134
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
/* always ignore case */
return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{
@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{
std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_CHANNEL_INDEX
#ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important
* to test that the group meta is both non null and actually corresponds to the group id we seek. */
* mDataStore may in some situations allocate an empty group meta
* data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId);
@ -5332,18 +5330,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId);
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
return;
}
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
RS_WARN("Received search results for unknown request: ", req);
return;
}
RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32];
@ -5417,56 +5416,36 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId);
}
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos )
{
group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,12 +5471,11 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif
return !group_infos.empty();
return !group_infos.empty();
}
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)

62
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGXSNETSERVICE_H
#define RSGXSNETSERVICE_H
#pragma once
#include <list>
#include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
/// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -629,5 +665,3 @@ private:
bool mUseMetaCache;
};
#endif // RSGXSNETSERVICE_H

607
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
mLastDump = time(NULL) + (RSRandom::random_u32()%20);
/* adds some variance in order to avoid doing all this tasks at once across
* services */
auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
}
//===========================================================================================================================================//
// Transport Items //
//===========================================================================================================================================//
//============================================================================//
// Transport Items //
//============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem
struct RsGxsNetTunnelItem: RsItem
{
public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{
// no priority. All items are encapsulated into generic Turtle items anyway.
}
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
};
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
};
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{
public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
}
};
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{
public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer
{
public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
return NULL ;
RS_ERR( "received item with wrong service ID ", service);
print_stacktrace();
return nullptr;
}
switch(item_subtype)
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
return new RsGxsNetTunnelVirtualPeerItem;
case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
return new RsGxsNetTunnelKeepAliveItem;
case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
return new RsGxsNetTunnelRandomBiasItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
return NULL ;
RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return nullptr;
}
}
};
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ;
}
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ;
uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL)
return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
uint32_t size = tSerializer.size(&searchItem);
uint8_t* buf = rs_malloc<uint8_t>(size);
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ;
RsNetworkExchangeService *service = nullptr;
requestId = mTurtle->turtleSearch(buf, size, this);
if(!requestId) return std::errc::result_out_of_range;
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service ;
search_result_item.group_infos = results ;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
return true ;
}
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
search_result_item.service = substring_gr->service ;
search_result_item.encrypted_group_data = encrypted_group_data ;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
return true ;
}
}
delete item;
return false ;
return std::error_condition();
}
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl;
bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
RS_DBG3("");
if(result_gs != NULL)
RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
#ifdef DEBUG_RSGXSNETTUNNEL
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
delete item;
return ;
RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
print_stacktrace();
return false;
}
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
if(result_gd != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{
if(!search_result_data)
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
"results!" );
print_stacktrace();
break;
}
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
result_gd->encrypted_group_data = NULL ; // prevents deletion
delete item;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
return ;
}
tSerializer.serialise(
&search_result_item, search_result_data,
&search_result_data_size );
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
return false;
}
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RS_DBG3(request_id);
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
auto castFailedWarn = [](const uint8_t subtype)
{
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
return it->second;
};
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{
groups = mGroups ;
virtual_peers = mVirtualPeers ;

62
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once
#include <map>
#include <system_error>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService:
@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
public RsGxsDistSync
{
public:
RsGxsNetTunnelService() ;
virtual ~RsGxsNetTunnelService() ;
RsGxsNetTunnelService();
~RsGxsNetTunnelService() override;
/*!
* \brief registerSearchableService
@ -181,24 +184,38 @@ public:
*/
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
/*!
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
void connectToTurtleRouter(p3turtle *tr) override;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/** Gxs services (channels, forums...) are supposed to use this to request
* searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
///@see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* search_request_data,
uint32_t search_request_data_len,
unsigned char*& search_result_data,
uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread
///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected:
// interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

145
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
, RsGixs* gixs )
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true;
}
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
bool RsGxsSinglePassIntegrityCheck::check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
{
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
}
else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(nxsMsg)
nxsMsgS.insert(nxsMsg->msgId);
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
msgsToDel[grpId].insert(msgId);
}
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete msg;

36
libretroshare/src/gxs/rsgxsutil.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType&,
RsGixs *gixs);
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
RsGixs* gixs );
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
bool isDone();
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
static bool check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
};
class GroupUpdate

62
libretroshare/src/gxs/rsnxs.h
View File

@ -3,8 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker *
* Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGNP_H
#define RSGNP_H
#pragma once
#include <set>
#include <string>
#include "util/rstime.h"
#include <stdlib.h>
#include <list>
#include <map>
#include "util/rstime.h"
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService
{
public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() = default;
virtual uint16_t serviceType() const =0;
/*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*!
* \brief turtleGroupRequest
@ -115,13 +129,17 @@ public:
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*!
* \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
@ -306,5 +326,3 @@ public:
}
}
};
#endif // RSGNP_H

89
libretroshare/src/util/rsmemory.cc → libretroshare/src/gxs/rsnxsobserver.cpp
View File

@ -1,53 +1,36 @@
/*******************************************************************************
* libretroshare/src/util: rsmemory.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "util/rsmemory.h"
void *rs_malloc(size_t size)
{
static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything!
if(size == 0)
{
std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
void *mem = malloc(size) ;
if(mem == NULL)
{
std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
return mem ;
}
/*******************************************************************************
* RetroShare General eXchange System *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "gxs/rsnxsobserver.h"
const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
{
switch(static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return std::errc::operation_not_supported;
default:
return std::error_condition(ev, *this);
}
}

104
libretroshare/src/gxs/rsnxsobserver.h
View File

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher *
* Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSOBSERVER_H
#define RSNXSOBSERVER_H
#pragma once
#include <set>
#include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId ;
typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver
{
public:
RsNxsObserver() {}
public:
/*!
@ -56,6 +99,42 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*!
* @param grpId group id
*/
@ -70,6 +149,7 @@ public:
* @param grpId group id
*/
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H
RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

12
libretroshare/src/libretroshare.pro
View File

@ -633,7 +633,6 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \
util/rsnet.cc \
util/rsnet_ss.cc \
@ -712,7 +711,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc
gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# gxs tunnels
HEADERS += gxstunnel/p3gxstunnel.h \
@ -936,6 +936,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp

147
libretroshare/src/retroshare/rsgxschannels.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -46,6 +46,11 @@ class RsGxsChannels;
extern RsGxsChannels* rsGxsChannels;
/* TODO: At this abstraction level that turtle is used for distant searches
* should be an hidden implementation detail. As of today a bit of those
* implementation details leaks around because we use TurtleRequestId in this
* interface */
struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData
{
RsGxsChannelGroup() : mAutoDownload(false) {}
@ -108,29 +113,38 @@ struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData
enum class RsChannelEventCode: uint8_t
{
UNKNOWN = 0x00,
NEW_CHANNEL = 0x01, // emitted when new channel is received
UPDATED_CHANNEL = 0x02, // emitted when existing channel is updated
NEW_MESSAGE = 0x03, // new message reeived in a particular channel (group and msg id)
UPDATED_MESSAGE = 0x04, // existing message has been updated in a particular channel
RECEIVED_PUBLISH_KEY = 0x05, // publish key for this channel has been received
SUBSCRIBE_STATUS_CHANGED = 0x06, // subscription for channel mChannelGroupId changed.
READ_STATUS_CHANGED = 0x07, // existing message has been read or set to unread
RECEIVED_DISTANT_SEARCH_RESULT = 0x08, // result for the given group id available for the given turtle request id
STATISTICS_CHANGED = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed
SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed
NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message
NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system
NEW_CHANNEL = 0x01, /// emitted when new channel is received
UPDATED_CHANNEL = 0x02, /// emitted when existing channel is updated
NEW_MESSAGE = 0x03, /// new message reeived in a particular channel (group and msg id)
UPDATED_MESSAGE = 0x04, /// existing message has been updated in a particular channel
RECEIVED_PUBLISH_KEY = 0x05, /// publish key for this channel has been received
SUBSCRIBE_STATUS_CHANGED = 0x06, /// subscription for channel mChannelGroupId changed.
READ_STATUS_CHANGED = 0x07, /// existing message has been read or set to unread
/** Result for the given group id available for the given turtle request id
* @deprecated kept for retrocompatibility with old search system new code
* should use @see DISTANT_SEARCH_RESULT instead */
RECEIVED_TURTLE_SEARCH_RESULT = 0x08,
STATISTICS_CHANGED = 0x09, /// stats (nb of supplier friends, how many msgs they have etc) has changed
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
NEW_COMMENT = 0x0b, /// new comment arrived/published. mChannelThreadId gives the ID of the commented message
NEW_VOTE = 0x0c, /// new vote arrived/published. mChannelThreadId gives the ID of the votes message comment
DELETED_CHANNEL = 0x0d, /// channel was deleted by auto-cleaning system
DELETED_POST = 0x0e, /// Post deleted (usually by cleaning)
DISTANT_SEARCH_RESULT = 0x0f /// Distant search result received
};
struct RsGxsChannelEvent: RsEvent
{
RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
RsGxsChannelEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::UNKNOWN) {}
RsChannelEventCode mChannelEventCode;
RsGxsGroupId mChannelGroupId;
RsGxsMessageId mChannelMsgId;
RsGxsMessageId mChannelThreadId;
RsGxsMessageId mChannelThreadId;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override
@ -143,13 +157,14 @@ struct RsGxsChannelEvent: RsEvent
}
};
// This event is used to factor multiple search results notifications in a single event.
struct RsGxsChannelSearchResultEvent: RsEvent
/** This event is used to factor multiple search results notifications in a
* single event.*/
struct RS_DEPRECATED_FOR(RsGxsChannelDistantSearchResultEvent)
RsGxsChannelSearchResultEvent: RsEvent
{
RsGxsChannelSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {}
mChannelEventCode(RsChannelEventCode::RECEIVED_TURTLE_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsMap;
@ -164,6 +179,29 @@ struct RsGxsChannelSearchResultEvent: RsEvent
}
};
/** This event is fired once distant search results are received */
struct RsGxsChannelDistantSearchResultEvent: RsEvent
{
RsGxsChannelDistantSearchResultEvent():
RsEvent(RsEventType::GXS_CHANNELS),
mChannelEventCode(RsChannelEventCode::DISTANT_SEARCH_RESULT) {}
RsChannelEventCode mChannelEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mChannelEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
{
public:
@ -386,11 +424,15 @@ public:
* @brief Get channel content summaries
* @jsonapi{development}
* @param[in] channelId id of the channel of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries
* @return false if something failed, true otherwhise
*/
virtual bool getContentSummaries( const RsGxsGroupId& channelId,
std::vector<RsMsgMetaData>& summaries ) = 0;
virtual std::error_condition getContentSummaries(
const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/**
* @brief Toggle post read status. Blocking API.
@ -422,22 +464,23 @@ public:
virtual bool subscribeToChannel( const RsGxsGroupId& channelId,
bool subscribe ) = 0;
/**
* \brief Retrieve statistics about the channel service
/**
* @brief Retrieve statistics about the channel service
* @jsonapi{development}
* \param[out] stat Statistics structure
* \return
*/
virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0;
* @param[out] stat storage for statistics
* @return true on success false otherwise
*/
virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0;
/**
* \brief Retrieve statistics about the given channel
/**
* @brief Retrieve statistics about the given channel
* @jsonapi{development}
* \param[in] channelId Id of the channel group
* \param[out] stat Statistics structure
* \return
*/
virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0;
* @param[in] channelId Id of the channel group
* @param[out] stat storage for statistics
* @return true on success false otherwise
*/
virtual bool getChannelStatistics(
const RsGxsGroupId& channelId, GxsGroupStatistic& stat ) =0;
/// default base URL used for channels links @see exportChannelLink
static const std::string DEFAULT_CHANNEL_BASE_URL;
@ -496,14 +539,18 @@ public:
std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
/**
* @brief Search the turtle reachable network for matching channels
* @brief Search the whole reachable network for matching channels and
* contents
* @jsonapi{development}
* An @see RsGxsChannelSearchResultEvent is emitted when matching channels
* arrives from the network
* An @see RsGxsChannelSearchResultEvent is emitted when matching results
* arrives from the network
* @param[in] matchString string to search into the channels
* @return search id
* @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/
virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0;
virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/**
* @brief Retrieve available search results
@ -533,16 +580,18 @@ public:
* @param[out] distantGroup storage for group data
* @return false on error, true otherwise
*/
virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
virtual bool getDistantSearchResultGroupData(
const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0;
/**
* @brief getDistantSearchStatus
* Returns the status of ongoing search: unknown (probably not even searched), known as a search result,
* data request ongoing and data available
*/
virtual DistantSearchGroupStatus getDistantSearchStatus(const RsGxsGroupId& group_id) =0;
/**
* @brief Get the status of ongoing search
* @return unknown (probably not even searched), known as a search result,
* data request ongoing and data available
*/
virtual DistantSearchGroupStatus getDistantSearchStatus(
const RsGxsGroupId& group_id ) =0;
/**
/**
* @brief Clear accumulated search results
* @jsonapi{development}
* @param[in] reqId search id

62
libretroshare/src/retroshare/rsgxsiface.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*!
* This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSummary : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{
RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSearchResults : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default;
};
/*!
* Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes.

2
libretroshare/src/retroshare/rsgxsifacetypes.h
View File

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {}
virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

65
libretroshare/src/rsitems/rsgxschannelitems.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_GXS_CHANNEL_ITEMS_H
#define RS_GXS_CHANNEL_ITEMS_H
#pragma once
#include <map>
@ -30,14 +31,23 @@
#include "serialiser/rstlvfileitem.h"
#include "serialiser/rstlvimage.h"
#include "serialiser/rsserializable.h"
#include "retroshare/rsgxschannels.h"
#include "serialiser/rsserializer.h"
#include "util/rsdir.h"
enum class RsGxsChannelItems : uint8_t
{
GROUP_ITEM = 0x02,
POST_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsChannelItems)
const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03;
class RsGxsChannelGroupItem : public RsGxsGrpItem
@ -79,6 +89,47 @@ public:
RsTlvImage mThumbnail;
};
struct RsGxsChannelsSearchRequest : RsSerializable
{
RsGxsChannelsSearchRequest() : mType(RsGxsChannelItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsChannelsSearchRequest() override = default;
};
struct RsGxsChannelsSearchReply : RsSerializable
{
RsGxsChannelsSearchReply() : mType(RsGxsChannelItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsChannelItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsChannelsSearchReply() override = default;
};
class RsGxsChannelSerialiser : public RsGxsCommentSerialiser
{
@ -89,5 +140,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
};
#endif /* RS_GXS_CHANNEL_ITEMS_H */

5
libretroshare/src/rsitems/rsitem.h
View File

@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0;
{
std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
<< "new serialization classes, but not derived! Class is "
<< typeid(*this).name() << std::endl;
RS_ERR( "called by an item using new serialization system without "
"overriding Class is: ", typeid(*this).name() );
print_stacktrace();
}

14
libretroshare/src/rsitems/rsserviceids.h
View File

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t
{
NONE = 0, /// To detect non-initialized reads
/// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012,
MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028,
BANLIST = 0x0101,
STATUS = 0x0102,
NXS = 0x0200,
@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216,
CHANNELS = 0x0217,
GXSCIRCLE = 0x0218,
/// not gxs, but used with identities.
REPUTATION = 0x0219,
GXS_RECOGN = 0x0220,
@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB,

74
libretroshare/src/services/p3gxschannels.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -22,6 +23,8 @@
*******************************************************************************/
#pragma once
#include <map>
#include <string>
#include "retroshare/rsgxschannels.h"
#include "services/p3gxscommon.h"
@ -30,9 +33,7 @@
#include "util/rsmemory.h"
#include "util/rsdebug.h"
#include "util/rstickevent.h"
#include <map>
#include <string>
#include "deep_search/channelsindex.hpp"
// This class is only a helper to parse the channel group service string.
@ -56,6 +57,11 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels,
public:
p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes,
RsGixs* gixs );
/// @see RsGxsChannels
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
virtual RsServiceInfo getServiceInfo() override;
virtual void service_tick() override;
@ -69,7 +75,7 @@ protected:
virtual bool loadList(std::list<RsItem *>& loadList) override; // @see p3Config::loadList(std::list<RsItem *>&)
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &results) override;
virtual bool clearDistantSearchResults(TurtleRequestId req) override;
virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override;
@ -112,24 +118,6 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled)
virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override;
virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override;
#ifdef TO_REMOVE
/// @see RsGxsChannels::turtleSearchRequest
virtual bool turtleSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::turtleChannelRequest
virtual bool turtleChannelRequest(
const RsGxsGroupId& channelId,
const std::function<void (const RsGxsChannelGroup& result)>& multiCallback,
rstime_t maxWait = 300 ) override;
/// @see RsGxsChannels::localSearchRequest
virtual bool localSearchRequest(const std::string& matchString,
const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
rstime_t maxWait = 30 ) override;
#endif
/**
* Receive results from turtle search @see RsGenExchange @see RsNxsObserver
* @see RsGxsNetService::receiveTurtleSearchResults
@ -215,9 +203,10 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
const std::set<RsGxsMessageId> &contentIds,
std::vector<RsGxsComment> &comments) override;
/// Implementation of @see RsGxsChannels::getContentSummaries
bool getContentSummaries(
/// @see RsGxsChannels
std::error_condition getContentSummaries(
const RsGxsGroupId& channelId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override;
/// Implementation of @see RsGxsChannels::getChannelStatistics
@ -297,6 +286,17 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
virtual bool shareChannelKeys(
const RsGxsGroupId& channelId, const std::set<RsPeerId>& peers ) override;
#ifdef RS_DEEP_CHANNEL_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// Implementation of @see RsGxsChannels::createChannel
RS_DEPRECATED_FOR(createChannelV2)
bool createChannel(RsGxsChannelGroup& channel) override;
@ -313,7 +313,6 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override;
RS_DEPRECATED_FOR(createVoteV2)
bool createVote(RsGxsVote& vote) override;
protected:
// Overloaded from GxsTokenQueue for Request callbacks.
virtual void handleResponse(uint32_t token, uint32_t req_type
@ -329,7 +328,6 @@ static uint32_t channelsAuthenPolicy();
void request_SpecificSubscribedGroups(const std::list<RsGxsGroupId> &groups);
void load_SubscribedGroups(const uint32_t &token);
void request_SpecificUnprocessedPosts(std::list<std::pair<RsGxsGroupId, RsGxsMessageId> > &ids);
void request_GroupUnprocessedPosts(const std::list<RsGxsGroupId> &grouplist);
void load_unprocessedPosts(uint32_t token);
@ -390,26 +388,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
rstime_t mLastDistantSearchNotificationTS;
std::map<TurtleRequestId,std::set<RsGxsGroupId> > mSearchResultsToNotify;
#ifdef TO_REMOVE
/** Store search callbacks with timeout*/
std::map<
TurtleRequestId,
std::pair<
std::function<void (const RsGxsGroupSummary&)>,
std::chrono::system_clock::time_point >
> mSearchCallbacksMap;
RsMutex mSearchCallbacksMapMutex;
/** Store distant channels requests callbacks with timeout*/
std::map<
TurtleRequestId,
std::pair<
std::function<void (const RsGxsChannelGroup&)>,
std::chrono::system_clock::time_point >
> mDistantChannelsCallbacksMap;
RsMutex mDistantChannelsCallbacksMapMutex;
/// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap
void cleanTimedOutCallbacks();
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex mDeepIndex;
#endif
};

9
libretroshare/src/util/rsdebug.cc
View File

@ -3,9 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -36,6 +36,9 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
" not available in category: " + categoryName;
}
std::error_condition rs_errno_to_condition(int errno_code)
{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

10
libretroshare/src/util/rsdebug.h
View File

@ -2,8 +2,8 @@
* RetroShare debugging utilities *
* *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
* the message around */
std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
/** Convert C errno codes to modern C++11 std::error_condition, this is quite
* useful to use toghether with C functions used around the code like `malloc`,
* `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
*/
std::error_condition rs_errno_to_condition(int errno_code);
template <RsLoggerCategories CATEGORY>
struct t_RsLogger : std::ostringstream

70
libretroshare/src/util/rsmemory.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -25,8 +26,10 @@
#include <cstdlib>
#include <iostream>
#include <memory>
#include <system_error>
#include "util/stacktrace.h"
#include "util/rsdebug.h"
/**
* @brief Shorthand macro to declare optional functions output parameters
@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
* @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
template<typename T> using rs_owner_ptr = T*;
void *rs_malloc(size_t size) ;
/// 1Gb should be enough for everything!
static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
/** Comfortable templated safer malloc, just use it specifing the type of the
* pointer to be returned without need of ugly casting the returned pointer
* `uint8_t* ptr = rs_malloc<uint8_t>(40);`
* @param[in] size number of bytes to allocate
* @param[out] ec optional storage for error details. Value is meaningful only
* whem nullptr is returned.
* @return nullptr on error, pointer to the allocated chuck of memory on success
*/
template<typename T = void> rs_owner_ptr<T> rs_malloc(
size_t size,
rs_view_ptr<std::error_condition> ec = nullptr )
{
if(size == 0)
{
if(!ec)
{
RS_ERR("A chunk of size 0 was requested");
print_stacktrace();
exit(static_cast<int>(std::errc::invalid_argument));
}
*ec = std::errc::invalid_argument;
return nullptr;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
if(!ec)
{
RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
" was requested" );
exit(static_cast<int>(std::errc::argument_out_of_domain));
}
*ec = std::errc::argument_out_of_domain;
return nullptr;
}
void* mem = malloc(size);
if(!mem)
{
if(!ec)
{
RS_ERR( "Allocation failed for a chunk of ", size,
" bytes with: ", errno);
print_stacktrace();
exit(errno);
}
*ec = rs_errno_to_condition(errno);
return nullptr;
}
return static_cast<rs_owner_ptr<T>>(mem);
}
/** @deprecated use std::unique_ptr instead
// This is a scope guard to release the memory block when going of of the current scope.
@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
//
// } // mem gets freed automatically
*/
class RsTemporaryMemory
class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
{
public:
explicit RsTemporaryMemory(size_t s)

4
retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp
View File

@ -411,7 +411,9 @@ void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id)
TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string)
{
return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ;
TurtleRequestId searchId;
rsGxsChannels->distantSearchRequest(search_string.toStdString(), searchId);
return searchId;
}
bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)