Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-25 15:39:27 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2301 from G10h4ck/forums_deep_indexing

Browse Source 
Implement deep indexing and search for forums
This commit is contained in:
G10h4ck 2021-10-28 11:17:51 +02:00 committed by GitHub
commit dc1d5de0db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 2272 additions and 929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
libretroshare/src/deep_search/commonutils.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,54 +18,49 @@
* *
*******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
#ifndef XAPIAN_AT_LEAST
/// Added in Xapian 1.4.2.
#define XAPIAN_AT_LEAST(A,B,C) \
(XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
try
{
#if XAPIAN_AT_LEAST(1,3,2)
std::unique_ptr<Xapian::Database> dbPtr(
new Xapian::Database(path, flags) );
#else
std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
if(flags)
{
RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
"library version: ", XAPIAN_VERSION, " < 1.3.2" );
}
#endif
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
catch(Xapian::DatabaseOpeningError& e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
catch(Xapian::DatabaseLockError&)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
return date;
}
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(rstime_t(50), maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
// strip also CSS inside <style></style>
oSize = retVal.size();
auto styleTagBegin(retVal.find("<style"));
if(styleTagBegin < oSize)
{
auto styleEnd(retVal.find("</style>", styleTagBegin));
if(styleEnd < oSize)
retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
}
std::string::size_type oPos;
std::string::size_type cPos;
int itCount = 0;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
// Avoid infinite loop with crafty input
if(itCount > 1000)
{
RS_WARN( "Breaking stripping loop due to max allowed iterations ",
"rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
break;
}
++itCount;
}
return retVal;
}
}

34
libretroshare/src/deep_search/commonutils.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h>
#include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
}

83
libretroshare/src/deep_search/filesindex.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,47 +18,47 @@
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
if(db)
{
oldDoc = db.get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
if( pIt != db->postlist_end(idTerm) )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
oldDoc = db->get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
}
db.reset(); // Release DB read lock ASAP
}
Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
RS_DBG3(hash);
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
mWriteQueue.push([hash](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + hash.toStdString()); });
db->delete_document("Q" + hash.toStdString());
return true;
return std::error_condition();
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
std::error_condition DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}

23
libretroshare/src/deep_search/filesindex.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
@ -29,6 +26,9 @@
#include <map>
#include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
std::error_condition indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

208
libretroshare/src/deep_search/forumsindex.cpp Normal file
View File

@ -0,0 +1,208 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/forumsindex.hpp"
#include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "retroshare/rsgxsforums.h"
#include "util/rsdebuglevel4.h"
std::error_condition DeepForumsIndex::search(
const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
{
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepForumsSearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
{
RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
forumIndexId.setQueryKV(
RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
return forumIndexId.toString();
}
/*static*/ std::string DeepForumsIndex::postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
return postIndexId.toString();
}
std::error_condition DeepForumsIndex::indexForumGroup(
const RsGxsForumGroup& forum )
{
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
termgenerator.index_text(
DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
termgenerator.index_text(forum.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(forum.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(forum.mDescription);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink);
/* Store some fields for display purposes. Retrieved later to provide the
* matching snippet on search */
doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
/* We use the identifier to ensure each object ends up in the database only
* once no matter how many times we run the indexer.
* "Q" prefix is a Xapian convention for unique id term. */
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumFromIndex(
const RsGxsGroupId& grpId )
{
mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + forumIndexId(grpId)); });
return std::error_condition();
}
std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
{
RS_DBG4(post);
const auto& groupId = post.mMeta.mGroupId;
const auto& msgId = post.mMeta.mMsgId;
if(groupId.isNull() || msgId.isNull())
{
RS_ERR("Got post with invalid id ", post);
print_stacktrace();
return std::errc::invalid_argument;
}
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// Avoid indexing RetroShare-gui HTML tags
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
termgenerator.index_text(cleanMsg, 1, "XD" );
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(groupId, msgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

81
libretroshare/src/deep_search/forumsindex.hpp Normal file
View File

@ -0,0 +1,81 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <system_error>
#include <vector>
#include <xapian.h>
#include "util/rstime.h"
#include "retroshare/rsgxsforums.h"
#include "retroshare/rsevents.h"
#include "deep_search/commonutils.hpp"
struct DeepForumsSearchResult
{
std::string mUrl;
double mWeight;
std::string mSnippet;
};
struct DeepForumsIndex
{
explicit DeepForumsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
std::error_condition search( const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results,
uint32_t maxResults = 100 );
std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
std::error_condition indexForumPost(const RsGxsForumMsg& post);
std::error_condition removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
static std::string dbDefaultPath();
private:
static std::string forumIndexId(const RsGxsGroupId& grpId);
static std::string postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
};

6
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -549,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX

4
libretroshare/src/ft/ftserver.cc
View File

@ -5,7 +5,7 @@
* *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -2037,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{
RsFileSearchResultItem resIt;

57
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60)
{
mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
}
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
msgDeleted.push_back(note.msgIds);
}
for(const auto& msgreq:msgDeleted)
for(const auto& msgit:msgreq)
for(const auto& msg:msgit.second)
mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
/* Three nested for looks like a performance bomb, but as Cyril says here
* https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
* this should actually not explode at all because it is just one message at
* time that get notified */
for(const auto& msd : mMsgDeletePublish)
for(auto& msgMap : msd.mMsgs)
for(auto& msgId : msgMap.second)
mNotifications.push_back(
new RsGxsMsgDeletedChange(msgMap.first, msgId) );
mMsgDeletePublish.clear();
}

140
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
/* always ignore case */
return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{
@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{
std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_CHANNEL_INDEX
#ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important
* to test that the group meta is both non null and actually corresponds to the group id we seek. */
* mDataStore may in some situations allocate an empty group meta
* data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId);
@ -5332,18 +5330,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId);
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
return;
}
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
RS_WARN("Received search results for unknown request: ", req);
return;
}
RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32];
@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId);
}
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
if(!mGxsNetTunnel)
{
free(searchData);
return std::errc::function_not_supported;
}
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos )
{
group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,12 +5477,11 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif
return !group_infos.empty();
return !group_infos.empty();
}
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)

62
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGXSNETSERVICE_H
#define RSGXSNETSERVICE_H
#pragma once
#include <list>
#include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
/// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -629,5 +665,3 @@ private:
bool mUseMetaCache;
};
#endif // RSGXSNETSERVICE_H

607
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
mLastDump = time(NULL) + (RSRandom::random_u32()%20);
/* adds some variance in order to avoid doing all this tasks at once across
* services */
auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
}
//===========================================================================================================================================//
// Transport Items //
//===========================================================================================================================================//
//============================================================================//
// Transport Items //
//============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem
struct RsGxsNetTunnelItem: RsItem
{
public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{
// no priority. All items are encapsulated into generic Turtle items anyway.
}
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
};
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
};
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{
public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
}
};
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{
public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer
{
public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
return NULL ;
RS_ERR( "received item with wrong service ID ", service);
print_stacktrace();
return nullptr;
}
switch(item_subtype)
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
return new RsGxsNetTunnelVirtualPeerItem;
case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
return new RsGxsNetTunnelKeepAliveItem;
case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
return new RsGxsNetTunnelRandomBiasItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
return NULL ;
RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return nullptr;
}
}
};
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ;
}
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ;
uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL)
return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
uint32_t size = tSerializer.size(&searchItem);
uint8_t* buf = rs_malloc<uint8_t>(size);
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ;
RsNetworkExchangeService *service = nullptr;
requestId = mTurtle->turtleSearch(buf, size, this);
if(!requestId) return std::errc::result_out_of_range;
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service ;
search_result_item.group_infos = results ;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
return true ;
}
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
search_result_item.service = substring_gr->service ;
search_result_item.encrypted_group_data = encrypted_group_data ;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
return true ;
}
}
delete item;
return false ;
return std::error_condition();
}
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl;
bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
RS_DBG3("");
if(result_gs != NULL)
RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
#ifdef DEBUG_RSGXSNETTUNNEL
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
delete item;
return ;
RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
print_stacktrace();
return false;
}
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
if(result_gd != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{
if(!search_result_data)
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
"results!" );
print_stacktrace();
break;
}
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
result_gd->encrypted_group_data = NULL ; // prevents deletion
delete item;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
return ;
}
tSerializer.serialise(
&search_result_item, search_result_data,
&search_result_data_size );
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
return false;
}
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RS_DBG3(request_id);
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
auto castFailedWarn = [](const uint8_t subtype)
{
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
return it->second;
};
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{
groups = mGroups ;
virtual_peers = mVirtualPeers ;

62
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once
#include <map>
#include <system_error>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService:
@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
public RsGxsDistSync
{
public:
RsGxsNetTunnelService() ;
virtual ~RsGxsNetTunnelService() ;
RsGxsNetTunnelService();
~RsGxsNetTunnelService() override;
/*!
* \brief registerSearchableService
@ -181,24 +184,38 @@ public:
*/
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
/*!
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
void connectToTurtleRouter(p3turtle *tr) override;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/** Gxs services (channels, forums...) are supposed to use this to request
* searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
///@see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* search_request_data,
uint32_t search_request_data_len,
unsigned char*& search_result_data,
uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread
///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected:
// interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

10
libretroshare/src/gxs/rsgxsnotify.h
View File

@ -97,3 +97,13 @@ private:
bool mMetaChange;
};
struct RsGxsMsgDeletedChange : RsGxsNotify
{
RsGxsMsgDeletedChange(
const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
RsGxsNotify(gid), messageId(msgId) {}
NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
const RsGxsMessageId messageId;
};

145
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
, RsGixs* gixs )
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true;
}
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
bool RsGxsSinglePassIntegrityCheck::check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
{
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
}
else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(nxsMsg)
nxsMsgS.insert(nxsMsg->msgId);
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
msgsToDel[grpId].insert(msgId);
}
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete msg;

36
libretroshare/src/gxs/rsgxsutil.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType&,
RsGixs *gixs);
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
RsGixs* gixs );
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
bool isDone();
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
static bool check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
};
class GroupUpdate

62
libretroshare/src/gxs/rsnxs.h
View File

@ -3,8 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker *
* Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGNP_H
#define RSGNP_H
#pragma once
#include <set>
#include <string>
#include "util/rstime.h"
#include <stdlib.h>
#include <list>
#include <map>
#include "util/rstime.h"
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService
{
public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() = default;
virtual uint16_t serviceType() const =0;
/*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*!
* \brief turtleGroupRequest
@ -115,13 +129,17 @@ public:
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*!
* \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
@ -306,5 +326,3 @@ public:
}
}
};
#endif // RSGNP_H

89
libretroshare/src/util/rsmemory.cc → libretroshare/src/gxs/rsnxsobserver.cpp
View File

@ -1,53 +1,36 @@
/*******************************************************************************
* libretroshare/src/util: rsmemory.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "util/rsmemory.h"
void *rs_malloc(size_t size)
{
static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything!
if(size == 0)
{
std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
void *mem = malloc(size) ;
if(mem == NULL)
{
std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
return mem ;
}
/*******************************************************************************
* RetroShare General eXchange System *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "gxs/rsnxsobserver.h"
const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
{
switch(static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return std::errc::operation_not_supported;
default:
return std::error_condition(ev, *this);
}
}

108
libretroshare/src/gxs/rsnxsobserver.h
View File

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher *
* Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSOBSERVER_H
#define RSNXSOBSERVER_H
#pragma once
#include <set>
#include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId ;
typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver
{
public:
RsNxsObserver() {}
public:
/*!
@ -56,6 +99,46 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data storage for a pointer to search
* result reply data or nullptr if no mathing results where found
* @param[out] resultSize storage for results data size or 0 if no matching
* results where found
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
/* Avoid unused paramethers warning this way so doxygen can still parse
* paramethers documentation */
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*!
* @param grpId group id
*/
@ -70,6 +153,7 @@ public:
* @param grpId group id
*/
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H
RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

12
libretroshare/src/libretroshare.pro
View File

@ -633,7 +633,6 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \
util/rsnet.cc \
util/rsnet_ss.cc \
@ -712,7 +711,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc
gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# gxs tunnels
HEADERS += gxstunnel/p3gxstunnel.h \
@ -936,6 +936,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp

5
libretroshare/src/retroshare/rsgxscircles.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
{
UNKNOWN = 0, /// Used to detect uninizialized values.
PUBLIC = 1, /// Public distribution, based on GxsIds
PUBLIC = 1, /// Public distribution
EXTERNAL = 2, /// Restricted to an external circle, based on GxsIds
NODES_GROUP = 3, /// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

86
libretroshare/src/retroshare/rsgxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -115,9 +115,13 @@ enum class RsForumEventCode: uint8_t
READ_STATUS_CHANGED = 0x06, /// msg was read or marked unread
STATISTICS_CHANGED = 0x07, /// suppliers and how many messages they have changed
MODERATOR_LIST_CHANGED = 0x08, /// forum moderation list has changed.
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
DELETED_POST = 0x0d, /// Post deleted (usually by cleaning)
/// Distant search result received
DISTANT_SEARCH_RESULT = 0x0e
};
struct RsGxsForumEvent: RsEvent
@ -129,8 +133,8 @@ struct RsGxsForumEvent: RsEvent
RsForumEventCode mForumEventCode;
RsGxsGroupId mForumGroupId;
RsGxsMessageId mForumMsgId;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
///* @see RsEvent @see RsSerializable
void serial_process(
@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mForumGroupId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mModeratorsAdded);
RS_SERIAL_PROCESS(mModeratorsRemoved);
}
@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
~RsGxsForumEvent() override;
};
/** This event is fired once distant search results are received */
struct RsGxsForumsDistantSearchEvent: RsEvent
{
RsGxsForumsDistantSearchEvent():
RsEvent(RsEventType::GXS_FORUMS),
mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
RsForumEventCode mForumEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsForums: public RsGxsIfaceHelper
{
public:
@ -385,6 +411,50 @@ public:
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) = 0;
/**
* @brief Get forum content summaries
* @jsonapi{development}
* @param[in] forumId id of the forum of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries
* @return success or error details if something failed
*/
virtual std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/**
* @brief Search the whole reachable network for matching forums and
* posts
* @jsonapi{development}
* An @see RsGxsForumsDistantSearchEvent is emitted when matching results
* arrives from the network
* @param[in] matchString string to search into the forum and posts
* @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/
virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/**
* @brief Search the local index for matching forums and posts
* @jsonapi{development}
* @param[in] matchString string to search into the index
* @param[out] searchResults storage for searchr esults
* @return success or error details
*/
virtual std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) = 0;
////////////////////////////////////////////////////////////////////////////
/* Following functions are deprecated and should not be considered a stable
* to use API */
/**
* @brief Create forum. Blocking API.
* @jsonapi{development}

62
libretroshare/src/retroshare/rsgxsiface.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*!
* This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSummary : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{
RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSearchResults : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default;
};
/*!
* Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes.

2
libretroshare/src/retroshare/rsgxsifacetypes.h
View File

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {}
virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

62
libretroshare/src/rsitems/rsgxsforumitems.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_GXS_FORUM_ITEMS_H
#define RS_GXS_FORUM_ITEMS_H
#pragma once
#include <map>
@ -31,7 +32,18 @@
#include "retroshare/rsgxsforums.h"
enum class RsGxsForumsItems : uint8_t
{
GROUP_ITEM = 0x02,
MESSAGE_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
class RsGxsForumGroupItem : public RsGxsGrpItem
@ -61,6 +73,48 @@ public:
RsGxsForumMsg mMsg;
};
struct RsGxsForumsSearchRequest : RsSerializable
{
RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsForumsSearchRequest() override = default;
};
struct RsGxsForumsSearchReply : RsSerializable
{
RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsForumsSearchReply() override = default;
};
class RsGxsForumSerialiser : public RsServiceSerializer
{
public:
@ -69,5 +123,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
};
#endif /* RS_GXS_FORUM_ITEMS_H */

5
libretroshare/src/rsitems/rsitem.h
View File

@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0;
{
std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
<< "new serialization classes, but not derived! Class is "
<< typeid(*this).name() << std::endl;
RS_ERR( "called by an item using new serialization system without "
"overriding Class is: ", typeid(*this).name() );
print_stacktrace();
}

14
libretroshare/src/rsitems/rsserviceids.h
View File

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t
{
NONE = 0, /// To detect non-initialized reads
/// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012,
MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028,
BANLIST = 0x0101,
STATUS = 0x0102,
NXS = 0x0200,
@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216,
CHANNELS = 0x0217,
GXSCIRCLE = 0x0218,
/// not gxs, but used with identities.
REPUTATION = 0x0219,
GXS_RECOGN = 0x0220,
@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB,

34
libretroshare/src/rsserver/rsinit.cc
View File

@ -2,7 +2,8 @@
* libretroshare/src/retroshare: rsinit.cc *
* *
* Copyright (C) 2004-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -1336,22 +1337,26 @@ int RsServer::StartupRetroShare()
mWiki->setNetworkExchangeService(wiki_ns) ;
#endif
/**** Forum GXS service ****/
/************************* Forum GXS service ******************************/
RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
RsGeneralDataService* gxsforums_ds = new RsDataService(
currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
nullptr, rsInitConfig->gxs_passwd );
p3GxsForums* mGxsForums = new p3GxsForums(
gxsforums_ds, nullptr, mGxsIdService );
p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
#ifdef RS_DEEP_FORUMS_INDEX
gxsForumsTunnelService = mGxsNetTunnel;
#endif
// create GXS photo service
RsGxsNetService* gxsforums_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
mGxsForums, mGxsForums->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
RsGxsNetService* gxsforums_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
mGxsForums->setNetworkExchangeService(gxsforums_ns);
mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
/**** Channel GXS service ****/
@ -1598,7 +1603,10 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
// Turtle search for GXS services
mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
mGxsNetTunnel->registerSearchableService(gxschannels_ns);
#ifdef RS_DEEP_FORUMS_INDEX
mGxsNetTunnel->registerSearchableService(gxsforums_ns);
#endif
/**************************************************************************/

686
libretroshare/src/services/p3gxsforums.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
mGenActive(false), mGenCount(0),
mKnownForumsMutex("GXS forums known forums timestamp cache")
#ifdef RS_DEEP_FORUMS_INDEX
, mDeepIndex(DeepForumsIndex::dbDefaultPath())
#endif
{
// Test Data disabled in Repo.
//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@ -190,223 +194,254 @@ RsSerialiser* p3GxsForums::setupSerialiser()
return rss;
}
void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
{
#ifdef GXSFORUMS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
#endif
RS_DBG2(changes.size(), " changes to notify");
std::vector<RsGxsNotify *>::iterator it;
for(it = changes.begin(); it != changes.end(); ++it)
for(RsGxsNotify* gxsChange: changes)
{
RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
// Let the compiler delete the change for us
std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
if (msgChange)
switch(gxsChange->getType())
{
if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
if (rsEvents)
case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
case RsGxsNotify::TYPE_PUBLISHED:
{
auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
if(msgChange) /* Message received */
{
uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(msgSubtype))
{
case RsGxsForumsItems::MESSAGE_ITEM:
{
auto newForumMessageItem =
dynamic_cast<RsGxsForumMsgItem*>(
msgChange->mNewMsgItem );
if(!newForumMessageItem)
{
RS_ERR("Received message change with mNewMsgItem type "
"mismatching or null");
print_stacktrace();
return;
}
#ifdef RS_DEEP_FORUMS_INDEX
RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
tmpPost.mMeta = newForumMessageItem->meta;
mDeepIndex.indexForumPost(tmpPost);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumMsgId = msgChange->mMsgId;
ev->mForumGroupId = msgChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
rsEvents->postEvent(ev);
break;
}
#ifdef NOT_USED_YET
if (!msgChange->metaChange())
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
std::cerr << std::endl;
#endif
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
std::cerr << std::endl;
#endif
bool enabled = false;
if (autoDownloadEnabled(mit->first, enabled) && enabled)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
std::cerr << std::endl;
#endif
/* problem is most of these will be comments and votes,
* should make it occasional - every 5mins / 10minutes TODO */
unprocessedGroups.push_back(mit->first);
}
default:
RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
break;
}
}
#endif
auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
if(groupChange) /* Group received */
{
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = ( mKnownForums.find(gxsChange->mGroupId)
== mKnownForums.end() );
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
rsEvents->postEvent(ev);
}
#ifdef RS_DEEP_FORUMS_INDEX
uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(itemType))
{
case RsGxsForumsItems::GROUP_ITEM:
{
auto newForumGroupItem =
static_cast<RsGxsForumGroupItem*>(
groupChange->mNewGroupItem );
mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
break;
}
default:
RS_WARN("Got unknown gxs group subtype: ", itemType);
break;
}
#endif // def RS_DEEP_FORUMS_INDEX
}
break;
}
else
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
if (rsEvents)
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_MESSAGE_DELETED:
{
auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
if(!delChange)
{
RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
if (grpChange)
{
switch (grpChange->getType())
{
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_PUBLISHED:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
/* group received */
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
rsEvents->postEvent(ev);
}
else
RsInfo() << __PRETTY_FUNCTION__
<< " Not notifying already known forum "
<< grpChange->mGroupId << std::endl;
}
break;
case RsGxsNotify::TYPE_GROUP_DELETED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_UPDATED:
{
// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
if(old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
{
RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
delete grpChange;
continue;
}
// First of all, we check if there is a difference between the old and new list of moderators
std::list<RsGxsId> added_mods, removed_mods;
for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
added_mods.push_back(gxs_id);
for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
removed_mods.push_back(gxs_id);
if(!added_mods.empty() || !removed_mods.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mModeratorsAdded = added_mods;
ev->mModeratorsRemoved = removed_mods;
ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED;
rsEvents->postEvent(ev);
}
// check the list of pinned posts
std::list<RsGxsMessageId> added_pins, removed_pins;
for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
added_pins.push_back(msg_id);
for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
removed_pins.push_back(msg_id);
if(!added_pins.empty() || !removed_pins.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
rsEvents->postEvent(ev);
}
if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription
|| old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName
|| old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
|| old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId
)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
rsEvents->postEvent(ev);
}
}
break;
default:
RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
break;
}
}
RS_ERR( "Got mismatching notification type: ",
gxsChange->getType() );
print_stacktrace();
break;
}
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumPostFromIndex(
delChange->mGroupId, delChange->messageId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumEventCode = RsForumEventCode::DELETED_POST;
ev->mForumGroupId = delChange->mGroupId;
ev->mForumMsgId = delChange->messageId;
break;
}
case RsGxsNotify::TYPE_GROUP_DELETED:
{
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
break;
}
case RsGxsNotify::TYPE_UPDATED:
{
/* Happens when the group data has changed. In this case we need to
* analyse the old and new group in order to detect possible
* notifications for clients */
auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
RsGxsForumGroupItem* old_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem* new_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
{
RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
"mNewGroup not of type RsGxsForumGroupItem or NULL. "
"This is inconsistent!");
print_stacktrace();
break;
}
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
#endif
/* First of all, we check if there is a difference between the old
* and new list of moderators */
std::list<RsGxsId> added_mods, removed_mods;
for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== old_forum_grp_item->mGroup.mAdminList.ids.end() )
added_mods.push_back(gxs_id);
for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== new_forum_grp_item->mGroup.mAdminList.ids.end() )
removed_mods.push_back(gxs_id);
if(!added_mods.empty() || !removed_mods.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mModeratorsAdded = added_mods;
ev->mModeratorsRemoved = removed_mods;
ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED;
rsEvents->postEvent(ev);
}
// check the list of pinned posts
std::list<RsGxsMessageId> added_pins, removed_pins;
for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
added_pins.push_back(msg_id);
for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
removed_pins.push_back(msg_id);
if(!added_pins.empty() || !removed_pins.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
rsEvents->postEvent(ev);
}
if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription
|| old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName
|| old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
|| old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId )
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
rsEvents->postEvent(ev);
}
break;
}
/* shouldn't need to worry about groups - as they need to be subscribed to */
delete *it;
default:
RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
" Currently not handled." );
break;
}
}
}
@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
| GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
/* This flag have always this value even for circle restricted forums due to
* how GXS distribute/verify groups */
forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
forum.mMeta.mCircleId.clear();
@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
id == mMeta.mAuthorId;
}
std::error_condition p3GxsForums::getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries )
{
uint32_t token;
RsTokReqOptions opts;
opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
GxsMsgReq msgReq;
msgReq[forumId] = contentIds;
if(!requestMsgInfo(token, opts, msgReq))
{
RS_ERR("requestMsgInfo failed");
return std::errc::invalid_argument;
}
switch(waitToken(token, std::chrono::seconds(5)))
{
case RsTokenService::COMPLETE:
{
GxsMsgMetaMap metaMap;
if(!RsGenExchange::getMsgMeta(token, metaMap))
return std::errc::result_out_of_range;
summaries = metaMap[forumId];
return std::error_condition();
}
case RsTokenService::PARTIAL: // [[fallthrough]];
case RsTokenService::PENDING:
return std::errc::timed_out;
default:
return std::errc::not_supported;
}
}
#ifdef RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG1("");
RsGxsForumsSearchRequest request;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(request);
}
if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(request.mType) );
return std::errc::bad_message;
}
RsGxsForumsSearchReply reply;
auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
if(mErr || reply.mResults.empty()) return mErr;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(reply);
resultSize = ctx.mOffset;
}
resultData = rs_malloc<uint8_t>(resultSize);
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(reply);
return std::error_condition();
}
std::error_condition p3GxsForums::distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId )
{
RsGxsForumsSearchRequest request;
request.mQuery = matchString;
uint32_t requestSize;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(request);
requestSize = ctx.mOffset;
}
std::error_condition ec;
auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
if(!requestData) return ec;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(request);
}
return netService()->distantSearchRequest(
requestData, requestSize,
static_cast<RsServiceType>(serviceType()), searchId );
}
std::error_condition p3GxsForums::localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults )
{ return prepareSearchResults(matchString, false, searchResults); }
std::error_condition p3GxsForums::prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults )
{
std::vector<DeepForumsSearchResult> results;
auto mErr = mDeepIndex.search(matchString, results);
if(mErr) return mErr;
searchResults.clear();
for(auto uRes: results)
{
RsUrl resUrl(uRes.mUrl);
const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
if(!forumIdStr)
{
RS_ERR( "Forum URL retrieved from deep index miss ID. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::address_not_available;
}
std::vector<RsGxsForumGroup> forumsInfo;
RsGxsGroupId forumId(*forumIdStr);
if(forumId.isNull())
{
RS_ERR( "Forum ID retrieved from deep index is invalid. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
forumsInfo.empty() )
{
RS_ERR( "Forum just parsed from deep index link not found. "
"Should never happen! ", forumId, " ", uRes.mUrl );
print_stacktrace();
return std::errc::identifier_removed;
}
RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
// Avoid leaking sensitive information to unkown peers
if( publicOnly &&
( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
RsGxsCircleType::PUBLIC ) ) continue;
RsGxsSearchResult res;
res.mGroupId = forumId;
res.mGroupName = fMeta.mGroupName;
res.mAuthorId = fMeta.mAuthorId;
res.mPublishTs = fMeta.mPublishTs;
res.mSearchContext = uRes.mSnippet;
auto postIdStr =
resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
if(postIdStr)
{
RsGxsMessageId msgId(*postIdStr);
if(msgId.isNull())
{
RS_ERR( "Post just parsed from deep index link is invalid. "
"Should never happen! ", postIdStr, " ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
std::vector<RsMsgMetaData> msgSummaries;
auto errc = getContentSummaries(
forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
if(errc) return errc;
if(msgSummaries.size() != 1)
{
RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
"should never happen!" );
return std::errc::result_out_of_range;
}
RsMsgMetaData& msgMeta(msgSummaries[0]);
res.mMsgId = msgMeta.mMsgId;
res.mMsgName = msgMeta.mMsgName;
res.mAuthorId = msgMeta.mAuthorId;
}
RS_DBG4(res);
searchResults.push_back(res);
}
return std::error_condition();
}
std::error_condition p3GxsForums::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RsGxsForumsSearchReply reply;
{
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(reply);
}
free(resultData);
if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(reply.mType) );
return std::errc::bad_message;
}
auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
event->mSearchId = requestId;
event->mSearchResults = reply.mResults;
rsEvents->postEvent(event);
return std::error_condition();
}
#else // def RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::distantSearchRequest(
const std::string&, TurtleRequestId& )
{ return std::errc::function_not_supported; }
std::error_condition p3GxsForums::localSearch(
const std::string&,
std::vector<RsGxsSearchResult>& )
{ return std::errc::function_not_supported; }
#endif // def RS_DEEP_FORUMS_INDEX
/*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
"retroshare:///forums";
/*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

52
libretroshare/src/services/p3gxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -32,6 +32,10 @@
#include "util/rstickevent.h"
#include "util/rsdebug.h"
#ifdef RS_DEEP_FORUMS_INDEX
#include "deep_search/forumsindex.hpp"
#endif
class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
public RsTickEvent /* only needed for testing - remove after */
@ -142,7 +146,34 @@ public:
/// @see RsGxsForums
std::error_condition setPostKeepForever(
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) override;
bool keepForever ) override;
/// @see RsGxsForums
std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override;
/// @see RsGxsForums
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
/// @see RsGxsForums
std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) override;
#ifdef RS_DEEP_FORUMS_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNxsObserver
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// implementation of rsGxsGorums
///
@ -155,6 +186,17 @@ public:
bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
protected:
#ifdef RS_DEEP_FORUMS_INDEX
/** Internal usage
* @param[in] publicOnly if true is passed only results pertaining to
* publicly shared forums are returned
*/
std::error_condition prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults );
#endif //def RS_DEEP_FORUMS_INDEX
private:
static uint32_t forumsAuthenPolicy();
@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
std::map<RsGxsGroupId,rstime_t> mKnownForums ;
RsMutex mKnownForumsMutex;
#ifdef RS_DEEP_FORUMS_INDEX
DeepForumsIndex mDeepIndex;
#endif
};

73
libretroshare/src/turtle/turtleclientservice.h
View File

@ -19,23 +19,25 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
// This class is the parent class for any service that will use the turtle router to distribute its packets.
// Typical representative clients include:
//
// p3ChatService: opens tunnels to distant peers for chatting
// ftServer: searches and open tunnels to distant sources for file transfer
//
#pragma once
#include <string>
#include <stdlib.h>
#include <serialiser/rsserial.h>
#include <turtle/rsturtleitem.h>
#include "serialiser/rsserial.h"
#include "turtle/rsturtleitem.h"
#include "util/rsdebug.h"
struct RsItem;
class p3turtle ;
/** This class is the parent class for any service that will use the turtle
* router to distribute its packets.
* Typical representative clients include:
* p3ChatService: opens tunnels to distant peers for chatting
* ftServer: searches and open tunnels to distant sources for file
* transfer
*/
class RsTurtleClientService
{
public:
@ -87,30 +89,35 @@ class RsTurtleClientService
std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ;
}
/*!
* \brief receiveSearchRequest
* This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
* result contains the serialised generic result returned by the client.
*
* The turtle router keeps the memory ownership over search_request_data
*
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and forwarded
*
* \return true if the search is successful.
*/
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
uint32_t /*search_request_data_len*/,
unsigned char *& /*search_result_data*/,
uint32_t& /*search_result_data_len*/,
uint32_t& /* max_allows_hits */)
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
return false;
}
/*!
* This method is called by the turtle router to notify the client of a
* search request in the form generic data.
* The returned result contains the serialised generic result returned by the
* client service.
* The turtle router keeps the memory ownership over search_request_data
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and
* forwarded
* \return true if matching results are available, false otherwise.
*/
virtual bool receiveSearchRequest(
unsigned char *search_request_data, uint32_t search_request_data_len,
unsigned char *& search_result_data, uint32_t& search_result_data_len,
uint32_t& max_allows_hits )
{
/* Suppress unused warning this way and not commenting the param names
* so doxygen match documentation against params */
(void) search_request_data; (void) search_request_data_len;
(void) search_result_data; (void) search_result_data_len;
(void) max_allows_hits;
RS_WARN( "Received search request from turtle router, but the client "
"is not handling it!" );
return false;
}
/*!
* \brief receiveSearchResult

9
libretroshare/src/util/rsdebug.cc
View File

@ -3,9 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -36,6 +36,9 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
" not available in category: " + categoryName;
}
std::error_condition rs_errno_to_condition(int errno_code)
{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

10
libretroshare/src/util/rsdebug.h
View File

@ -2,8 +2,8 @@
* RetroShare debugging utilities *
* *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
* the message around */
std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
/** Convert C errno codes to modern C++11 std::error_condition, this is quite
* useful to use toghether with C functions used around the code like `malloc`,
* `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
*/
std::error_condition rs_errno_to_condition(int errno_code);
template <RsLoggerCategories CATEGORY>
struct t_RsLogger : std::ostringstream

70
libretroshare/src/util/rsmemory.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -25,8 +26,10 @@
#include <cstdlib>
#include <iostream>
#include <memory>
#include <system_error>
#include "util/stacktrace.h"
#include "util/rsdebug.h"
/**
* @brief Shorthand macro to declare optional functions output parameters
@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
* @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
template<typename T> using rs_owner_ptr = T*;
void *rs_malloc(size_t size) ;
/// 1Gb should be enough for everything!
static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
/** Comfortable templated safer malloc, just use it specifing the type of the
* pointer to be returned without need of ugly casting the returned pointer
* `uint8_t* ptr = rs_malloc<uint8_t>(40);`
* @param[in] size number of bytes to allocate
* @param[out] ec optional storage for error details. Value is meaningful only
* whem nullptr is returned.
* @return nullptr on error, pointer to the allocated chuck of memory on success
*/
template<typename T = void> rs_owner_ptr<T> rs_malloc(
size_t size,
rs_view_ptr<std::error_condition> ec = nullptr )
{
if(size == 0)
{
if(!ec)
{
RS_ERR("A chunk of size 0 was requested");
print_stacktrace();
exit(static_cast<int>(std::errc::invalid_argument));
}
*ec = std::errc::invalid_argument;
return nullptr;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
if(!ec)
{
RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
" was requested" );
exit(static_cast<int>(std::errc::argument_out_of_domain));
}
*ec = std::errc::argument_out_of_domain;
return nullptr;
}
void* mem = malloc(size);
if(!mem)
{
if(!ec)
{
RS_ERR( "Allocation failed for a chunk of ", size,
" bytes with: ", errno);
print_stacktrace();
exit(errno);
}
*ec = rs_errno_to_condition(errno);
return nullptr;
}
return static_cast<rs_owner_ptr<T>>(mem);
}
/** @deprecated use std::unique_ptr instead
// This is a scope guard to release the memory block when going of of the current scope.
@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
//
// } // mem gets freed automatically
*/
class RsTemporaryMemory
class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
{
public:
explicit RsTemporaryMemory(size_t s)

5
retroshare-gui/src/gui/NewsFeed.cpp
View File

@ -270,7 +270,10 @@ void NewsFeed::handleForumEvent(std::shared_ptr<const RsEvent> event)
case RsForumEventCode::UPDATED_MESSAGE:
case RsForumEventCode::NEW_MESSAGE:
addFeedItem(new GxsForumMsgItem(this, NEWSFEED_NEW_FORUM, pe->mForumGroupId, pe->mForumMsgId, false, true));
addFeedItem(new GxsForumMsgItem(
this, NEWSFEED_NEW_FORUM,
pe->mForumGroupId, pe->mForumMsgId,
false, true ));
break;
default: break;

6
retroshare.pri
View File

@ -141,6 +141,11 @@ rs_macos10.15:CONFIG -= rs_macos10.11
CONFIG *= no_rs_jsonapi
rs_jsonapi:CONFIG -= no_rs_jsonapi
# To enable forums indexing append the following assignation to qmake command
# line "CONFIG+=rs_deep_forums_index"
CONFIG *= no_rs_deep_forums_index
rs_deep_forums_index:CONFIG -= no_rs_deep_forums_index
# To enable channel indexing append the following assignation to qmake command
# line "CONFIG+=rs_deep_channels_index"
CONFIG *= no_rs_deep_channels_index
@ -561,6 +566,7 @@ rs_webui {
DEFINES *= RS_WEBUI
}
rs_deep_forums_index:DEFINES *= RS_DEEP_FORUMS_INDEX
rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX
rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX