Merge pull request #2301 from G10h4ck/forums_deep_indexing

Implement deep indexing and search for forums

libretroshare/src/deep_search/commonutils.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,54 +18,49 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <algorithm>
+#include <thread>
+
 #include "deep_search/commonutils.hpp"
 #include "util/stacktrace.h"
-#include "util/rsdebug.h"
+#include "util/rsthreads.h"
+#include "util/rsdebuglevel0.h"
+
+#ifndef XAPIAN_AT_LEAST
+/// Added in Xapian 1.4.2.
+#define XAPIAN_AT_LEAST(A,B,C) \
+	(XAPIAN_MAJOR_VERSION > (A) || \
+	(XAPIAN_MAJOR_VERSION == (A) && \
+	(XAPIAN_MINOR_VERSION > (B) || \
+	(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
+#endif
 
 namespace DeepSearch
 {
-
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags, int blockSize )
-{
-	try
-	{
-		std::unique_ptr<Xapian::WritableDatabase> dbPtr(
-		        new Xapian::WritableDatabase(path, flags, blockSize) );
-		return dbPtr;
-	}
-	catch(Xapian::DatabaseLockError)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-	catch(...)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
-		        << "deleting it might help without causing any harm: "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-
-	return nullptr;
-}
-
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags )
 {
 	try
 	{
+#if XAPIAN_AT_LEAST(1,3,2)
 		std::unique_ptr<Xapian::Database> dbPtr(
 		        new Xapian::Database(path, flags) );
+#else
+		std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
+		if(flags)
+		{
+			RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
+			         "library version: ", XAPIAN_VERSION, " < 1.3.2" );
+		}
+#endif
 		return dbPtr;
 	}
-	catch(Xapian::DatabaseOpeningError e)
+	catch(Xapian::DatabaseOpeningError& e)
 	{
 		RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
 		          << ", probably nothing has been indexed yet." << std::endl;
 	}
-	catch(Xapian::DatabaseLockError)
+	catch(Xapian::DatabaseLockError&)
 	{
 		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
 		        << path << std::endl;
@@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
 	return date;
 }
 
+StubbornWriteOpQueue::~StubbornWriteOpQueue()
+{
+	auto fErr = flush(0);
+	if(fErr)
+	{
+		RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
+		          " operations irreparably lost ", fErr );
+		print_stacktrace();
+	}
+}
+
+void StubbornWriteOpQueue::push(write_op op)
+{
+	RS_DBG4("");
+
+	{
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		mOpStore.push(op);
+	}
+
+	flush();
+}
+
+std::error_condition StubbornWriteOpQueue::flush(
+        rstime_t acceptDelay, rstime_t callTS )
+{
+	RS_DBG4("");
+
+	{
+		// Return without attempt to open the database if the queue is empty
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		if(mOpStore.empty()) return std::error_condition();
+	}
+
+	std::unique_ptr<Xapian::WritableDatabase> dbPtr;
+	try
+	{
+		dbPtr = std::make_unique<Xapian::WritableDatabase>(
+		            mDbPath, Xapian::DB_CREATE_OR_OPEN );
+	}
+	catch(Xapian::DatabaseLockError)
+	{
+		if(acceptDelay)
+		{
+			rstime_t tNow = time(nullptr);
+			rstime_t maxRemaining = tNow - (callTS + acceptDelay);
+			if(maxRemaining > 0)
+			{
+				std::chrono::milliseconds interval(
+				            std::max(rstime_t(50), maxRemaining*1000/5) );
+				RS_DBG3( "Cannot acquire database write lock, retrying in:",
+				         interval.count(), "ms" );
+				RsThread::async([this, acceptDelay, callTS, interval]()
+				{
+					std::this_thread::sleep_for(interval);
+					flush(acceptDelay, callTS);
+				});
+				return std::error_condition();
+			}
+			else
+			{
+				RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
+				return std::errc::timed_out;
+			}
+		}
+		else return std::errc::resource_unavailable_try_again;
+	}
+	catch(...)
+	{
+		RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
+		print_stacktrace();
+		return std::errc::io_error;
+	}
+
+	std::unique_lock<std::mutex> lock(mQueueMutex);
+	while(!mOpStore.empty())
+	{
+		auto op = mOpStore.front(); mOpStore.pop();
+		op(*dbPtr);
+	}
+	return std::error_condition();
+}
+
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
+{
+	if(rsHtmlDoc.empty()) return rsHtmlDoc;
+
+	const bool isPlainMsg =
+	        rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
+	if(isPlainMsg) return rsHtmlDoc;
+
+	auto oSize = rsHtmlDoc.size();
+	auto bodyTagBegin(rsHtmlDoc.find("<body"));
+	if(bodyTagBegin >= oSize) return rsHtmlDoc;
+
+	auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
+	if(bodyTagEnd >= oSize) return rsHtmlDoc;
+
+	std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
+
+	// strip also CSS inside <style></style>
+	oSize = retVal.size();
+	auto styleTagBegin(retVal.find("<style"));
+	if(styleTagBegin < oSize)
+	{
+		auto styleEnd(retVal.find("</style>", styleTagBegin));
+		if(styleEnd < oSize)
+			retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
+	}
+
+	std::string::size_type oPos;
+	std::string::size_type cPos;
+	int itCount = 0;
+	while((oPos = retVal.find("<")) < retVal.size())
+	{
+		if((cPos = retVal.find(">")) <= retVal.size())
+			retVal.erase(oPos, 1+cPos-oPos);
+		else break;
+
+		// Avoid infinite loop with crafty input
+		if(itCount > 1000)
+		{
+			RS_WARN( "Breaking stripping loop due to max allowed iterations ",
+			         "rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
+			break;
+		}
+		++itCount;
+	}
+
+	return retVal;
+}
+
 }

libretroshare/src/deep_search/commonutils.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -21,6 +21,9 @@
 
 #include <xapian.h>
 #include <memory>
+#include <functional>
+#include <queue>
+#include <mutex>
 
 #include "util/rstime.h"
 
@@ -33,13 +36,34 @@
 
 namespace DeepSearch
 {
-
+typedef std::function<void(Xapian::WritableDatabase&)> write_op;
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags = 0, int blockSize = 0 );
 
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags = 0 );
 
 std::string timetToXapianDate(const rstime_t& time);
 
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
+
+struct StubbornWriteOpQueue
+{
+	explicit StubbornWriteOpQueue(const std::string& dbPath):
+	    mDbPath(dbPath) {}
+
+	~StubbornWriteOpQueue();
+
+	void push(write_op op);
+
+	std::error_condition flush(
+	        rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
+
+private:
+	std::queue<write_op> mOpStore;
+	rstime_t mLastFlush;
+
+	std::mutex mQueueMutex;
+
+	const std::string mDbPath;
+};
+
 }

libretroshare/src/deep_search/filesindex.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,48 +18,48 @@
  *                                                                             *
  *******************************************************************************/
 
-#include "deep_search/filesindex.hpp"
-#include "deep_search/commonutils.hpp"
-#include "util/rsdebug.h"
-#include "retroshare/rsinit.h"
-#include "retroshare/rsversion.h"
 
 #include <utility>
 
+#include "deep_search/filesindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "util/rsdebuglevel1.h"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsversion.h"
+
 /*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
 DeepFilesIndex::indexersRegister = {};
 
-bool DeepFilesIndex::indexFile(
+std::error_condition DeepFilesIndex::indexFile(
         const std::string& path, const std::string& name,
         const RsFileHash& hash )
 {
-	auto dbPtr = DeepSearch::openWritableDatabase(
-	            mDbPath, Xapian::DB_CREATE_OR_OPEN );
-	if(!dbPtr) return false;
-	Xapian::WritableDatabase& db(*dbPtr);
-
 	const std::string hashString = hash.toStdString();
 	const std::string idTerm("Q" + hashString);
 
-	Xapian::Document oldDoc;
+	auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
-	Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
+	if(db)
-	if( pIt != db.postlist_end(idTerm) )
 	{
-		oldDoc = db.get_document(*pIt);
+		Xapian::Document oldDoc;
+		Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
+		if( pIt != db->postlist_end(idTerm) )
+		{
+			oldDoc = db->get_document(*pIt);
 			if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
 			        RS_HUMAN_READABLE_VERSION &&
 			    std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
 			        indexersRegister.size() )
 			{
-			/* Looks like this file has already been indexed by this RetroShare
+				/* Looks like this file has already been indexed by this
-			 * exact version, so we can skip it. If the version was different it
+				 * RetroShare exact version, so we can skip it. If the version
-			 * made sense to reindex it as better indexers might be available
+				 * was different it made sense to reindex it as better indexers
-			 * since last time it was indexed */
+				 * might be available since last time it was indexed */
-			Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
+				RS_DBG3("skipping laready indexed file: ", hash, " ", name);
-			       << hash << " " << name << std::endl;
+				return std::error_condition();
-			return true;
 			}
 		}
+		db.reset(); // Release DB read lock ASAP
+	}
 
 	Xapian::Document doc;
 
@@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
 	doc.add_value(
 	            INDEXERS_COUNT_VALUENO,
 	            std::to_string(indexersRegister.size()) );
-	db.replace_document(idTerm, doc);
 
-	return true;
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); });
+
+	return std::error_condition();
 }
 
-bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
+std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 {
-	Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
+	RS_DBG3(hash);
-	       << hash << std::endl;
 
-	std::unique_ptr<Xapian::WritableDatabase> db =
+	mWriteQueue.push([hash](Xapian::WritableDatabase& db)
-	        DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
+	{ db.delete_document("Q" + hash.toStdString()); });
-	if(!db) return false;
 
-	db->delete_document("Q" + hash.toStdString());
+	return std::error_condition();
-	return true;
 }
 
 /*static*/ std::string DeepFilesIndex::dbDefaultPath()
@@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 /*static*/ bool DeepFilesIndex::registerIndexer(
         int order, const DeepFilesIndex::IndexerFunType& indexerFun )
 {
-	Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
+	RS_DBG1(order);
 
 	indexersRegister.insert(std::make_pair(order, indexerFun));
 	return true;
 }
 
-uint32_t DeepFilesIndex::search(
+std::error_condition DeepFilesIndex::search(
         const std::string& queryStr,
         std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
 {
 	results.clear();
 
 	auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
-	if(!dbPtr) return 0;
+	if(!dbPtr) return std::errc::bad_file_descriptor;
 	Xapian::Database& db(*dbPtr);
 
 	// Set up a QueryParser with a stemmer and suitable prefixes.
@@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
 		results.push_back(s);
 	}
 
-	return static_cast<uint32_t>(results.size());
+	return std::error_condition();
 }
 
 

libretroshare/src/deep_search/filesindex.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org>              *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -19,9 +19,6 @@
  *******************************************************************************/
 #pragma once
 
-#include "retroshare/rstypes.h"
-#include "util/rsdebug.h"
-
 #include <string>
 #include <cstdint>
 #include <vector>
@@ -29,6 +26,9 @@
 #include <map>
 #include <functional>
 
+#include "retroshare/rstypes.h"
+#include "deep_search/commonutils.hpp"
+
 struct DeepFilesSearchResult
 {
 	DeepFilesSearchResult() : mWeight(0) {}
@@ -41,7 +41,8 @@ struct DeepFilesSearchResult
 class DeepFilesIndex
 {
 public:
-	DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
+	explicit DeepFilesIndex(const std::string& dbPath):
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
 
 	/**
 	 * @brief Search indexed files
@@ -49,7 +50,7 @@ public:
 	 * no limits
 	 * @return search results count
 	 */
-	uint32_t search( const std::string& queryStr,
+	std::error_condition search( const std::string& queryStr,
 	                 std::vector<DeepFilesSearchResult>& results,
 	                 uint32_t maxResults = 100 );
 
@@ -57,7 +58,7 @@ public:
 	 * @return false if file could not be indexed because of error or
 	 *	unsupported type, true otherwise.
 	 */
-	bool indexFile(
+	std::error_condition indexFile(
 	        const std::string& path, const std::string& name,
 	        const RsFileHash& hash );
 
@@ -65,7 +66,7 @@ public:
 	 * @brief Remove file entry from database
 	 * @return false on error, true otherwise.
 	 */
-	bool removeFileFromIndex(const RsFileHash& hash);
+	std::error_condition removeFileFromIndex(const RsFileHash& hash);
 
 	static std::string dbDefaultPath();
 
@@ -96,8 +97,8 @@ private:
 
 	const std::string mDbPath;
 
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+
 	/** Storage for indexers function by order */
 	static std::multimap<int, IndexerFunType> indexersRegister;
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/deep_search/forumsindex.cpp

@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "deep_search/forumsindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsgxsforums.h"
+#include "util/rsdebuglevel4.h"
+
+std::error_condition DeepForumsIndex::search(
+        const std::string& queryStr,
+        std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
+{
+	results.clear();
+
+	std::unique_ptr<Xapian::Database> dbPtr(
+	            DeepSearch::openReadOnlyDatabase(mDbPath) );
+	if(!dbPtr) return std::errc::bad_file_descriptor;
+
+	Xapian::Database& db(*dbPtr);
+
+	// Set up a QueryParser with a stemmer and suitable prefixes.
+	Xapian::QueryParser queryparser;
+	//queryparser.set_stemmer(Xapian::Stem("en"));
+	queryparser.set_stemming_strategy(queryparser.STEM_SOME);
+	// Start of prefix configuration.
+	//queryparser.add_prefix("title", "S");
+	//queryparser.add_prefix("description", "XD");
+	// End of prefix configuration.
+
+	// And parse the query.
+	Xapian::Query query = queryparser.parse_query(queryStr);
+
+	// Use an Enquire object on the database to run the query.
+	Xapian::Enquire enquire(db);
+	enquire.set_query(query);
+
+	Xapian::MSet mset = enquire.get_mset(
+	            0, maxResults ? maxResults : db.get_doccount() );
+
+	for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
+	{
+		const Xapian::Document& doc = m.get_document();
+		DeepForumsSearchResult s;
+		s.mUrl = doc.get_value(URL_VALUENO);
+#if XAPIAN_AT_LEAST(1,3,5)
+		s.mSnippet = mset.snippet(doc.get_data());
+#endif // XAPIAN_AT_LEAST(1,3,5)
+		results.push_back(s);
+	}
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
+{
+	RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	forumIndexId.setQueryKV(
+	            RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
+	return forumIndexId.toString();
+}
+
+/*static*/ std::string DeepForumsIndex::postIndexId(
+        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
+{
+	RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
+	return postIndexId.toString();
+}
+
+std::error_condition DeepForumsIndex::indexForumGroup(
+        const RsGxsForumGroup& forum )
+{
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
+	termgenerator.index_text(forum.mDescription, 1, "XD");
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(forum.mMeta.mGroupName);
+	termgenerator.increase_termpos();
+	termgenerator.index_text(forum.mDescription);
+
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	/* Store some fields for display purposes. Retrieved later to provide the
+	 * matching snippet on search */
+	doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
+
+	/* We use the identifier to ensure each object ends up in the database only
+	 * once no matter how many times we run the indexer.
+	 * "Q" prefix is a Xapian convention for unique id term. */
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumFromIndex(
+        const RsGxsGroupId& grpId )
+{
+	mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
+	{ db.delete_document("Q" + forumIndexId(grpId)); });
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
+{
+	RS_DBG4(post);
+
+	const auto& groupId = post.mMeta.mGroupId;
+	const auto& msgId = post.mMeta.mMsgId;
+
+	if(groupId.isNull() || msgId.isNull())
+	{
+		RS_ERR("Got post with invalid id ", post);
+		print_stacktrace();
+		return std::errc::invalid_argument;
+	}
+
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
+
+	// Avoid indexing RetroShare-gui HTML tags
+	const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
+	termgenerator.index_text(cleanMsg, 1, "XD" );
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(post.mMeta.mMsgName);
+
+	termgenerator.increase_termpos();
+	termgenerator.index_text(cleanMsg);
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(postIndexId(groupId, msgId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	// Store some fields for display purposes.
+	doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
+
+	// We use the identifier to ensure each object ends up in the
+	// database only once no matter how many times we run the
+	// indexer.
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumPostFromIndex(
+        RsGxsGroupId grpId, RsGxsMessageId msgId )
+{
+	// "Q" prefix is a Xapian convention for unique id term.
+	std::string idTerm("Q" + postIndexId(grpId, msgId));
+	mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
+	{ db.delete_document(idTerm); } );
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::dbDefaultPath()
+{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

libretroshare/src/deep_search/forumsindex.hpp

@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#pragma once
+
+#include <system_error>
+#include <vector>
+#include <xapian.h>
+
+#include "util/rstime.h"
+#include "retroshare/rsgxsforums.h"
+#include "retroshare/rsevents.h"
+#include "deep_search/commonutils.hpp"
+
+struct DeepForumsSearchResult
+{
+	std::string mUrl;
+	double mWeight;
+	std::string mSnippet;
+};
+
+struct DeepForumsIndex
+{
+	explicit DeepForumsIndex(const std::string& dbPath) :
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
+
+	/**
+	 * @brief Search indexed GXS groups and messages
+	 * @param[in] maxResults maximum number of acceptable search results, 0 for
+	 * no limits
+	 * @return search results count
+	 */
+	std::error_condition search( const std::string& queryStr,
+	                             std::vector<DeepForumsSearchResult>& results,
+	                             uint32_t maxResults = 100 );
+
+	std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
+
+	std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
+
+	std::error_condition indexForumPost(const RsGxsForumMsg& post);
+
+	std::error_condition removeForumPostFromIndex(
+	        RsGxsGroupId grpId, RsGxsMessageId msgId );
+
+	static std::string dbDefaultPath();
+
+private:
+	static std::string forumIndexId(const RsGxsGroupId& grpId);
+	static std::string postIndexId(
+	        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
+
+	enum : Xapian::valueno
+	{
+		/// Used to store retroshare url of indexed documents
+		URL_VALUENO,
+
+		/// @see Xapian::BAD_VALUENO
+		BAD_VALUENO = Xapian::BAD_VALUENO
+	};
+
+	const std::string mDbPath;
+
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+};

libretroshare/src/file_sharing/directory_storage.cc

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
- * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -549,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
 	        fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
 	{
 		DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-		ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
+		ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
 	}
 #endif // def RS_DEEP_FILES_INDEX
 

libretroshare/src/ft/ftserver.cc

@@ -5,7 +5,7 @@
  *                                                                             *
  * Copyright (C) 2008  Robert Fernie <retroshare@lunamutt.com>                 *
  * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -2037,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
 
 	std::vector<DeepFilesSearchResult> dRes;
 	DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-	if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
+	if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
 	{
 		RsFileSearchResultItem resIt;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
   VALIDATE_MAX_WAITING_TIME(60)
 {
     mDataAccess = new RsGxsDataAccess(gds);
-
-    // Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-    // This code is only called because it of deep indexing in channels. But loading
-    // the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
-    // eats memory, as many tests have shown. Not because of leaks, but because new threads are
-    // apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
-    // that RS will use them again.
-    //
-    //    * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
-    //      service (e.g. channels here) should update the index when a new message is received. The question to how old messages
-    //	    are processed is open. I believe that they shouldn't. New users will progressively process them.
-    //
-    //    * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
-    //      unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
-    //      It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
-    //      doesn't counterbalance such a low risk.
-    //
-    if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
-    {
-        std::vector<RsGxsGroupId> grpsToDel;
-        GxsMsgReq msgsToDel;
-
-        RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
-                                             this, *mSerialiser,
-                                             grpsToDel,msgsToDel);
-
-        for(auto& grpId: grpsToDel)
-        {
-            uint32_t token2=0;
-            deleteGroup(token2,grpId);
-        }
-
-        if(!msgsToDel.empty())
-        {
-            uint32_t token1=0;
-            deleteMsgs(token1,msgsToDel);
-        }
-    }
-#endif
 }
 
 void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
             msgDeleted.push_back(note.msgIds);
     }
 
-    for(const auto& msgreq:msgDeleted)
+	/* Three nested for looks like a performance bomb, but as Cyril says here
-        for(const auto& msgit:msgreq)
+	 * https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
-            for(const auto& msg:msgit.second)
+	 * this should actually not explode at all because it is just one message at
-                mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
+	 * time that get notified */
+	for(const auto& msd : mMsgDeletePublish)
+		for(auto& msgMap : msd.mMsgs)
+			for(auto& msgId : msgMap.second)
+				mNotifications.push_back(
+				            new RsGxsMsgDeletedChange(msgMap.first, msgId) );
 
 	mMsgDeletePublish.clear();
 }

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
     return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
 }
 
-#ifndef RS_DEEP_CHANNEL_INDEX
 static bool termSearch(const std::string& src, const std::string& substring)
 {
 	/* always ignore case */
-	return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
+	return src.end() != std::search(
+	            src.begin(), src.end(), substring.begin(), substring.end(),
+	            RsRegularExpression::CompareCharIC() );
 }
-#endif // ndef RS_DEEP_CHANNEL_INDEX
+
 
 bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
@@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
     return true ;
 }
 
-void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
 {
 	std::set<RsGxsGroupId> groupsToNotifyResults;
 
@@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 
 		for (const RsGxsGroupSummary& gps : group_infos)
 		{
-#ifndef RS_DEEP_CHANNEL_INDEX
+#ifdef TO_REMOVE
+			/* Because of deep search is enabled search results may bring more
+			 * info then we already have also about post that are indexed by
+			 * xapian, so we don't apply this filter anymore. */
+
 			/* Only keep groups that are not locally known, and groups that are
 			 * not already in the mDistantSearchResults structure.
-			 * mDataStore may in some situations allocate an empty group meta data, so it's important
+			 * mDataStore may in some situations allocate an empty group meta
-			 * to test that the group meta is both non null and actually corresponds to the group id we seek. */
+			 * data, so it's important to test that the group meta is both non
-
+			 * null and actually corresponds to the group id we seek. */
 			auto& meta(grpMeta[gps.mGroupId]);
+			if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
+#endif // def TO_REMOVE
 
-			if(meta != nullptr && meta->mGroupId == gps.mGroupId)
-                continue;
-
-#ifdef NXS_NET_DEBUG_9
-            std::cerr << "  group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
-#endif
-
-#else // ndef RS_DEEP_CHANNEL_INDEX
-			/* When deep search is enabled search results may bring more info
-			 * then we already have also about post that are indexed by xapian,
-			 * so we don't apply this filter in this case. */
-#endif
 			const RsGxsGroupId& grpId(gps.mGroupId);
 
 			groupsToNotifyResults.insert(grpId);
@@ -5332,16 +5330,17 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 		mObserver->receiveDistantSearchResults(req, grpId);
 }
 
-void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req,
+        const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
 {
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
 #endif
 	auto it = mSearchRequests.find(req);
-
 	if(mSearchRequests.end() == it)
 	{
-        std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
+		RS_WARN("Received search results for unknown request: ", req);
 		return;
 	}
     RsGxsGroupId grpId = it->second;
@@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
 	mObserver->receiveDistantSearchResults(req, grpId);
 }
 
+std::error_condition RsGxsNetService::distantSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
+{
+	if(!mGxsNetTunnel)
+	{
+		free(searchData);
+		return std::errc::function_not_supported;
+	}
+
+	return mGxsNetTunnel->turtleSearchRequest(
+	            searchData, dataSize, serviceType, requestId );
+}
+
+std::error_condition RsGxsNetService::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG("");
+	return mObserver->handleDistantSearchRequest(
+	            requestData, requestSize, resultData, resultSize );
+}
+
+std::error_condition RsGxsNetService::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	return mObserver->receiveDistantSearchResult(
+	            requestId, resultData, resultSize );
+}
+
 bool RsGxsNetService::search( const std::string& substring,
                               std::list<RsGxsGroupSummary>& group_infos )
 {
 	group_infos.clear();
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-
-#warning TODO: filter deep index search result to non circle-restricted groups.
-//    /!\
-//    /!\   These results should be filtered to only return results coming from a non restricted group!
-//    /!\
-
-	std::vector<DeepChannelsSearchResult> results;
-	DeepChannelsIndex::search(substring, results);
-
-	for(auto dsr : results)
-	{
-		RsUrl rUrl(dsr.mUrl);
-		const auto& uQ(rUrl.query());
-		auto rit = uQ.find("id");
-		if(rit != rUrl.query().end())
-		{
-			RsGroupNetworkStats stats;
-			RsGxsGroupId grpId(rit->second);
-			if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
-			{
-				RsGxsGroupSummary s;
-
-				s.mGroupId = grpId;
-
-				if((rit = uQ.find("name")) != uQ.end())
-					s.mGroupName = rit->second;
-				if((rit = uQ.find("signFlags")) != uQ.end())
-					s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
-				if((rit = uQ.find("publishTs")) != uQ.end())
-					s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
-				if((rit = uQ.find("authorId")) != uQ.end())
-					s.mAuthorId  = RsGxsId(rit->second);
-
-				s.mSearchContext = dsr.mSnippet;
-
-				s.mNumberOfMessages = stats.mMaxVisibleCount;
-				s.mLastMessageTs    = stats.mLastGroupModificationTS;
-				s.mPopularity       = stats.mSuppliers;
-
-				group_infos.push_back(s);
-			}
-		}
-	}
-#else // RS_DEEP_CHANNEL_INDEX
 	RsGxsGrpMetaTemporaryMap grpMetaMap;
 	{
 		RS_STACK_MUTEX(mNxsMutex) ;
@@ -5492,7 +5477,6 @@ bool RsGxsNetService::search( const std::string& substring,
 
 			group_infos.push_back(s);
 		}
-#endif // RS_DEEP_CHANNEL_INDEX
 
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << "  performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;

libretroshare/src/gxs/rsgxsnetservice.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSGXSNETSERVICE_H
+#pragma once
-#define RSGXSNETSERVICE_H
 
 #include <list>
 #include <queue>
@@ -130,18 +131,53 @@ public:
 
     virtual bool msgAutoSync() const override { return mAllowMsgSync; }
     virtual bool grpAutoSync() const override { return mGrpAutoSync; }
-    /*!
-     * \brief Search methods.
-     * 			These four methods are used to request distant search and receive the results.
-     * \param group_id
-     */
-    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
-    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
 
-    virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
+	/// @see RsNetworkExchangeService
+	std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/** Request group data via turtle search
+	 * @param group_id */
+	TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
+
+	/**
+	 * @brief Search for matching groups names over turtle search.
+	 * @deprecated this method is kept mostly for retrocompatibility with older
+	 * peers, newly implemented search functions should instead be based on the
+	 * service generic search.
+	 * @see RsNetworkExchangeService
+	 */
+	RS_DEPRECATED_FOR(distantSearchRequest)
+	TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
+
+	/** @see RsNetworkExchangeService
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(receiveDistantSearchResult)
+	void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        const uint8_t* encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) override;
+
+	/**
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(handleRemoteSearchRequest)
+	virtual bool search( const std::string& substring,
+	                     std::list<RsGxsGroupSummary>& group_infos) override;
     virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
     virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
-    virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
 
     virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@@ -629,5 +665,3 @@ private:
 
     bool mUseMetaCache;
 };
-
-#endif // RSGXSNETSERVICE_H

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -27,6 +29,8 @@
 #include "gxs/rsnxs.h"
 #include "rsgxsnettunnel.h"
 
+/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
+
 //#define DEBUG_RSGXSNETTUNNEL 1
 
 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@@ -36,42 +40,93 @@
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA    =   1;
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH  = 100;
 
-RsGxsDistSync *rsGxsDistSync = NULL;
 
 RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
 {
 	mRandomBias.clear();
 
-	mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20);	// adds some variance in order to avoid doing all this tasks at once across services
+	/* adds some variance in order to avoid doing all this tasks at once across
-	mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
+	 * services */
-	mLastDump = time(NULL) + (RSRandom::random_u32()%20);
+	auto now = time(nullptr);
+	mLastKeepAlive = now + (RsRandom::random_u32()%20);
+	mLastAutoWash = now + (RsRandom::random_u32()%20);
+	mLastDump = now + (RsRandom::random_u32()%20);
 }
 
-//===========================================================================================================================================//
+//============================================================================//
 //                         Transport Items                                    //
-//===========================================================================================================================================//
+//============================================================================//
 
-const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
+enum class RsGxsNetTunnelItemSubtypes : uint8_t
+{
+	VIRTUAL_PEER             = 0x01,
 
+	KEEP_ALIVE               = 0x02,
+
+	RANDOM_BIAS              = 0x03,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
+	SEARCH_SUBSTRING         = 0x04,
+
+	SEARCH_GROUP_REQUEST     = 0x05,
+
+	// SEARCH_GROUP_SUMMARY  = 0x06, removed
+
+	SEARCH_GROUP_DATA        = 0x07,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
+	SEARCH_GROUP_SUMMARY     = 0x08,
+
+	/** Generic search request generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REQUEST   = 0x09,
+
+	/** Generic search reply generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REPLY     = 0x0a
+};
+
+RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
+constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
+        static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
+
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                = 0x01 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  = 0x02 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 = 0x03 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     = 0x04 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
-// const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    = 0x07 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
+// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
 
-class RsGxsNetTunnelItem: public RsItem
+struct RsGxsNetTunnelItem: RsItem
 {
 public:
-	explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
+	explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            static_cast<uint8_t>(subtype) )
+	{
+		/* no priority. All items are encapsulated into generic Turtle items
+		 * anyway. */
+	}
+
+	virtual ~RsGxsNetTunnelItem() = default;
+	virtual void clear() {}
+
+	RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
+	explicit RsGxsNetTunnelItem(uint8_t item_subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            item_subtype )
 	{
 		// no priority. All items are encapsulated into generic Turtle items anyway.
 	}
-
-	virtual ~RsGxsNetTunnelItem() {}
-	virtual void clear() {}
 };
 
 class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@@ -113,7 +168,86 @@ public:
 	Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
 };
 
-class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
+struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReqItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(RsServiceType::NONE), mSearchData(nullptr),
+	    mSearchDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mSearchData; /// Service search request data
+	uint32_t mSearchDataSize; /// Search data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mSearchData);
+		mSearchData = nullptr;
+		mSearchDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReqItem() override { clear(); }
+};
+
+struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReplyItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(RsServiceType::NONE), mReplyData(nullptr),
+	    mReplyDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mReplyData; /// Service search reply data
+	uint32_t mReplyDataSize; /// Search reply data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mReplyData);
+		mReplyData = nullptr;
+		mReplyDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
+};
+
+class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
+RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
 {
 public:
     explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@@ -164,6 +298,7 @@ public:
 		RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
 	}
 };
+
 class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
 {
 public:
@@ -193,28 +328,41 @@ public:
 class RsGxsNetTunnelSerializer: public RsServiceSerializer
 {
 public:
-	RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
+	RsGxsNetTunnelSerializer():
+	    RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
 
 	virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
 	{
 		if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
 		{
-			GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
+			RS_ERR( "received item with wrong service ID ", service);
-			return NULL ;
+			print_stacktrace();
+			return nullptr;
 		}
 
-		switch(item_subtype)
+		switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
 		{
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                : return new RsGxsNetTunnelVirtualPeerItem ;
+		case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  : return new RsGxsNetTunnelKeepAliveItem ;
+			return new RsGxsNetTunnelVirtualPeerItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 : return new RsGxsNetTunnelRandomBiasItem ;
+		case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     : return new RsGxsNetTunnelTurtleSearchSubstringItem;
+			return new RsGxsNetTunnelKeepAliveItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+			return new RsGxsNetTunnelRandomBiasItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
+			return new RsGxsNetTunnelTurtleSearchSubstringItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+			return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
+			return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+			return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+			return new RsGxsServiceTurtleSearchReqItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+			return new RsGxsServiceTurtleSearchReplyItem;
 		default:
-			GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
+			RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
-			return NULL ;
+			return nullptr;
 		}
 	}
 };
@@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
+TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
+        const std::string& match_string,
+        RsNetworkExchangeService* client_service )
 {
     GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
 
@@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
 	search_item.service = client_service->serviceType() ;
 
     uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
-    unsigned char *mem = (unsigned char*)rs_malloc(size) ;
+	uint8_t* mem = rs_malloc<uint8_t>(size);
 
     if(mem == NULL)
         return 0 ;
@@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
+std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
 {
-	GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
+	if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
+		return std::errc::invalid_argument;
 
-	RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
+	RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
+	searchItem.mSearchDataSize = dataSize;
+	searchItem.mSearchData = searchData;
 
-    RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
+	RsGxsNetTunnelSerializer tSerializer;
 
-    if(substring_sr != NULL)
+	uint32_t size = tSerializer.size(&searchItem);
-    {
+	uint8_t* buf = rs_malloc<uint8_t>(size);
-		GXS_NET_TUNNEL_DEBUG() << "  : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
 
-        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
+	tSerializer.serialise(&searchItem, buf, &size);
 
-        std::list<RsGxsGroupSummary> results ;
+	requestId = mTurtle->turtleSearch(buf, size, this);
-        RsNetworkExchangeService *service = nullptr;
+	if(!requestId) return std::errc::result_out_of_range;
 
-        {
+	return std::error_condition();
+}
+
+rs_view_ptr<RsNetworkExchangeService>
+RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
+{
 	RS_STACK_MUTEX(mGxsNetTunnelMtx);
+	auto it = mSearchableServices.find(serviceType);
+	if( it != mSearchableServices.end()) return it->second;
+	return nullptr;
+}
 
-            auto it = mSearchableServices.find(substring_sr->service) ;
+bool RsGxsNetTunnelService::receiveSearchRequest(
+        uint8_t* search_request_data, uint32_t search_request_data_len,
+        uint8_t*& search_result_data, uint32_t& search_result_data_size,
+        uint32_t& max_allowed_hits )
+{
+	/* Must return true only if there are matching results available, false in
+	 * all other cases. @see RsTurleClientService */
 
-            if(it != mSearchableServices.end())
+	RS_DBG3("");
-                service = it->second;
+
+	RsGxsNetTunnelSerializer tSerializer;
+
+	std::unique_ptr<RsItem> item;
+	item.reset(tSerializer.deserialise(
+	               search_request_data, &search_request_data_len ));
+
+	if(!item)
+	{
+		RS_ERR( "Deserialization failed: ",
+		        search_request_data, search_request_data_len, item.get() );
+		print_stacktrace();
+		return false;
 	}
 
-        if(service != nullptr && service->search(substring_sr->substring_match,results))
+	switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
 	{
-			RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
+	case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
-
+	{
-			GXS_NET_TUNNEL_DEBUG() << "  : " << results.size() << " result found. Sending back." << std::endl;
+		if(!search_result_data)
-
+		{
-            search_result_item.service = substring_sr->service ;
+			RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
-            search_result_item.group_infos = results ;
+			        "results!" );
-
+			print_stacktrace();
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			break;
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
-
-            delete item;
-
-			if(search_result_data == NULL)
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
-
-			return true ;
-        }
 		}
 
-    RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
+		auto substring_sr =
-
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
-    if(substring_gr != NULL)
+		if(!substring_sr)
 		{
-		RS_STACK_MUTEX(mGxsNetTunnelMtx);
+			RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
-		auto it = mSearchableServices.find(substring_gr->service) ;
+			         item->PacketSubType(), " but casting failed!");
+			break;
+		}
 
-        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
-
+		std::list<RsGxsGroupSummary> results;
-        unsigned char *encrypted_group_data = NULL ;
+		auto tService = retrievieSearchableServiceLocking(substring_sr->service);
-        uint32_t encrypted_group_data_len = 0 ;
+		if(tService && tService->search(substring_sr->substring_match, results))
-
-        if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
 		{
-			RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
+			RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
+			search_result_item.service = substring_sr->service;
+			search_result_item.group_infos = results;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_item.service = substring_gr->service ;
+			tSerializer.serialise(
-            search_result_item.encrypted_group_data = encrypted_group_data ;
+			            &search_result_item, search_result_data,
+			            &search_result_data_size );
+
+			return true;
+		}
+
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+	{
+		auto *substring_gr =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
+
+		if(!substring_gr)
+		{
+			RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
+			         item->PacketSubType(), " but casting failed!" );
+			break;
+		}
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
+		uint8_t* encrypted_group_data = nullptr;
+		uint32_t encrypted_group_data_len = 0;
+
+		auto tService = retrievieSearchableServiceLocking(substring_gr->service);
+		if(tService && tService->search(
+		            substring_gr->hashed_group_id,
+		            encrypted_group_data, encrypted_group_data_len ))
+		{
+			RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
+			search_result_item.service = substring_gr->service;
+			search_result_item.encrypted_group_data = encrypted_group_data;
 			search_result_item.encrypted_group_data_len = encrypted_group_data_len;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			tSerializer.serialise(
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
+			            &search_result_item,
-
+			            search_result_data, &search_result_data_size );
-			if(search_result_data == NULL)
+			return true;
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
-
-            delete item;
-			return true ;
 		}
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+	{
+		RS_DBG3("SERVICE_SEARCH_REQUEST");
+
+		auto searchItem =
+		        static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
+
+		uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
+		auto sService = retrievieSearchableServiceLocking(sType);
+		if(!sService)
+		{
+			RS_WARN("Got search request for non searchable service: ", sType);
+			break;
 		}
 
-    delete item;
+		RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
-    return false ;
+
+		auto errc = sService->handleDistantSearchRequest(
+		            searchItem->mSearchData, searchItem->mSearchDataSize,
+		            replyItem.mReplyData, replyItem.mReplyDataSize );
+		if(errc)
+		{
+			// Some error has been reported by the searchable service
+			RS_WARN("searchable service: ", sType , " reported: ", errc);
+			break;
+		}
+
+		if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
+		    (replyItem.mReplyData && !replyItem.mReplyDataSize) )
+		{
+			// Inconsistent behaviour from searcheable service
+			RS_ERR( "searchable service: ", sType , " silently failed handling "
+			        "inconsistent result mReplyData: ", replyItem.mReplyData,
+			        " mReplyDataSize: ", replyItem.mReplyDataSize );
+			break;
+		}
+
+		/* Our node have 0 matching results */
+		if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
+			break;
+
+		search_result_data_size = tSerializer.size(&replyItem);
+		search_result_data = rs_malloc<uint8_t>(search_result_data_size);
+
+		tSerializer.serialise(
+		            &replyItem, search_result_data, &search_result_data_size );
+
+		return true;
+	}
+	default:
+		RS_WARN("Got unknown item type: ", item->PacketSubType());
+		break;
+	}
+
+	return false;
 }
 
-void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
+void RsGxsNetTunnelService::receiveSearchResult(
+        TurtleSearchRequestId request_id,
+        uint8_t* search_result_data, uint32_t search_result_data_len )
 {
-    RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
+	RS_DBG3(request_id);
 
-	GXS_NET_TUNNEL_DEBUG() << "  : received search result for search request " << std::hex << request_id << "" << std::endl;
+	std::unique_ptr<RsItem> item;
+	item.reset(RsGxsNetTunnelSerializer().deserialise(
+	               search_result_data,&search_result_data_len ));
 
-    RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
+	auto castFailedWarn = [](const uint8_t subtype)
-
-    if(result_gs != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
+		RS_WARN( "Got item with subtype: ", subtype,
-
+		         " but cast failed!" );
-#ifdef DEBUG_RSGXSNETTUNNEL
+	};
-		for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
-			std::cerr << "   group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
-#endif
-
-		auto it = mSearchableServices.find(result_gs->service) ;
 
+	auto searchableServiceGet = [this](const auto pservice)
+	{
+		auto service = static_cast<uint16_t>(pservice);
+		auto it = mSearchableServices.find(service);
 		if(it == mSearchableServices.end())
 		{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
+			RS_WARN( "got item for service ", service,
-            delete item;
+			         " which is not in the searchable services list." );
-			return ;
+			return static_cast<RsNetworkExchangeService*>(nullptr);
 		}
 
-		it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
+		return it->second;
+	};
 
-		delete item;
+	const auto tSubtype = item->PacketSubType();
-		return ;
+	switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
-	}
-
-    RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
-
-    if(result_gd != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
-
-		auto it = mSearchableServices.find(result_gd->service) ;
-
-		if(it == mSearchableServices.end())
 	{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
+		auto result_gs =
-            delete item;
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
-			return ;
+		            item.get() );
+
+		if(!result_gs)
+		{
+			castFailedWarn(tSubtype);
+			break;
 		}
 
-		it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
+		RS_DBG2( " got result is of type group summary result for service ",
+		         result_gs->service );
 
-        result_gd->encrypted_group_data = NULL ; // prevents deletion
+		auto service = searchableServiceGet(result_gs->service);
-		delete item;
+		if(service)
+			service->receiveTurtleSearchResults(
+			            request_id, result_gs->group_infos );
+		return;
+	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+	{
+		auto result_gd =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
 
-		return ;
+		if(!result_gd)
+		{
+			castFailedWarn(tSubtype);
+			break;
 		}
 
-	GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
+		RS_DBG2("got group data result for service: ", result_gd->service);
+
+		auto service = searchableServiceGet(result_gd->service);
+		if(service)
+			service->receiveTurtleSearchResults(
+			        request_id,
+			        result_gd->encrypted_group_data,
+			        result_gd->encrypted_group_data_len );
+
+		/* Ensure ownershipt is passed down preventing deletion */
+		result_gd->encrypted_group_data = nullptr;
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+	{
+		auto searchReply =
+		        static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
+
+		auto service = searchableServiceGet(searchReply->mServiceType);
+		if(service)
+			service->receiveDistantSearchResult(
+			        request_id,
+			        searchReply->mReplyData,
+			        searchReply->mReplyDataSize );
+
+		/* Ensure memory ownership is passed down preventing deletion */
+		searchReply->mReplyData = nullptr;
+		break;
+	}
+	default:
+		RS_WARN("got item of unknown type: ", item->PacketSubType());
+		break;
+	}
 }
 
-void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias      ) const
+void RsGxsNetTunnelService::getStatistics(
+        std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
+        std::map<RsGxsNetTunnelVirtualPeerId,
+        RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
+        std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
+        turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
 {
     groups = mGroups ;
     virtual_peers = mVirtualPeers ;

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -23,6 +25,7 @@
 #pragma once
 
 #include <map>
+#include <system_error>
 
 #include "turtle/p3turtle.h"
 #include "retroshare/rsgxsdistsync.h"
@@ -100,7 +103,7 @@
 //        and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
 //
 
-class RsGxsNetTunnelItem ;
+struct RsGxsNetTunnelItem;
 class RsNetworkExchangeService ;
 
 class RsGxsNetTunnelService:
@@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
         public RsGxsDistSync
 {
 public:
-	  RsGxsNetTunnelService() ;
+	RsGxsNetTunnelService();
-	  virtual ~RsGxsNetTunnelService() ;
+	~RsGxsNetTunnelService() override;
 
       /*!
        * \brief registerSearchableService
@@ -182,23 +185,37 @@ public:
 	  void dump() const;
 
 	/*!
-       * \brief connectToTurtleRouter
 	* Should be called after allocating a RsGxsNetTunnelService
 	* \param tr turtle router object
 	*/
-	  virtual void connectToTurtleRouter(p3turtle *tr) ;
+	void connectToTurtleRouter(p3turtle *tr) override;
 
-      TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
+	/** Gxs services (channels, forums...) are supposed to use this to request
-      TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
+	 * searches on distant peers */
+	std::error_condition turtleSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId );
 
-      /*!
+	///@see RsTurtleClientService
-       * \brief receiveSearchRequest
+	bool receiveSearchRequest(
-       * 			See RsTurtleClientService::@
+	        unsigned char* search_request_data,
-       */
+	        uint32_t search_request_data_len,
-	  virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
+	        unsigned char*& search_result_data,
-	  virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
+	        uint32_t& search_result_data_len,
+	        uint32_t& max_allowed_hits ) override;
 
-	void threadTick() override; /// @see RsTickingThread
+	///@see RsTurtleClientService
+	virtual void receiveSearchResult(
+	        TurtleSearchRequestId request_id,
+	        unsigned char* search_result_data,
+	        uint32_t search_result_data_len ) override;
+
+	TurtleRequestId turtleGroupRequest(
+	        const RsGxsGroupId& group_id,
+	        RsNetworkExchangeService* client_service );
+
+	/// @see RsTickingThread
+	void threadTick() override;
 
 	  // Overloads p3Config
 
@@ -213,6 +230,11 @@ public:
             std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
 		    Bias20Bytes& bias) const;
 
+	RS_DEPRECATED
+	TurtleRequestId turtleSearchRequest(
+	        const std::string& match_string,
+	        RsNetworkExchangeService* client_service );
+
 protected:
 	  // interaction with turtle router
 
@@ -233,6 +255,8 @@ private:
 	  void sendKeepAlivePackets() ;
 	  void handleIncoming(RsGxsNetTunnelItem *item) ;
 	  void flush_pending_items();
+	rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
+	        uint16_t serviceType );
 
 	  std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ;	// groups on the client and server side
 

libretroshare/src/gxs/rsgxsnotify.h

@@ -97,3 +97,13 @@ private:
     bool mMetaChange;
 };
 
+struct RsGxsMsgDeletedChange : RsGxsNotify
+{
+	RsGxsMsgDeletedChange(
+	        const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
+	    RsGxsNotify(gid), messageId(msgId) {}
+
+	NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
+
+	const RsGxsMessageId messageId;
+};

libretroshare/src/gxs/rsgxsutil.cc

@@ -3,8 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013  Christopher Evi-Parker                                  *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -29,12 +29,6 @@
 #include "pqi/pqihash.h"
 #include "gxs/rsgixs.h"
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-#	include "deep_search/channelsindex.hpp"
-#	include "services/p3gxschannels.h"
-#	include "rsitems/rsgxschannelitems.h"
-#endif
-
 // The goals of this set of methods is to check GXS messages and groups for consistency, mostly
 // re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
 // happen anyway, but we still conduct these test as an extra safety measure.
@@ -198,8 +192,7 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
 
 RsGxsIntegrityCheck::RsGxsIntegrityCheck(
         RsGeneralDataService* const dataService, RsGenExchange* genex,
-    RsSerialType&
+        RsSerialType&, RsGixs* gixs )
-                            , RsGixs* gixs )
   : mDs(dataService), mGenExchangeClient(genex),
     mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
 
@@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
     return true;
 }
 
-bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
+bool RsGxsSinglePassIntegrityCheck::check(
-#ifdef RS_DEEP_CHANNEL_INDEX
+        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-                                , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
+        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
-#endif
-                                , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
 {
 #ifdef DEBUG_GXSUTIL
     GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
 #endif
-#ifdef RS_DEEP_CHANNEL_INDEX
-    bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
-    std::set<RsGxsGroupId> indexedGroups;
-#endif
 
     // first take out all the groups
     std::map<RsGxsGroupId, RsNxsGrp*> grp;
@@ -393,54 +380,13 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
             }
             else
                 msgIds.erase(msgIds.find(grp->grpId));	// could not get them, so group is removed from list.
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-            // This should be moved to p3gxschannels. It is really not the place for this here!
-
-            if( isGxsChannels
-                    && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
-                    && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
-            {
-                RsGxsGrpMetaData meta;
-                meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
-
-                uint32_t blz = grp->grp.bin_len;
-                RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
-                                                      &blz);
-
-                if( RsGxsChannelGroupItem* cgIt =
-                        dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
-                {
-                    RsGxsChannelGroup cg;
-                    cgIt->toChannelGroup(cg, false);
-                    cg.mMeta = meta;
-
-                    indexedGroups.insert(grp->grpId);
-                    DeepChannelsIndex::indexChannelGroup(cg);
         }
 		else
 		{
-                    std::cerr << __PRETTY_FUNCTION__ << " Group: "
+			RS_WARN( "deleting group ", grp->grpId,
-                              << meta.mGroupId.toStdString() << " "
+			         " with wrong hash or null/corrupted meta data. meta=",
-                              << meta.mGroupName
+			         grp->metaData );
-                              << " doesn't seems a channel, please "
-                              << "report to developers"
-                              << std::endl;
-                    print_stacktrace();
-                }
-
-                delete rIt;
-            }
-#endif // def RS_DEEP_CHANNEL_INDEX
-        }
-        else
-        {
-            std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
 			grpsToDel.push_back(grp->grpId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-            if(isGxsChannels)
-                DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
-#endif // def RS_DEEP_CHANNEL_INDEX
 		}
 
         delete grp;
@@ -471,13 +417,7 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
 		for (auto& msgId:msgIdV)
 			if(nxsMsgS.find(msgId) == nxsMsgS.end())
-            {
 				msgsToDel[grpId].insert(msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
-#endif // def  RS_DEEP_CHANNEL_INDEX
-            }
     }
 
     for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
             if(msg->metaData == NULL || currHash != msg->metaData->mHash)
             {
-                std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
+				RS_WARN( "deleting message ", msg->msgId, " in group ",
+				         msg->grpId,
+				         " with wrong hash or null/corrupted meta data. meta=",
+				         static_cast<void*>(msg->metaData) );
                 msgsToDel[msg->grpId].insert(msg->msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(
-                                msg->grpId, msg->msgId );
-#endif // def RS_DEEP_CHANNEL_INDEX
-            }
-            else if (subscribed_groups.count(msg->metaData->mGroupId))
-            {
-#ifdef RS_DEEP_CHANNEL_INDEX
-                // This should be moved to p3gxschannels. It is really not the place for this here!
-
-                if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
-                {
-                    RsGxsMsgMetaData meta;
-                    meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
-
-                    uint32_t blz = msg->msg.bin_len;
-                    RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
-                                                          &blz);
-
-                    if( RsGxsChannelPostItem* cgIt =
-                            dynamic_cast<RsGxsChannelPostItem*>(rIt) )
-                    {
-                        RsGxsChannelPost cg;
-                        cgIt->toChannelPost(cg, false);
-                        cg.mMeta = meta;
-
-                        DeepChannelsIndex::indexChannelPost(cg);
-                    }
-                    else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
-                    else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
-                    else
-                    {
-                        std::cerr << __PRETTY_FUNCTION__ << " Message: "
-                                  << meta.mMsgId.toStdString()
-                                  << " in group: "
-                                  << meta.mGroupId.toStdString() << " "
-                                  << " doesn't seems a channel post, please "
-                                  << "report to developers"
-                                  << std::endl;
-                        print_stacktrace();
-                    }
-
-                    delete rIt;
-                }
-#endif // def RS_DEEP_CHANNEL_INDEX
             }
 
             delete msg;

libretroshare/src/gxs/rsgxsutil.h

@@ -3,8 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013 Christopher Evi-Parker                                   *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
     enum CheckState { CheckStart, CheckChecking };
 
 public:
-
+	RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
-
+	                     RsGenExchange* genex, RsSerialType&,
-    /*!
+	                     RsGixs* gixs );
-     *
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
-    RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
-                         RsGenExchange *genex, RsSerialType&,
-                         RsGixs *gixs);
 
     static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
     bool isDone();
@@ -201,19 +193,9 @@ private:
 class RsGxsSinglePassIntegrityCheck
 {
 public:
-
+	static bool check(
-    /*!
+	        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-     *
+	        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
-    static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
-#ifdef RS_DEEP_CHANNEL_INDEX
-                      , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
-#endif
-                      , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
 };
 
 class GroupUpdate

libretroshare/src/gxs/rsnxs.h

@@ -3,8 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com>         *
+ * Copyright (C) 2011  Robert Fernie <retroshare.project@gmail.com>            *
- * Copyright 2011-2011 by Christopher Evi-Parker                               *
+ * Copyright (C) 2011  Christopher Evi-Parker                                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -20,17 +22,15 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
+#pragma once
-#ifndef RSGNP_H
-#define RSGNP_H
 
 #include <set>
 #include <string>
-#include "util/rstime.h"
 #include <stdlib.h>
 #include <list>
 #include <map>
 
+#include "util/rstime.h"
 #include "services/p3service.h"
 #include "retroshare/rsreputations.h"
 #include "retroshare/rsidentity.h"
@@ -61,9 +61,8 @@
 class RsNetworkExchangeService
 {
 public:
-
+	RsNetworkExchangeService() = default;
-	RsNetworkExchangeService(){ return;}
+	virtual ~RsNetworkExchangeService() = default;
-    virtual ~RsNetworkExchangeService() {}
 
     virtual uint16_t serviceType() const =0;
     /*!
@@ -85,9 +84,24 @@ public:
     virtual bool msgAutoSync() const =0;
     virtual bool grpAutoSync() const =0;
 
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
 	///                    DISTANT SEARCH FUNCTIONS                          ///
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
+
+	/// Trigger remote generic GXS service search
+	virtual std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
+
+	/// Handle remote generic GXS services search requests to specific service
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
+
+	/// Receive remote generic GXS services search result
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
 
     /*!
      * \brief turtleGroupRequest
@@ -121,7 +135,11 @@ public:
 	 * \param req Turtle search request ID associated with this result
 	 * \param encrypted_group_data  Group data
 	 */
-	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
+	RS_DEPRECATED_FOR("receiveDistantSearchResult")
+	virtual void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        rs_owner_ptr<const uint8_t> encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) = 0;
 
     /*!
      * \brief retrieveTurtleSearchResults
@@ -141,7 +159,9 @@ public:
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
     virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
 
+	RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
 	virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
+
 	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
 
     /*!
@@ -306,5 +326,3 @@ public:
 		}
 	}
 };
-
-#endif // RSGNP_H

libretroshare/src/gxs/rsnxsobserver.cpp

@@ -1,9 +1,8 @@
 /*******************************************************************************
- * libretroshare/src/util: rsmemory.cc                                         *
+ * RetroShare General eXchange System                                          *
  *                                                                             *
- * libretroshare: retroshare core library                                      *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
- *                                                                             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
- * Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net>           *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,35 +18,19 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#include "util/rsmemory.h"
 
-void *rs_malloc(size_t size) 
+#include "gxs/rsnxsobserver.h"
+
+const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
+
+std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
+const noexcept
 {
-    static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything!
+	switch(static_cast<RsNxsObserverErrorNum>(ev))
-    
-    if(size == 0)
 	{
-        std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl;
+	case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
-	print_stacktrace() ;
+		return std::errc::operation_not_supported;
-    	return NULL ;
+	default:
+		return std::error_condition(ev, *this);
 	}
-    
-    if(size > SAFE_MEMALLOC_THRESHOLD)
-    {
-        std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
-	print_stacktrace() ;
-    	return NULL ;
-    }
-    
-    void *mem = malloc(size) ;
-    
-    if(mem == NULL)
-    {
-        std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
-	print_stacktrace() ;
-    	return NULL ;
-    }
-    
-    return mem ;
 }
-

libretroshare/src/gxs/rsnxsobserver.h

@@ -3,7 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher                *
+ * Copyright (C) 2011-2012 Robert Fernie                                       *
+ * Copyright (C) 2011-2012 Christopher Evi-Parker                              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,21 +22,61 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSNXSOBSERVER_H
+#pragma once
-#define RSNXSOBSERVER_H
 
-#include <set>
+#include <system_error>
+#include <vector>
+
+#include "retroshare/rsgxsiface.h"
 #include "rsitems/rsnxsitems.h"
+#include "util/rsdebug.h"
 
-typedef uint32_t TurtleRequestId ;
+typedef uint32_t TurtleRequestId;
+
+enum class RsNxsObserverErrorNum : int32_t
+{
+	NOT_OVERRIDDEN_BY_OBSERVER = 2004,
+};
+
+struct RsNxsObserverErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare NXS Observer"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsNxsObserverErrorNum>(ev))
+		{
+		case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
+			return "Method not overridden by observer";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	std::error_condition default_error_condition(int ev) const noexcept override;
+
+	const static RsNxsObserverErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
+}
+
+/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsJsonApiErrorNum */
+inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsNxsObserverErrorCategory::instance );
+};
 
 class RsNxsObserver
 {
-public:
-
-    RsNxsObserver() {}
-
-
 public:
 
     /*!
@@ -56,6 +99,46 @@ public:
         std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
     }
 
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * requests there.
+	 * @param[in] requestData search query
+	 * @param[in] requestSize search query size
+	 * @param[out] resultData results data storage for a pointer to search
+	 * result reply data or nullptr if no mathing results where found
+	 * @param[out] resultSize storage for results data size or 0 if no matching
+	 * results where found
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		/* Avoid unused paramethers warning this way so doxygen can still parse
+		 * paramethers documentation */
+		(void) requestData; (void) requestSize;
+		(void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * results there.
+	 * @param[in] requestId search query id
+	 * @param[out] resultData results data
+	 * @param[out] resultSize results data size
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		(void) requestId; (void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
     /*!
      * @param grpId group id
      */
@@ -70,6 +153,7 @@ public:
      * @param grpId group id
      */
     virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
-};
 
-#endif // RSNXSOBSERVER_H
+	RsNxsObserver() = default;
+	virtual ~RsNxsObserver() = default;
+};

libretroshare/src/libretroshare.pro

@@ -633,7 +633,6 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsexpr.cc \
 			util/smallobject.cc \
 			util/rsdir.cc \
-			util/rsmemory.cc \
 			util/rsdiscspace.cc \
 			util/rsnet.cc \
 			util/rsnet_ss.cc \
@@ -712,7 +711,8 @@ SOURCES += rsitems/rsnxsitems.cc \
 	gxs/gxstokenqueue.cc \
 	gxs/rsgxsnetutils.cc \
 	gxs/rsgxsutil.cc \
-	gxs/rsgxsrequesttypes.cc
+        gxs/rsgxsrequesttypes.cc \
+        gxs/rsnxsobserver.cpp
 
 # gxs tunnels
 HEADERS += gxstunnel/p3gxstunnel.h \
@@ -936,6 +936,14 @@ rs_jsonapi {
     SOURCES += jsonapi/jsonapi.cpp
 }
 
+rs_deep_forums_index {
+    HEADERS *= deep_search/commonutils.hpp
+    SOURCES *= deep_search/commonutils.cpp
+
+    HEADERS += deep_search/forumsindex.hpp
+    SOURCES += deep_search/forumsindex.cpp
+}
+
 rs_deep_channels_index {
     HEADERS *= deep_search/commonutils.hpp
     SOURCES *= deep_search/commonutils.cpp

libretroshare/src/retroshare/rsgxscircles.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
 enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
 {
 	UNKNOWN            = 0,	/// Used to detect uninizialized values.
-	PUBLIC             = 1,	/// Public distribution, based on GxsIds
+	PUBLIC             = 1,	/// Public distribution
 	EXTERNAL           = 2,	/// Restricted to an external circle, based on GxsIds
 
 	NODES_GROUP        = 3,	/// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

libretroshare/src/retroshare/rsgxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -118,6 +118,10 @@ enum class RsForumEventCode: uint8_t
 	SYNC_PARAMETERS_UPDATED  = 0x0a, /// sync and storage times have changed
 	PINNED_POSTS_CHANGED     = 0x0b, /// some posts where pinned or un-pinned
 	DELETED_FORUM            = 0x0c, /// forum was deleted by cleaning
+	DELETED_POST             = 0x0d,  /// Post deleted (usually by cleaning)
+
+	/// Distant search result received
+	DISTANT_SEARCH_RESULT    = 0x0e
 };
 
 struct RsGxsForumEvent: RsEvent
@@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
 		RS_SERIAL_PROCESS(mForumEventCode);
 		RS_SERIAL_PROCESS(mForumGroupId);
 		RS_SERIAL_PROCESS(mForumMsgId);
-		RS_SERIAL_PROCESS(mForumMsgId);
 		RS_SERIAL_PROCESS(mModeratorsAdded);
 		RS_SERIAL_PROCESS(mModeratorsRemoved);
 	}
@@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
 	~RsGxsForumEvent() override;
 };
 
+/** This event is fired once distant search results are received */
+struct RsGxsForumsDistantSearchEvent: RsEvent
+{
+	RsGxsForumsDistantSearchEvent():
+	    RsEvent(RsEventType::GXS_FORUMS),
+	    mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
+
+	RsForumEventCode mForumEventCode;
+	TurtleRequestId mSearchId;
+	std::vector<RsGxsSearchResult> mSearchResults;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mForumEventCode);
+		RS_SERIAL_PROCESS(mSearchId);
+		RS_SERIAL_PROCESS(mSearchResults);
+	}
+};
+
 class RsGxsForums: public RsGxsIfaceHelper
 {
 public:
@@ -385,6 +411,50 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) = 0;
 
+	/**
+	 * @brief Get forum content summaries
+	 * @jsonapi{development}
+	 * @param[in] forumId id of the forum of which the content is requested
+	 * @param[in] contentIds ids of requested contents, if empty summaries of
+	 *	all messages are reqeusted
+	 * @param[out] summaries storage for summaries
+	 * @return success or error details if something failed
+	 */
+	virtual std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) = 0;
+
+	/**
+	 * @brief Search the whole reachable network for matching forums and
+	 *	posts
+	 * @jsonapi{development}
+	 * An @see RsGxsForumsDistantSearchEvent is emitted when matching results
+	 *	arrives from the network
+	 * @param[in] matchString string to search into the forum and posts
+	 * @param[out] searchId storage for search id, useful to track search events
+	 *	and retrieve search results
+	 * @return success or error details
+	 */
+	virtual std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) = 0;
+
+	/**
+	 * @brief Search the local index for matching forums and posts
+	 * @jsonapi{development}
+	 * @param[in] matchString string to search into the index
+	 * @param[out] searchResults storage for searchr esults
+	 * @return success or error details
+	 */
+	virtual std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) = 0;
+
+
+	////////////////////////////////////////////////////////////////////////////
+	/* Following functions are deprecated and should not be considered a stable
+	 * to use API */
+
 	/**
 	 * @brief Create forum. Blocking API.
 	 * @jsonapi{development}

libretroshare/src/retroshare/rsgxsiface.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -31,13 +32,63 @@
 #include "rsitems/rsserviceids.h"
 #include "retroshare/rsevents.h"
 
+/*!
+ * This structure is used to transport GXS search results.
+ * It contains the group information as well as a context string to tell where
+ * the information was found.
+ * Keep it small as to make search responses as light as possible.
+ * It differs from RsGxsGroupSearchResults because it supports also results from
+ * message matches not just groups.
+ */
+struct RsGxsSearchResult : RsSerializable
+{
+	RsGxsSearchResult(): mPublishTs(0) {}
+
+	/** Id of the group which match*/
+	RsGxsGroupId mGroupId;
+
+	/** Title of the group which match */
+	std::string mGroupName;
+
+	/** Optional message id if the search match is against a message */
+	RsGxsMessageId mMsgId;
+
+	/** Optional message title if the search match is against a message */
+	std::string mMsgName;
+
+	/** Author id of the element which matched (group or message) */
+	RsGxsId mAuthorId;
+
+	/** Publish timestamp of the element which matched (group or message) */
+	rstime_t mPublishTs;
+
+	/** A snippet of content around the exact match */
+	std::string mSearchContext;
+
+	/// @see RsSerializable::serial_process
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mGroupId);
+		RS_SERIAL_PROCESS(mGroupName);
+		RS_SERIAL_PROCESS(mMsgId);
+		RS_SERIAL_PROCESS(mMsgName);
+		RS_SERIAL_PROCESS(mAuthorId);
+		RS_SERIAL_PROCESS(mPublishTs);
+		RS_SERIAL_PROCESS(mSearchContext);
+	}
+
+	virtual ~RsGxsSearchResult() = default;
+};
+
 /*!
  * This structure is used to transport group summary information when a GXS
  * service is searched. It contains the group information as well as a context
  * string to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSummary : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSummary : RsSerializable
 {
 	RsGxsGroupSummary() :
 	    mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
  * strings to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSearchResults : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSearchResults : RsSerializable
 {
+	/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
+	 * do we really need both? */
+
 	RsGxsGroupSearchResults()
         : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
     {}
@@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
 	virtual ~RsGxsGroupSearchResults() = default;
 };
 
+
 /*!
  * Stores ids of changed gxs groups and messages.
  * It is used to notify about GXS changes.

libretroshare/src/retroshare/rsgxsifacetypes.h

@@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
 	    mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
 	    mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
 
-    virtual ~RsGroupMetaData() {}
+	virtual ~RsGroupMetaData() = default;
 
     void operator =(const RsGxsGrpMetaData& rGxsMeta);
     RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

libretroshare/src/rsitems/rsgxsforumitems.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2012  Robert Fernie <retroshare@lunamutt.com>                 *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RS_GXS_FORUM_ITEMS_H
+#pragma once
-#define RS_GXS_FORUM_ITEMS_H
 
 #include <map>
 
@@ -31,7 +32,18 @@
 
 #include "retroshare/rsgxsforums.h"
 
+enum class RsGxsForumsItems : uint8_t
+{
+	GROUP_ITEM     = 0x02,
+	MESSAGE_ITEM   = 0x03,
+	SEARCH_REQUEST = 0x04,
+	SEARCH_REPLY   = 0x05,
+};
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM   = 0x02;
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
 
 class RsGxsForumGroupItem : public RsGxsGrpItem
@@ -61,6 +73,48 @@ public:
 	RsGxsForumMsg mMsg;
 };
 
+struct RsGxsForumsSearchRequest : RsSerializable
+{
+	RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Store search match string
+	std::string mQuery;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mQuery);
+	}
+
+	~RsGxsForumsSearchRequest() override = default;
+};
+
+struct RsGxsForumsSearchReply : RsSerializable
+{
+	RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Results storage
+	std::vector<RsGxsSearchResult> mResults;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mResults);
+	}
+
+	~RsGxsForumsSearchReply() override = default;
+};
+
 class RsGxsForumSerialiser : public RsServiceSerializer
 {
 public:
@@ -69,5 +123,3 @@ public:
 
 	virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
 };
-
-#endif /* RS_GXS_FORUM_ITEMS_H */

libretroshare/src/rsitems/rsitem.h

@@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
 	virtual void serial_process(RsGenericSerializer::SerializeJob,
 	                            RsGenericSerializer::SerializeContext&)// = 0;
 	{
-		std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
+		RS_ERR( "called by an item using new serialization system without "
-		          << "new serialization classes, but not derived! Class is "
+		        "overriding Class is: ", typeid(*this).name() );
-		          << typeid(*this).name() << std::endl;
 		print_stacktrace();
 	}
 

libretroshare/src/rsitems/rsserviceids.h

@@ -28,7 +28,9 @@
 
 enum class RsServiceType : uint16_t
 {
-	NONE                       = 0, /// To detect non-initialized reads
+	/// To detect non-initialized items
+	NONE                       = 0,
+
 	GOSSIP_DISCOVERY           = 0x0011,
 	CHAT                       = 0x0012,
 	MSG                        = 0x0013,
@@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
 	GWEMAIL_MAIL               = 0x0025,
 	SERVICE_CONTROL            = 0x0026,
 	DISTANT_CHAT               = 0x0027,
+
+	/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
 	GXS_TUNNEL                 = 0x0028,
+
 	BANLIST                    = 0x0101,
 	STATUS                     = 0x0102,
 	NXS                        = 0x0200,
@@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
 	POSTED                     = 0x0216,
 	CHANNELS                   = 0x0217,
 	GXSCIRCLE                  = 0x0218,
+
 	/// not gxs, but used with identities.
 	REPUTATION                 = 0x0219,
 	GXS_RECOGN                 = 0x0220,
@@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
 	CHANNELS_CONFIG            = 0x0317,
 	RTT                        = 0x1011, /// Round Trip Time
 
-
-	/***************** IDS ALLOCATED FOR PLUGINS ******************/
-	// 2000+
 	PLUGIN_ARADO_ID            = 0x2001,
 	PLUGIN_QCHESS_ID           = 0x2002,
 	PLUGIN_FEEDREADER          = 0x2003,
 
+	/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
+	GXS_DISTANT                = 0x2233,
+
 	/// Reserved for packet slicing probes.
 	PACKET_SLICING_PROBE       = 0xAABB,
 

libretroshare/src/rsserver/rsinit.cc

@@ -2,7 +2,8 @@
  * libretroshare/src/retroshare: rsinit.cc                                     *
  *                                                                             *
  * Copyright (C) 2004-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2016-2019  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2021       Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -1336,22 +1337,26 @@ int RsServer::StartupRetroShare()
     mWiki->setNetworkExchangeService(wiki_ns) ;
 #endif
 
-        /**** Forum GXS service ****/
+	/************************* Forum GXS service ******************************/
 
-        RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
+	RsGeneralDataService* gxsforums_ds = new RsDataService(
-                                                            RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
+	            currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
+	            nullptr, rsInitConfig->gxs_passwd );
 
+	p3GxsForums* mGxsForums = new p3GxsForums(
+	            gxsforums_ds, nullptr, mGxsIdService );
 
-        p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
+	RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
+#ifdef RS_DEEP_FORUMS_INDEX
+	gxsForumsTunnelService = mGxsNetTunnel;
+#endif
 
-        // create GXS photo service
 	RsGxsNetService* gxsforums_ns = new RsGxsNetService(
-                        RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
+	            RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
-                        mGxsForums, mGxsForums->getServiceInfo(),
+	            mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
-			mReputations, mGxsCircles,mGxsIdService,
+	            mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
-		    pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
+	mGxsForums->setNetworkExchangeService(gxsforums_ns);
 
-    mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
 
         /**** Channel GXS service ****/
 
@@ -1598,7 +1603,10 @@ int RsServer::StartupRetroShare()
 	/**************************************************************************/
     // Turtle search for GXS services
 
-    mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
+	mGxsNetTunnel->registerSearchableService(gxschannels_ns);
+#ifdef RS_DEEP_FORUMS_INDEX
+	mGxsNetTunnel->registerSearchableService(gxsforums_ns);
+#endif
 
 	/**************************************************************************/
 

libretroshare/src/services/p3gxsforums.cc

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
     RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
                    RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
     RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
-    mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
+    mGenActive(false), mGenCount(0),
+    mKnownForumsMutex("GXS forums known forums timestamp cache")
+#ifdef RS_DEEP_FORUMS_INDEX
+    , mDeepIndex(DeepForumsIndex::dbDefaultPath())
+#endif
 {
 	// Test Data disabled in Repo.
 	//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@@ -190,160 +194,199 @@ RsSerialiser* p3GxsForums::setupSerialiser()
 	return rss;
 }
 
-void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
+void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
 {
-#ifdef GXSFORUMS_DEBUG
+	RS_DBG2(changes.size(), " changes to notify");
-	std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
+
+	for(RsGxsNotify* gxsChange: changes)
+	{
+		// Let the compiler delete the change for us
+		std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
+
+		switch(gxsChange->getType())
+		{
+		case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
+		case RsGxsNotify::TYPE_PUBLISHED:
+		{
+			auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
+
+			if(msgChange) /* Message received */
+			{
+				uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
+				switch(static_cast<RsGxsForumsItems>(msgSubtype))
+				{
+				case RsGxsForumsItems::MESSAGE_ITEM:
+				{
+					auto newForumMessageItem =
+					        dynamic_cast<RsGxsForumMsgItem*>(
+					            msgChange->mNewMsgItem );
+
+					if(!newForumMessageItem)
+					{
+						RS_ERR("Received message change with mNewMsgItem type "
+						       "mismatching or null");
+						print_stacktrace();
+						return;
+					}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+					RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
+					tmpPost.mMeta = newForumMessageItem->meta;
+					mDeepIndex.indexForumPost(tmpPost);
 #endif
-
-	std::vector<RsGxsNotify *>::iterator it;
-	for(it = changes.begin(); it != changes.end(); ++it)
-	{
-		RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
-
-		if (msgChange)
-		{
-			if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
-				if (rsEvents)
-				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
 					ev->mForumMsgId = msgChange->mMsgId;
 					ev->mForumGroupId = msgChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
 					rsEvents->postEvent(ev);
-				}
-
-#ifdef NOT_USED_YET
-			if (!msgChange->metaChange())
-			{
-#ifdef GXSCHANNELS_DEBUG
-				std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
-				std::cerr << std::endl;
-#endif
-
-				std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
-				for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
-				{
-#ifdef GXSCHANNELS_DEBUG
-					std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
-					std::cerr << std::endl;
-#endif
-					bool enabled = false;
-					if (autoDownloadEnabled(mit->first, enabled) && enabled)
-					{
-#ifdef GXSCHANNELS_DEBUG
-						std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
-						std::cerr << std::endl;
-#endif
-
-						/* problem is most of these will be comments and votes,
-						 * should make it occasional - every 5mins / 10minutes TODO */
-						unprocessedGroups.push_back(mit->first);
-					}
-				}
-			}
-#endif
-		}
-		else
-		{
-			if (rsEvents)
-			{
-				RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
-				if (grpChange)
-				{
-					switch (grpChange->getType())
-					{
-					case RsGxsNotify::TYPE_PROCESSED:	// happens when the group is subscribed
-					{
-							auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
-							ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
-							rsEvents->postEvent(ev);
-					}
 					break;
-
-                    case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
-                    {
-                            auto ev = std::make_shared<RsGxsForumEvent>();
-                            ev->mForumGroupId = grpChange->mGroupId;
-                            ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
-                            rsEvents->postEvent(ev);
 				}
+				default:
+					RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
 					break;
+				}
+			}
 
-                    case RsGxsNotify::TYPE_PUBLISHED:
+			auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-					case RsGxsNotify::TYPE_RECEIVED_NEW:
+			if(groupChange) /* Group received */
 			{
-						/* group received */
-
 				bool unknown;
 				{
 					RS_STACK_MUTEX(mKnownForumsMutex);
-                            unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
+					unknown = ( mKnownForums.find(gxsChange->mGroupId)
-                            mKnownForums[grpChange->mGroupId] = time(nullptr);
+					            == mKnownForums.end() );
+					mKnownForums[gxsChange->mGroupId] = time(nullptr);
 					IndicateConfigChanged();
 				}
 
 				if(unknown)
 				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
+					ev->mForumGroupId = gxsChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
 					rsEvents->postEvent(ev);
 				}
-						else
+
-							RsInfo() << __PRETTY_FUNCTION__
+#ifdef RS_DEEP_FORUMS_INDEX
-							         << " Not notifying already known forum "
+				uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
-							         << grpChange->mGroupId << std::endl;
+				switch(static_cast<RsGxsForumsItems>(itemType))
+				{
+				case RsGxsForumsItems::GROUP_ITEM:
+				{
+					auto newForumGroupItem =
+					        static_cast<RsGxsForumGroupItem*>(
+					            groupChange->mNewGroupItem );
+					mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
+					break;
+				}
+				default:
+					RS_WARN("Got unknown gxs group subtype: ", itemType);
+					break;
+				}
+#endif // def RS_DEEP_FORUMS_INDEX
+
 			}
 			break;
-
+		}
-                    case RsGxsNotify::TYPE_GROUP_DELETED:
+		case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-                        ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
+		{
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_MESSAGE_DELETED:
+		{
+			auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
+			if(!delChange)
+			{
+				RS_ERR( "Got mismatching notification type: ",
+				        gxsChange->getType() );
+				print_stacktrace();
+				break;
+			}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumPostFromIndex(
+			            delChange->mGroupId, delChange->messageId);
+#endif
+
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumEventCode = RsForumEventCode::DELETED_POST;
+			ev->mForumGroupId = delChange->mGroupId;
+			ev->mForumMsgId = delChange->messageId;
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_DELETED:
+		{
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
+#endif
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
 			rsEvents->postEvent(ev);
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_STATISTICS_CHANGED:
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-						ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
 			rsEvents->postEvent(ev);
 
 			RS_STACK_MUTEX(mKnownForumsMutex);
-                        mKnownForums[grpChange->mGroupId] = time(nullptr);
+			mKnownForums[gxsChange->mGroupId] = time(nullptr);
 			IndicateConfigChanged();
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_UPDATED:
 		{
-						// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
+			/* Happens when the group data has changed. In this case we need to
+			 * analyse the old and new group in order to detect possible
+			 * notifications for clients */
 
-						RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-						RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
 
-						if(old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
+			RsGxsForumGroupItem* old_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			RsGxsForumGroupItem* new_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
+
+			if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
 			{
-                            RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
+				RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
-							delete grpChange;
+				        "mNewGroup not of type RsGxsForumGroupItem or NULL. "
-							continue;
+				        "This is inconsistent!");
+				print_stacktrace();
+				break;
 			}
 
-						// First of all, we check if there is a difference between the old and new list of moderators
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
+#endif
+
+			/* First of all, we check if there is a difference between the old
+			 * and new list of moderators */
 
 			std::list<RsGxsId> added_mods, removed_mods;
-
 			for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
-							if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == old_forum_grp_item->mGroup.mAdminList.ids.end() )
 					added_mods.push_back(gxs_id);
 
 			for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
-							if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == new_forum_grp_item->mGroup.mAdminList.ids.end() )
 					removed_mods.push_back(gxs_id);
 
 			if(!added_mods.empty() || !removed_mods.empty())
@@ -359,24 +402,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			}
 
 			// check the list of pinned posts
-
 			std::list<RsGxsMessageId> added_pins, removed_pins;
 
 			for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					added_pins.push_back(msg_id);
 
 			for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					removed_pins.push_back(msg_id);
 
 			if(!added_pins.empty() || !removed_pins.empty())
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
-
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
-
 				rsEvents->postEvent(ev);
 			}
 
@@ -384,30 +426,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			        || old_forum_grp_item->meta.mGroupName  != new_forum_grp_item->meta.mGroupName
 			        || old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
 			        || old_forum_grp_item->meta.mAuthorId   != new_forum_grp_item->meta.mAuthorId
-                          || old_forum_grp_item->meta.mCircleId      != new_forum_grp_item->meta.mCircleId
+			        || old_forum_grp_item->meta.mCircleId   != new_forum_grp_item->meta.mCircleId )
-                             )
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
 				rsEvents->postEvent(ev);
 			}
-					}
-                        break;
 
+			break;
+		}
 
 		default:
-                        RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
+			RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
+			        " Currently not handled." );
 			break;
 		}
 	}
-			}
-		}
-
-		/* shouldn't need to worry about groups - as they need to be subscribed to */
-
-        delete *it;
-	}
 }
 
 void	p3GxsForums::service_tick()
@@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
 	forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
 	        | GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
 
+	/* This flag have always this value even for circle restricted forums due to
+	 * how GXS distribute/verify groups */
 	forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
 
 	forum.mMeta.mCircleId.clear();
@@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
 	        id == mMeta.mAuthorId;
 }
 
+std::error_condition p3GxsForums::getContentSummaries(
+        const RsGxsGroupId& forumId,
+        const std::set<RsGxsMessageId>& contentIds,
+        std::vector<RsMsgMetaData>& summaries )
+{
+	uint32_t token;
+	RsTokReqOptions opts;
+	opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
+
+	GxsMsgReq msgReq;
+	msgReq[forumId] = contentIds;
+
+
+	if(!requestMsgInfo(token, opts, msgReq))
+	{
+		RS_ERR("requestMsgInfo failed");
+		return std::errc::invalid_argument;
+	}
+
+	switch(waitToken(token, std::chrono::seconds(5)))
+	{
+	case RsTokenService::COMPLETE:
+	{
+		GxsMsgMetaMap metaMap;
+		if(!RsGenExchange::getMsgMeta(token, metaMap))
+			return std::errc::result_out_of_range;
+		summaries = metaMap[forumId];
+		return std::error_condition();
+	}
+	case RsTokenService::PARTIAL: // [[fallthrough]];
+	case RsTokenService::PENDING:
+		return std::errc::timed_out;
+	default:
+		return std::errc::not_supported;
+	}
+}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+std::error_condition p3GxsForums::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG1("");
+
+	RsGxsForumsSearchRequest request;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(request.mType) );
+		return std::errc::bad_message;
+	}
+
+	RsGxsForumsSearchReply reply;
+	auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
+	if(mErr || reply.mResults.empty()) return mErr;
+
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(reply);
+		resultSize = ctx.mOffset;
+	}
+
+	resultData = rs_malloc<uint8_t>(resultSize);
+	RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+	RsGenericSerializer::SerializeJob j =
+	        RsGenericSerializer::SerializeJob::SERIALIZE;
+	RS_SERIAL_PROCESS(reply);
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string& matchString, TurtleRequestId& searchId )
+{
+	RsGxsForumsSearchRequest request;
+	request.mQuery = matchString;
+
+	uint32_t requestSize;
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(request);
+		requestSize = ctx.mOffset;
+	}
+
+	std::error_condition ec;
+	auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
+	if(!requestData) return ec;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	return netService()->distantSearchRequest(
+	            requestData, requestSize,
+	            static_cast<RsServiceType>(serviceType()), searchId );
+}
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string& matchString,
+        std::vector<RsGxsSearchResult>& searchResults )
+{ return prepareSearchResults(matchString, false, searchResults); }
+
+std::error_condition p3GxsForums::prepareSearchResults(
+        const std::string& matchString, bool publicOnly,
+        std::vector<RsGxsSearchResult>& searchResults )
+{
+	std::vector<DeepForumsSearchResult> results;
+	auto mErr = mDeepIndex.search(matchString, results);
+	if(mErr) return mErr;
+
+	searchResults.clear();
+	for(auto uRes: results)
+	{
+		RsUrl resUrl(uRes.mUrl);
+		const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
+		if(!forumIdStr)
+		{
+			RS_ERR( "Forum URL retrieved from deep index miss ID. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::address_not_available;
+		}
+
+		std::vector<RsGxsForumGroup> forumsInfo;
+		RsGxsGroupId forumId(*forumIdStr);
+		if(forumId.isNull())
+		{
+			RS_ERR( "Forum ID retrieved from deep index is invalid. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::bad_address;
+		}
+
+		if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
+		        forumsInfo.empty() )
+		{
+			RS_ERR( "Forum just parsed from deep index link not found. "
+			        "Should never happen! ", forumId, " ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::identifier_removed;
+		}
+
+		RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
+
+		// Avoid leaking sensitive information to unkown peers
+		if( publicOnly &&
+		        ( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
+		          RsGxsCircleType::PUBLIC ) ) continue;
+
+		RsGxsSearchResult res;
+		res.mGroupId = forumId;
+		res.mGroupName = fMeta.mGroupName;
+		res.mAuthorId = fMeta.mAuthorId;
+		res.mPublishTs = fMeta.mPublishTs;
+		res.mSearchContext = uRes.mSnippet;
+
+		auto postIdStr =
+		        resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
+		if(postIdStr)
+		{
+			RsGxsMessageId msgId(*postIdStr);
+			if(msgId.isNull())
+			{
+				RS_ERR( "Post just parsed from deep index link is invalid. "
+				        "Should never happen! ", postIdStr, " ", uRes.mUrl );
+				print_stacktrace();
+				return std::errc::bad_address;
+			}
+
+			std::vector<RsMsgMetaData> msgSummaries;
+			auto errc = getContentSummaries(
+			            forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
+			if(errc) return errc;
+
+			if(msgSummaries.size() != 1)
+			{
+				RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
+				        "should never happen!" );
+				return std::errc::result_out_of_range;
+			}
+
+			RsMsgMetaData& msgMeta(msgSummaries[0]);
+			res.mMsgId = msgMeta.mMsgId;
+			res.mMsgName = msgMeta.mMsgName;
+			res.mAuthorId = msgMeta.mAuthorId;
+		}
+
+		RS_DBG4(res);
+		searchResults.push_back(res);
+	}
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RsGxsForumsSearchReply reply;
+	{
+		RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(reply);
+	}
+	free(resultData);
+
+	if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(reply.mType) );
+		return std::errc::bad_message;
+	}
+
+	auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
+	event->mSearchId = requestId;
+	event->mSearchResults = reply.mResults;
+	rsEvents->postEvent(event);
+	return std::error_condition();
+}
+
+#else  // def RS_DEEP_FORUMS_INDEX
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string&, TurtleRequestId& )
+{ return std::errc::function_not_supported; }
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string&,
+        std::vector<RsGxsSearchResult>& )
+{ return std::errc::function_not_supported; }
+
+#endif // def RS_DEEP_FORUMS_INDEX
+
 /*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
         "retroshare:///forums";
 /*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

libretroshare/src/services/p3gxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -32,6 +32,10 @@
 #include "util/rstickevent.h"
 #include "util/rsdebug.h"
 
+#ifdef RS_DEEP_FORUMS_INDEX
+#include "deep_search/forumsindex.hpp"
+#endif
+
 
 class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
 	public RsTickEvent	/* only needed for testing - remove after */
@@ -144,6 +148,33 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) override;
 
+	/// @see RsGxsForums
+	std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) override;
+
+	/// @see RsGxsForums
+	std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) override;
+
+	/// @see RsGxsForums
+	std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) override;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	/// @see RsNxsObserver
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNxsObserver
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+#endif
+
     /// implementation of rsGxsGorums
     ///
 	bool getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups) override;
@@ -155,6 +186,17 @@ public:
 
 	bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
 
+protected:
+#ifdef RS_DEEP_FORUMS_INDEX
+	/** Internal usage
+	 * @param[in] publicOnly if true is passed only results pertaining to
+	 * publicly shared forums are returned
+	 */
+	std::error_condition prepareSearchResults(
+	        const std::string& matchString, bool publicOnly,
+	        std::vector<RsGxsSearchResult>& searchResults );
+#endif //def RS_DEEP_FORUMS_INDEX
+
 private:
 
 static uint32_t forumsAuthenPolicy();
@@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
     std::map<RsGxsGroupId,rstime_t> mKnownForums ;
 	
 	RsMutex mKnownForumsMutex;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	DeepForumsIndex mDeepIndex;
+#endif
 };

libretroshare/src/turtle/turtleclientservice.h

@@ -19,23 +19,25 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
-// This class is the parent class for any service that will use the turtle router to distribute its packets.
-// Typical representative clients include:
-//
-// 	p3ChatService:		opens tunnels to distant peers for chatting
-// 	ftServer:			searches and open tunnels to distant sources for file transfer
-//
 #pragma once
 
 #include <string>
 #include <stdlib.h>
-#include <serialiser/rsserial.h>
+
-#include <turtle/rsturtleitem.h>
+#include "serialiser/rsserial.h"
+#include "turtle/rsturtleitem.h"
+#include "util/rsdebug.h"
 
 struct RsItem;
 class p3turtle ;
 
+/** This class is the parent class for any service that will use the turtle
+ * router to distribute its packets.
+ * Typical representative clients include:
+ * 	p3ChatService: opens tunnels to distant peers for chatting
+ * 	ftServer:      searches and open tunnels to distant sources for file
+ * 	               transfer
+ */
 class RsTurtleClientService
 {
 	public:
@@ -88,27 +90,32 @@ class RsTurtleClientService
 		}
 
 	/*!
-         * \brief receiveSearchRequest
+	* This method is called by the turtle router to notify the client of a
-         * 			This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
+	* search request in the form generic data.
-         * 			result contains the serialised generic result returned by the client.
+	* The returned result contains the serialised generic result returned by the
-         *
+	* client service.
 	* The turtle router keeps the memory ownership over search_request_data
-         *
 	* \param search_request_data generic serialized search data
 	* \param search_request_data_len  length of the serialized search data
 	* \param search_result_data generic serialized search result data
 	* \param search_result_data_len length of the serialized search result data
-         * \param max_allowed_hits         max number of hits allowed to be sent back and forwarded
+	* \param max_allowed_hits max number of hits allowed to be sent back and
-         *
+	* forwarded
-         * \return true if the search is successful.
+	* \return true if matching results are available, false otherwise.
 	*/
-    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
+	virtual bool receiveSearchRequest(
-                                          uint32_t /*search_request_data_len*/,
+	        unsigned char *search_request_data, uint32_t search_request_data_len,
-                                          unsigned char *& /*search_result_data*/,
+	        unsigned char *& search_result_data, uint32_t& search_result_data_len,
-                                          uint32_t& /*search_result_data_len*/,
+	        uint32_t& max_allows_hits )
-                                          uint32_t& /* max_allows_hits */)
 	{
-			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+		/* Suppress unused warning this way and not commenting the param names
+		 * so doxygen match documentation against params */
+		(void) search_request_data; (void) search_request_data_len;
+		(void) search_result_data; (void) search_result_data_len;
+		(void) max_allows_hits;
+
+		RS_WARN( "Received search request from turtle router, but the client "
+		         "is not handling it!" );
 		return false;
 	}
 

libretroshare/src/util/rsdebug.cc

@@ -3,9 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright (C) 2004-2008 by Robert Fernie   <retroshare@lunamutt.com>        *
+ * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2020-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -36,6 +36,9 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
 	        " not available in category: " + categoryName;
 }
 
+std::error_condition rs_errno_to_condition(int errno_code)
+{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
+
 
 ////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////

libretroshare/src/util/rsdebug.h

@@ -2,8 +2,8 @@
  * RetroShare debugging utilities                                              *
  *                                                                             *
  * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org>              *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
  * the message around */
 std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
 
+/** Convert C errno codes to modern C++11 std::error_condition, this is quite
+ * useful to use toghether with C functions used around the code like `malloc`,
+ * `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
+ */
+std::error_condition rs_errno_to_condition(int errno_code);
+
 
 template <RsLoggerCategories CATEGORY>
 struct t_RsLogger : std::ostringstream

libretroshare/src/util/rsmemory.h

@@ -3,8 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012 Cyril Soler <csoler@users.sourceforge.net>                   *
+ * Copyright (C) 2012  Cyril Soler <csoler@users.sourceforge.net>              *
- * Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net>                *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -25,8 +26,10 @@
 #include <cstdlib>
 #include <iostream>
 #include <memory>
+#include <system_error>
 
 #include "util/stacktrace.h"
+#include "util/rsdebug.h"
 
 /**
  * @brief Shorthand macro to declare optional functions output parameters
@@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
  * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
 template<typename T> using rs_owner_ptr = T*;
 
-void *rs_malloc(size_t size) ;
+
+/// 1Gb should be enough for everything!
+static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
+
+/** Comfortable templated safer malloc, just use it specifing the type of the
+ * pointer to be returned without need of ugly casting the returned pointer
+ * `uint8_t* ptr = rs_malloc<uint8_t>(40);`
+ * @param[in] size number of bytes to allocate
+ * @param[out] ec optional storage for error details. Value is meaningful only
+ *	whem nullptr is returned.
+ * @return nullptr on error, pointer to the allocated chuck of memory on success
+ */
+template<typename T = void> rs_owner_ptr<T> rs_malloc(
+        size_t size,
+        rs_view_ptr<std::error_condition> ec = nullptr )
+{
+	if(size == 0)
+	{
+		if(!ec)
+		{
+			RS_ERR("A chunk of size 0 was requested");
+			print_stacktrace();
+			exit(static_cast<int>(std::errc::invalid_argument));
+		}
+
+		*ec = std::errc::invalid_argument;
+		return nullptr;
+	}
+
+	if(size > SAFE_MEMALLOC_THRESHOLD)
+	{
+		if(!ec)
+		{
+			RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
+			        " was requested" );
+			exit(static_cast<int>(std::errc::argument_out_of_domain));
+		}
+
+		*ec = std::errc::argument_out_of_domain;
+		return nullptr;
+	}
+
+	void* mem = malloc(size);
+	if(!mem)
+	{
+		if(!ec)
+		{
+			RS_ERR( "Allocation failed for a chunk of ", size,
+			        " bytes with: ", errno);
+			print_stacktrace();
+			exit(errno);
+		}
+
+		*ec = rs_errno_to_condition(errno);
+		return nullptr;
+	}
+
+	return static_cast<rs_owner_ptr<T>>(mem);
+}
+
 
 /** @deprecated use std::unique_ptr instead
 // This is a scope guard to release the memory block when going of of the current scope.
@@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
 //
 // }	// mem gets freed automatically
 */
-class RsTemporaryMemory
+class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
 {
 public:
 	explicit RsTemporaryMemory(size_t s)

retroshare-gui/src/gui/NewsFeed.cpp

@@ -270,7 +270,10 @@ void NewsFeed::handleForumEvent(std::shared_ptr<const RsEvent> event)
 
 	case RsForumEventCode::UPDATED_MESSAGE:
 	case RsForumEventCode::NEW_MESSAGE:
-		addFeedItem(new GxsForumMsgItem(this, NEWSFEED_NEW_FORUM, pe->mForumGroupId, pe->mForumMsgId, false, true));
+			addFeedItem(new GxsForumMsgItem(
+			                this, NEWSFEED_NEW_FORUM,
+			                pe->mForumGroupId, pe->mForumMsgId,
+			                false, true ));
 		break;
 
 	default: break;

retroshare.pri

@@ -141,6 +141,11 @@ rs_macos10.15:CONFIG -= rs_macos10.11
 CONFIG *= no_rs_jsonapi
 rs_jsonapi:CONFIG -= no_rs_jsonapi
 
+# To enable forums indexing append the following assignation to qmake command
+# line "CONFIG+=rs_deep_forums_index"
+CONFIG *= no_rs_deep_forums_index
+rs_deep_forums_index:CONFIG -= no_rs_deep_forums_index
+
 # To enable channel indexing append the following assignation to qmake command
 # line "CONFIG+=rs_deep_channels_index"
 CONFIG *= no_rs_deep_channels_index
@@ -561,6 +566,7 @@ rs_webui {
     DEFINES *= RS_WEBUI
 }
 
+rs_deep_forums_index:DEFINES *= RS_DEEP_FORUMS_INDEX
 rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX
 
 rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX