merged with upstream/master

.gitignore

@@ -15,3 +15,5 @@ Thumbs.db
 *.pro.user
 .kdev4
 *.kdev4
+
+!supportlibs/libsam3/Makefile

.gitlab-ci.yml

@@ -13,7 +13,6 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
-  REPO_ARGS_CMD: '[ -n "$CI_MERGE_REQUEST_ID" ] && echo --build-arg REPO_URL=\"$CI_MERGE_REQUEST_SOURCE_PROJECT_URL\" --build-arg REPO_BRANCH=\"$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\" || echo --build-arg REPO_URL=\"$CI_REPOSITORY_URL\" --build-arg REPO_BRANCH=\"$CI_COMMIT_BRANCH\"'
   UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
         
 build-ubuntu-test-image:
@@ -27,7 +26,7 @@ build-ubuntu-test-image:
         - >
           docker build
           -t $UBUNTU_TESTING_IMAGE_TAG
-          $($REPO_ARGS_CMD)
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
           --file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
           Dockercontext
         - docker push $UBUNTU_TESTING_IMAGE_TAG
@@ -74,7 +73,9 @@ build-android-arm-apk:
     script:
         - mkdir Dockercontext
         - >
-          docker build -t retroshare:android_arm_latest $($REPO_ARGS_CMD)
+          docker build -t retroshare:android_arm_latest
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
+          --build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
           --file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
           Dockercontext
 # see https://stackoverflow.com/a/59055906
@@ -92,9 +93,11 @@ build-android-arm64-apk:
     script:
         - mkdir Dockercontext
         - >
-          docker build -t retroshare:android_arm64_latest $($REPO_ARGS_CMD)
+          docker build -t retroshare:android_arm64_latest
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
           --build-arg ANDROID_PLATFORM_VER=21
           --build-arg ANDROID_NDK_ARCH=arm64
+          --build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
           --file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
           Dockercontext
         - >

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "supportlibs/rapidjson"]
 	path = supportlibs/rapidjson
 	url = https://github.com/Tencent/rapidjson.git
+[submodule "supportlibs/libsam3"]
+	path = supportlibs/libsam3
+	url = https://github.com/i2p/libsam3.git

build_scripts/Android/Dockerfile

@@ -16,6 +16,11 @@
 # docker login registry.gitlab.com
 # docker push ${CI_IMAGE_NAME}
 
+## To extract the generated APK easily you can run after the build complete
+# docker cp \
+#	$(docker create --rm ${CI_IMAGE_NAME}):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk \
+#	/tmp/RetroShare_Android_Service.apk
+
 
 FROM ubuntu:20.04
 
@@ -80,12 +85,12 @@ WORKDIR /jsonapi-generator-build/
 RUN qmake ../RetroShare/jsonapi-generator/src/ \
 		CONFIG+=no_retroshare_plugins \
 		CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
-		CONFIG+=rs_jsonapi && \
+		CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
 	make -j$(nproc)
 
-#CONFIG+=no_keywords
 RUN mkdir /retroshare-service-android-build
 WORKDIR /retroshare-service-android-build
+ARG RS_SERVICE_QMAKE_EXTRA_OPTS
 RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
 	-spec android-clang \
 	CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
@@ -93,7 +98,9 @@ RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
 	JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
 	NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
 	CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
-	CONFIG+=no_rs_service_terminal_login
+	CONFIG+=no_rs_service_terminal_login \
+	CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
+	$RS_SERVICE_QMAKE_EXTRA_OPTS
 RUN	make -j$(nproc)
 RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
 RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \

build_scripts/Android/README.asciidoc

@@ -81,6 +81,7 @@ JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
 NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
 CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
 CONFIG+=no_rs_service_terminal_login
+CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3
 -------------------------------------------------------------------------------
 
 TIP: Some versions of QtCreator try to find the Android SDK in

build_scripts/GitlabCI/Android.Dockerfile

@@ -17,7 +17,7 @@ WORKDIR /jsonapi-generator-build/
 RUN qmake ../RetroShare/jsonapi-generator/src/ \
 		CONFIG+=no_retroshare_plugins \
 		CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
-		CONFIG+=rs_jsonapi && \
+		CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
 	make -j$(nproc)
 
 RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
@@ -25,6 +25,7 @@ WORKDIR /retroshare-service-android-build
 
 # ARG declared before FROM get wiped after it, so we need declaring it again
 ARG ANDROID_NDK_ARCH=arm
+ARG RS_SERVICE_QMAKE_EXTRA_OPTS
 RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
 	-spec android-clang \
 	CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
@@ -32,7 +33,9 @@ RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
 	JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
 	NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
 	CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
-	CONFIG+=no_rs_service_terminal_login
+	CONFIG+=no_rs_service_terminal_login \
+	CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
+	$RS_SERVICE_QMAKE_EXTRA_OPTS
 RUN	make -j$(nproc)
 RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
 

build_scripts/GitlabCI/getRepoArgs.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+[ -n "$CI_MERGE_REQUEST_ID" ] &&
+	echo \
+		--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
+		--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
+	echo \
+		--build-arg REPO_URL="$CI_REPOSITORY_URL" \
+		--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

build_scripts/OBS

@@ -1 +1 @@
-Subproject commit 984ed34b7f751bd407877603551ead49959f1901
+Subproject commit b260e58346b1eec782bdf88a7e8f3c9d36fd3ecb

build_scripts/Windows-msys2/build/pack.bat

@@ -147,8 +147,10 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
 	call :copy_dependencies "%%D" "%RsDeployPath%"
 )
 
-echo copy qss
+if exist "%SourcePath%\retroshare-gui\src\qss" (
-xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+	echo copy qss
+	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+)
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

build_scripts/Windows-msys2/env/tools/prepare-msys2.bat

@@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	)
 )
 
-set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
+if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
-set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
+if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
+
+set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
+set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
 
 %cecho% info "Remove previous MSYS2 version"
 call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
@@ -41,12 +44,10 @@ if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download M
 set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
 
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
-call "%EnvRootPath%\update-msys2.bat"
-call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows-msys2/env/tools/root/update-msys2.bat

@@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
 goto :EOF
 
 :update
-set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
+set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
 
 echo Update MSYS2 %~1
-"%MSYSSH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYSSH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 :exit
 endlocal

build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi

@@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /r "${DEPLOYDIR}\qss\*.*"
+  File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link

build_scripts/Windows/build/pack.bat

@@ -132,8 +132,10 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
 copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
 del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
 
-echo copy qss
+if exist "%SourcePath%\retroshare-gui\src\qss" (
-xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+	echo copy qss
+	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+)
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

build_scripts/Windows/env/tools/prepare-msys2.bat

@@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	)
 )
 
-set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
+if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
-set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
+if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
+
+set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
+set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
 set CMakeInstall=cmake-3.19.0-win32-x86.zip
 set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.19.0/%CMakeInstall%
 set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
@@ -64,17 +67,11 @@ if "%FoundProfile%"=="0" (
 
 set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
 
-%cecho% info "Update keyring"
-"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
-"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
-
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
-call "%EnvRootPath%\update-msys2.bat"
-call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows/env/tools/root/update-msys2.bat

@@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
 goto :EOF
 
 :update
-set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
+set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
 
 echo Update MSYS2 %~1
-"%MSYSSH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYSSH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 :exit
 endlocal

build_scripts/Windows/installer/retroshare-Qt5.nsi

@@ -334,7 +334,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
+  File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link

libretroshare/src/deep_search/commonutils.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,54 +18,49 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <algorithm>
+#include <thread>
+
 #include "deep_search/commonutils.hpp"
 #include "util/stacktrace.h"
-#include "util/rsdebug.h"
+#include "util/rsthreads.h"
+#include "util/rsdebuglevel0.h"
+
+#ifndef XAPIAN_AT_LEAST
+/// Added in Xapian 1.4.2.
+#define XAPIAN_AT_LEAST(A,B,C) \
+	(XAPIAN_MAJOR_VERSION > (A) || \
+	(XAPIAN_MAJOR_VERSION == (A) && \
+	(XAPIAN_MINOR_VERSION > (B) || \
+	(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
+#endif
 
 namespace DeepSearch
 {
-
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags, int blockSize )
-{
-	try
-	{
-		std::unique_ptr<Xapian::WritableDatabase> dbPtr(
-		        new Xapian::WritableDatabase(path, flags, blockSize) );
-		return dbPtr;
-	}
-	catch(Xapian::DatabaseLockError)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-	catch(...)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
-		        << "deleting it might help without causing any harm: "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-
-	return nullptr;
-}
-
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags )
 {
 	try
 	{
+#if XAPIAN_AT_LEAST(1,3,2)
 		std::unique_ptr<Xapian::Database> dbPtr(
 		        new Xapian::Database(path, flags) );
+#else
+		std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
+		if(flags)
+		{
+			RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
+			         "library version: ", XAPIAN_VERSION, " < 1.3.2" );
+		}
+#endif
 		return dbPtr;
 	}
-	catch(Xapian::DatabaseOpeningError e)
+	catch(Xapian::DatabaseOpeningError& e)
 	{
 		RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
 		          << ", probably nothing has been indexed yet." << std::endl;
 	}
-	catch(Xapian::DatabaseLockError)
+	catch(Xapian::DatabaseLockError&)
 	{
 		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
 		        << path << std::endl;
@@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
 	return date;
 }
 
+StubbornWriteOpQueue::~StubbornWriteOpQueue()
+{
+	auto fErr = flush(0);
+	if(fErr)
+	{
+		RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
+		          " operations irreparably lost ", fErr );
+		print_stacktrace();
+	}
+}
+
+void StubbornWriteOpQueue::push(write_op op)
+{
+	RS_DBG4("");
+
+	{
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		mOpStore.push(op);
+	}
+
+	flush();
+}
+
+std::error_condition StubbornWriteOpQueue::flush(
+        rstime_t acceptDelay, rstime_t callTS )
+{
+	RS_DBG4("");
+
+	{
+		// Return without attempt to open the database if the queue is empty
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		if(mOpStore.empty()) return std::error_condition();
+	}
+
+	std::unique_ptr<Xapian::WritableDatabase> dbPtr;
+	try
+	{
+		dbPtr = std::make_unique<Xapian::WritableDatabase>(
+		            mDbPath, Xapian::DB_CREATE_OR_OPEN );
+	}
+	catch(Xapian::DatabaseLockError)
+	{
+		if(acceptDelay)
+		{
+			rstime_t tNow = time(nullptr);
+			rstime_t maxRemaining = tNow - (callTS + acceptDelay);
+			if(maxRemaining > 0)
+			{
+				std::chrono::milliseconds interval(
+				            std::max(rstime_t(50), maxRemaining*1000/5) );
+				RS_DBG3( "Cannot acquire database write lock, retrying in:",
+				         interval.count(), "ms" );
+				RsThread::async([this, acceptDelay, callTS, interval]()
+				{
+					std::this_thread::sleep_for(interval);
+					flush(acceptDelay, callTS);
+				});
+				return std::error_condition();
+			}
+			else
+			{
+				RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
+				return std::errc::timed_out;
+			}
+		}
+		else return std::errc::resource_unavailable_try_again;
+	}
+	catch(...)
+	{
+		RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
+		print_stacktrace();
+		return std::errc::io_error;
+	}
+
+	std::unique_lock<std::mutex> lock(mQueueMutex);
+	while(!mOpStore.empty())
+	{
+		auto op = mOpStore.front(); mOpStore.pop();
+		op(*dbPtr);
+	}
+	return std::error_condition();
+}
+
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
+{
+	if(rsHtmlDoc.empty()) return rsHtmlDoc;
+
+	const bool isPlainMsg =
+	        rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
+	if(isPlainMsg) return rsHtmlDoc;
+
+	auto oSize = rsHtmlDoc.size();
+	auto bodyTagBegin(rsHtmlDoc.find("<body"));
+	if(bodyTagBegin >= oSize) return rsHtmlDoc;
+
+	auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
+	if(bodyTagEnd >= oSize) return rsHtmlDoc;
+
+	std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
+
+	// strip also CSS inside <style></style>
+	oSize = retVal.size();
+	auto styleTagBegin(retVal.find("<style"));
+	if(styleTagBegin < oSize)
+	{
+		auto styleEnd(retVal.find("</style>", styleTagBegin));
+		if(styleEnd < oSize)
+			retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
+	}
+
+	std::string::size_type oPos;
+	std::string::size_type cPos;
+	int itCount = 0;
+	while((oPos = retVal.find("<")) < retVal.size())
+	{
+		if((cPos = retVal.find(">")) <= retVal.size())
+			retVal.erase(oPos, 1+cPos-oPos);
+		else break;
+
+		// Avoid infinite loop with crafty input
+		if(itCount > 1000)
+		{
+			RS_WARN( "Breaking stripping loop due to max allowed iterations ",
+			         "rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
+			break;
+		}
+		++itCount;
+	}
+
+	return retVal;
+}
+
 }

libretroshare/src/deep_search/commonutils.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -21,6 +21,9 @@
 
 #include <xapian.h>
 #include <memory>
+#include <functional>
+#include <queue>
+#include <mutex>
 
 #include "util/rstime.h"
 
@@ -33,13 +36,34 @@
 
 namespace DeepSearch
 {
-
+typedef std::function<void(Xapian::WritableDatabase&)> write_op;
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags = 0, int blockSize = 0 );
 
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags = 0 );
 
 std::string timetToXapianDate(const rstime_t& time);
 
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
+
+struct StubbornWriteOpQueue
+{
+	explicit StubbornWriteOpQueue(const std::string& dbPath):
+	    mDbPath(dbPath) {}
+
+	~StubbornWriteOpQueue();
+
+	void push(write_op op);
+
+	std::error_condition flush(
+	        rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
+
+private:
+	std::queue<write_op> mOpStore;
+	rstime_t mLastFlush;
+
+	std::mutex mQueueMutex;
+
+	const std::string mDbPath;
+};
+
 }

libretroshare/src/deep_search/filesindex.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,48 +18,48 @@
  *                                                                             *
  *******************************************************************************/
 
-#include "deep_search/filesindex.hpp"
-#include "deep_search/commonutils.hpp"
-#include "util/rsdebug.h"
-#include "retroshare/rsinit.h"
-#include "retroshare/rsversion.h"
 
 #include <utility>
 
+#include "deep_search/filesindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "util/rsdebuglevel1.h"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsversion.h"
+
 /*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
 DeepFilesIndex::indexersRegister = {};
 
-bool DeepFilesIndex::indexFile(
+std::error_condition DeepFilesIndex::indexFile(
         const std::string& path, const std::string& name,
         const RsFileHash& hash )
 {
-	auto dbPtr = DeepSearch::openWritableDatabase(
-	            mDbPath, Xapian::DB_CREATE_OR_OPEN );
-	if(!dbPtr) return false;
-	Xapian::WritableDatabase& db(*dbPtr);
-
 	const std::string hashString = hash.toStdString();
 	const std::string idTerm("Q" + hashString);
 
-	Xapian::Document oldDoc;
+	auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
-	Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
+	if(db)
-	if( pIt != db.postlist_end(idTerm) )
 	{
-		oldDoc = db.get_document(*pIt);
+		Xapian::Document oldDoc;
+		Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
+		if( pIt != db->postlist_end(idTerm) )
+		{
+			oldDoc = db->get_document(*pIt);
 			if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
 			        RS_HUMAN_READABLE_VERSION &&
 			    std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
 			        indexersRegister.size() )
 			{
-			/* Looks like this file has already been indexed by this RetroShare
+				/* Looks like this file has already been indexed by this
-			 * exact version, so we can skip it. If the version was different it
+				 * RetroShare exact version, so we can skip it. If the version
-			 * made sense to reindex it as better indexers might be available
+				 * was different it made sense to reindex it as better indexers
-			 * since last time it was indexed */
+				 * might be available since last time it was indexed */
-			Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
+				RS_DBG3("skipping laready indexed file: ", hash, " ", name);
-			       << hash << " " << name << std::endl;
+				return std::error_condition();
-			return true;
 			}
 		}
+		db.reset(); // Release DB read lock ASAP
+	}
 
 	Xapian::Document doc;
 
@@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
 	doc.add_value(
 	            INDEXERS_COUNT_VALUENO,
 	            std::to_string(indexersRegister.size()) );
-	db.replace_document(idTerm, doc);
 
-	return true;
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); });
+
+	return std::error_condition();
 }
 
-bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
+std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 {
-	Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
+	RS_DBG3(hash);
-	       << hash << std::endl;
 
-	std::unique_ptr<Xapian::WritableDatabase> db =
+	mWriteQueue.push([hash](Xapian::WritableDatabase& db)
-	        DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
+	{ db.delete_document("Q" + hash.toStdString()); });
-	if(!db) return false;
 
-	db->delete_document("Q" + hash.toStdString());
+	return std::error_condition();
-	return true;
 }
 
 /*static*/ std::string DeepFilesIndex::dbDefaultPath()
@@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 /*static*/ bool DeepFilesIndex::registerIndexer(
         int order, const DeepFilesIndex::IndexerFunType& indexerFun )
 {
-	Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
+	RS_DBG1(order);
 
 	indexersRegister.insert(std::make_pair(order, indexerFun));
 	return true;
 }
 
-uint32_t DeepFilesIndex::search(
+std::error_condition DeepFilesIndex::search(
         const std::string& queryStr,
         std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
 {
 	results.clear();
 
 	auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
-	if(!dbPtr) return 0;
+	if(!dbPtr) return std::errc::bad_file_descriptor;
 	Xapian::Database& db(*dbPtr);
 
 	// Set up a QueryParser with a stemmer and suitable prefixes.
@@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
 		results.push_back(s);
 	}
 
-	return static_cast<uint32_t>(results.size());
+	return std::error_condition();
 }
 
 

libretroshare/src/deep_search/filesindex.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org>              *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -19,9 +19,6 @@
  *******************************************************************************/
 #pragma once
 
-#include "retroshare/rstypes.h"
-#include "util/rsdebug.h"
-
 #include <string>
 #include <cstdint>
 #include <vector>
@@ -29,6 +26,9 @@
 #include <map>
 #include <functional>
 
+#include "retroshare/rstypes.h"
+#include "deep_search/commonutils.hpp"
+
 struct DeepFilesSearchResult
 {
 	DeepFilesSearchResult() : mWeight(0) {}
@@ -41,7 +41,8 @@ struct DeepFilesSearchResult
 class DeepFilesIndex
 {
 public:
-	DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
+	explicit DeepFilesIndex(const std::string& dbPath):
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
 
 	/**
 	 * @brief Search indexed files
@@ -49,7 +50,7 @@ public:
 	 * no limits
 	 * @return search results count
 	 */
-	uint32_t search( const std::string& queryStr,
+	std::error_condition search( const std::string& queryStr,
 	                 std::vector<DeepFilesSearchResult>& results,
 	                 uint32_t maxResults = 100 );
 
@@ -57,7 +58,7 @@ public:
 	 * @return false if file could not be indexed because of error or
 	 *	unsupported type, true otherwise.
 	 */
-	bool indexFile(
+	std::error_condition indexFile(
 	        const std::string& path, const std::string& name,
 	        const RsFileHash& hash );
 
@@ -65,7 +66,7 @@ public:
 	 * @brief Remove file entry from database
 	 * @return false on error, true otherwise.
 	 */
-	bool removeFileFromIndex(const RsFileHash& hash);
+	std::error_condition removeFileFromIndex(const RsFileHash& hash);
 
 	static std::string dbDefaultPath();
 
@@ -96,8 +97,8 @@ private:
 
 	const std::string mDbPath;
 
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+
 	/** Storage for indexers function by order */
 	static std::multimap<int, IndexerFunType> indexersRegister;
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/deep_search/forumsindex.cpp

@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "deep_search/forumsindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsgxsforums.h"
+#include "util/rsdebuglevel4.h"
+
+std::error_condition DeepForumsIndex::search(
+        const std::string& queryStr,
+        std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
+{
+	results.clear();
+
+	std::unique_ptr<Xapian::Database> dbPtr(
+	            DeepSearch::openReadOnlyDatabase(mDbPath) );
+	if(!dbPtr) return std::errc::bad_file_descriptor;
+
+	Xapian::Database& db(*dbPtr);
+
+	// Set up a QueryParser with a stemmer and suitable prefixes.
+	Xapian::QueryParser queryparser;
+	//queryparser.set_stemmer(Xapian::Stem("en"));
+	queryparser.set_stemming_strategy(queryparser.STEM_SOME);
+	// Start of prefix configuration.
+	//queryparser.add_prefix("title", "S");
+	//queryparser.add_prefix("description", "XD");
+	// End of prefix configuration.
+
+	// And parse the query.
+	Xapian::Query query = queryparser.parse_query(queryStr);
+
+	// Use an Enquire object on the database to run the query.
+	Xapian::Enquire enquire(db);
+	enquire.set_query(query);
+
+	Xapian::MSet mset = enquire.get_mset(
+	            0, maxResults ? maxResults : db.get_doccount() );
+
+	for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
+	{
+		const Xapian::Document& doc = m.get_document();
+		DeepForumsSearchResult s;
+		s.mUrl = doc.get_value(URL_VALUENO);
+#if XAPIAN_AT_LEAST(1,3,5)
+		s.mSnippet = mset.snippet(doc.get_data());
+#endif // XAPIAN_AT_LEAST(1,3,5)
+		results.push_back(s);
+	}
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
+{
+	RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	forumIndexId.setQueryKV(
+	            RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
+	return forumIndexId.toString();
+}
+
+/*static*/ std::string DeepForumsIndex::postIndexId(
+        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
+{
+	RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
+	return postIndexId.toString();
+}
+
+std::error_condition DeepForumsIndex::indexForumGroup(
+        const RsGxsForumGroup& forum )
+{
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
+	termgenerator.index_text(forum.mDescription, 1, "XD");
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(forum.mMeta.mGroupName);
+	termgenerator.increase_termpos();
+	termgenerator.index_text(forum.mDescription);
+
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	/* Store some fields for display purposes. Retrieved later to provide the
+	 * matching snippet on search */
+	doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
+
+	/* We use the identifier to ensure each object ends up in the database only
+	 * once no matter how many times we run the indexer.
+	 * "Q" prefix is a Xapian convention for unique id term. */
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumFromIndex(
+        const RsGxsGroupId& grpId )
+{
+	mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
+	{ db.delete_document("Q" + forumIndexId(grpId)); });
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
+{
+	RS_DBG4(post);
+
+	const auto& groupId = post.mMeta.mGroupId;
+	const auto& msgId = post.mMeta.mMsgId;
+
+	if(groupId.isNull() || msgId.isNull())
+	{
+		RS_ERR("Got post with invalid id ", post);
+		print_stacktrace();
+		return std::errc::invalid_argument;
+	}
+
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
+
+	// Avoid indexing RetroShare-gui HTML tags
+	const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
+	termgenerator.index_text(cleanMsg, 1, "XD" );
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(post.mMeta.mMsgName);
+
+	termgenerator.increase_termpos();
+	termgenerator.index_text(cleanMsg);
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(postIndexId(groupId, msgId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	// Store some fields for display purposes.
+	doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
+
+	// We use the identifier to ensure each object ends up in the
+	// database only once no matter how many times we run the
+	// indexer.
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumPostFromIndex(
+        RsGxsGroupId grpId, RsGxsMessageId msgId )
+{
+	// "Q" prefix is a Xapian convention for unique id term.
+	std::string idTerm("Q" + postIndexId(grpId, msgId));
+	mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
+	{ db.delete_document(idTerm); } );
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::dbDefaultPath()
+{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

libretroshare/src/deep_search/forumsindex.hpp

@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#pragma once
+
+#include <system_error>
+#include <vector>
+#include <xapian.h>
+
+#include "util/rstime.h"
+#include "retroshare/rsgxsforums.h"
+#include "retroshare/rsevents.h"
+#include "deep_search/commonutils.hpp"
+
+struct DeepForumsSearchResult
+{
+	std::string mUrl;
+	double mWeight;
+	std::string mSnippet;
+};
+
+struct DeepForumsIndex
+{
+	explicit DeepForumsIndex(const std::string& dbPath) :
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
+
+	/**
+	 * @brief Search indexed GXS groups and messages
+	 * @param[in] maxResults maximum number of acceptable search results, 0 for
+	 * no limits
+	 * @return search results count
+	 */
+	std::error_condition search( const std::string& queryStr,
+	                             std::vector<DeepForumsSearchResult>& results,
+	                             uint32_t maxResults = 100 );
+
+	std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
+
+	std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
+
+	std::error_condition indexForumPost(const RsGxsForumMsg& post);
+
+	std::error_condition removeForumPostFromIndex(
+	        RsGxsGroupId grpId, RsGxsMessageId msgId );
+
+	static std::string dbDefaultPath();
+
+private:
+	static std::string forumIndexId(const RsGxsGroupId& grpId);
+	static std::string postIndexId(
+	        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
+
+	enum : Xapian::valueno
+	{
+		/// Used to store retroshare url of indexed documents
+		URL_VALUENO,
+
+		/// @see Xapian::BAD_VALUENO
+		BAD_VALUENO = Xapian::BAD_VALUENO
+	};
+
+	const std::string mDbPath;
+
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+};

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -29,6 +31,7 @@
 #include "dir_hierarchy.h"
 #include "filelist_io.h"
 #include "file_sharing_defaults.h"
+#include "util/cxx17retrocompat.h"
 
 #ifdef RS_DEEP_FILES_INDEX
 #	include "deep_search/filesindex.hpp"
@@ -70,7 +73,8 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
     mTotalFiles = 0 ;
 }
 
-bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
+bool InternalFileHierarchyStorage::getDirHashFromIndex(
+        const DirectoryStorage::EntryIndex& index, RsFileHash& hash ) const
 {
     if(!checkIndex(index,FileStorageNode::TYPE_DIR))
         return false ;
@@ -79,6 +83,7 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E
 
     return true;
 }
+
 bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
 {
     std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mDirHashes.find(hash) ;
@@ -88,32 +93,36 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di
 
     index = it->second;
 
-    // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them.
+	/* make sure the hash actually points to some existing directory. If not,
-    if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast<DirEntry*>(mNodes[index])->dir_hash != hash)
+	 * remove it. This is an opportunistic update of dir hashes: when we need
+	 * them, we check them. */
+	if( !checkIndex(index, FileStorageNode::TYPE_DIR) ||
+	        static_cast<DirEntry*>(mNodes[index])->dir_hash != hash )
 	{
-        std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl;
+		RS_INFO("removing non existing dir hash: ", hash, " from dir hash list");
-
+		mDirHashes.erase(it);
-        mDirHashes.erase(it) ;
+		return false;
-        return false ;
 	}
     return true;
 }
-bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
-{
-    std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mFileHashes.find(hash) ;
 
-    if(it == mFileHashes.end())
+bool InternalFileHierarchyStorage::getIndexFromFileHash(
-        return false;
+        const RsFileHash& hash, DirectoryStorage::EntryIndex& index )
+{
+	auto it = std::as_const(mFileHashes).find(hash);
+	if(it == mFileHashes.end()) return false;
 
 	index = it->second;
 
-    // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them.
+	/* make sure the hash actually points to some existing file. If not, remove
-    if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast<FileEntry*>(mNodes[index])->file_hash != hash)
+	 * it. This is an opportunistic update of file hashes: when we need them,
+	 * we check them. */
+	if( !checkIndex(it->second, FileStorageNode::TYPE_FILE) ||
+	        static_cast<FileEntry*>(mNodes[index])->file_hash != hash )
 	{
-        std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl;
+		RS_INFO("removing non existing file hash: ", hash, " from file hash list");
-
+		mFileHashes.erase(it);
-        mFileHashes.erase(it) ;
+		return false;
-        return false ;
 	}
 
 	return true;
@@ -155,7 +164,10 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
     return e < mNodes.size() && mNodes[e] != NULL ;
 }
 
-bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::set<std::string>& subdirs, const RsFileHash& random_hash_seed)
+bool InternalFileHierarchyStorage::updateSubDirectoryList(
+        const DirectoryStorage::EntryIndex& indx,
+        const std::set<std::string>& subdirs,
+        const RsFileHash& random_hash_seed )
 {
 	if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
 		return false;
@@ -284,10 +296,17 @@ bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,
     return true;
 }
 
-bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
+bool InternalFileHierarchyStorage::updateSubFilesList(
+        const DirectoryStorage::EntryIndex& indx,
+        const std::map<std::string,DirectoryStorage::FileTS>& subfiles,
+        std::map<std::string,DirectoryStorage::FileTS>& new_files )
 {
-    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+	if(!checkIndex(indx, FileStorageNode::TYPE_DIR))
+	{
+		RS_ERR("indx: ", indx, std::errc::not_a_directory);
+		print_stacktrace();
 		return false;
+	}
 
     DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
     new_files = subfiles ;
@@ -312,9 +331,11 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
             continue;
         }
 
-        if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)	// file is newer and/or has different size
+		// file is newer and/or has different size
+		if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)
         {
-            f.file_hash.clear();																// hash needs recomputing
+			// hash needs recomputing
+			f.file_hash.clear();
             f.file_modtime = it->second.modtime;
             f.file_size = it->second.size;
 
@@ -342,11 +363,14 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
     }
     return true;
 }
-bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
+bool InternalFileHierarchyStorage::updateHash(
+        const DirectoryStorage::EntryIndex& file_index, const RsFileHash& hash )
 {
-    if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
+	if(!checkIndex(file_index, FileStorageNode::TYPE_FILE))
 	{
-        std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
+		RS_ERR( "Cannot update file at index ", file_index,
+		        ". Not a valid index, or not a file." );
+		print_stacktrace();
 		return false;
 	}
 #ifdef DEBUG_DIRECTORY_STORAGE
@@ -436,14 +460,20 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
 	return mNodes.size()-1 ;
 }
 
-bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,rstime_t most_recent_time,rstime_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
+bool InternalFileHierarchyStorage::updateDirEntry(
+        const DirectoryStorage::EntryIndex& indx, const std::string& dir_name,
+        rstime_t most_recent_time, rstime_t dir_modtime,
+        const std::vector<RsFileHash>& subdirs_hash,
+        const std::vector<FileEntry>& subfiles_array )
 {
 	if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
 	{
-        std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
+		RS_ERR( "cannot update dir at index ", indx, ". Not a valid index, or "
+		        "not an existing dir." );
 		return false;
 	}
-    DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
+
+	DirEntry& d(*static_cast<DirEntry*>(mNodes[indx]));
 
 #ifdef DEBUG_DIRECTORY_STORAGE
     std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl;
@@ -703,14 +733,14 @@ const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorag
         return NULL ;
 }
 
-const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
+const InternalFileHierarchyStorage::DirEntry*
+InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
 {
-    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+	if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return nullptr;
-        return NULL ;
+	return static_cast<DirEntry*>(mNodes[indx]);
-
-    return static_cast<DirEntry*>(mNodes[indx]) ;
 }
-const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
+const InternalFileHierarchyStorage::FileEntry*
+InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
 {
     if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
         return NULL ;
@@ -754,7 +784,10 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,DirectorySt
 class DirectoryStorageExprFileEntry: public RsRegularExpression::ExpFileEntry
 {
 public:
-    DirectoryStorageExprFileEntry(const InternalFileHierarchyStorage::FileEntry& fe,const InternalFileHierarchyStorage::DirEntry& parent) : mFe(fe),mDe(parent) {}
+	DirectoryStorageExprFileEntry(
+	        const InternalFileHierarchyStorage::FileEntry& fe,
+	        const InternalFileHierarchyStorage::DirEntry& parent ) :
+	    mFe(fe), mDe(parent) {}
 
     inline virtual const std::string& file_name()       const { return mFe.file_name ; }
     inline virtual uint64_t           file_size()       const { return mFe.file_size ; }
@@ -768,14 +801,18 @@ private:
     const InternalFileHierarchyStorage::DirEntry& mDe ;
 };
 
-int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const
+int InternalFileHierarchyStorage::searchBoolExp(
+        RsRegularExpression::Expression* exp,
+        std::list<DirectoryStorage::EntryIndex>& results ) const
 {
-    for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
+	for(auto& it: std::as_const(mFileHashes))
-        if(mNodes[it->second] != NULL && exp->eval(
+		if(mNodes[it.second])
-                    DirectoryStorageExprFileEntry(*static_cast<const FileEntry*>(mNodes[it->second]),
+			if(exp->eval(
-                                                  *static_cast<const DirEntry*>(mNodes[mNodes[it->second]->parent_index])
+			            DirectoryStorageExprFileEntry(
-                                                  )))
+			                *static_cast<const FileEntry*>(mNodes[it.second]),
-            results.push_back(it->second);
+			                *static_cast<const DirEntry*>(mNodes[mNodes[it.second]->parent_index])
+			                                      ) ))
+			results.push_back(it.second);
 
     return 0;
 }
@@ -784,27 +821,43 @@ int InternalFileHierarchyStorage::searchTerms(
         const std::list<std::string>& terms,
         std::list<DirectoryStorage::EntryIndex>& results ) const
 {
-    // most entries are likely to be files, so we could do a linear search over the entries tab.
+	/* most entries are likely to be files, so we could do a linear search over
-    // instead we go through the table of hashes.
+	 * the entries tab. Instead we go through the table of hashes.*/
 
-    for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
+	for(auto& it : std::as_const(mFileHashes))
-        if(mNodes[it->second] != NULL)
 	{
-            const std::string &str1 = static_cast<FileEntry*>(mNodes[it->second])->file_name;
+		// node may be null for some hash waiting to be deleted
+		if(mNodes[it.second])
+		{
+			rs_view_ptr<FileEntry> tFileEntry =
+			        static_cast<FileEntry*>(mNodes[it.second]);
 
-            for(std::list<std::string>::const_iterator iter(terms.begin()); iter != terms.end(); ++iter)
+			/* Most file will just have file name stored, but single file shared
+			 * without a shared dir will contain full path instead of just the
+			 * name, so purify it to perform the search */
+			std::string tFilename = tFileEntry->file_name;
+			if(tFileEntry->file_name.find("/") != std::string::npos)
+			{
+				std::string _tParentDir;
+				RsDirUtil::splitDirFromFile(
+				            tFileEntry->file_name, _tParentDir, tFilename );
+			}
+
+			for(auto& termIt : std::as_const(terms))
 			{
 				/* always ignore case */
-                const std::string &str2 = (*iter);
+				if(tFilename.end() != std::search(
-
+				            tFilename.begin(), tFilename.end(),
-                if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() ))
+				            termIt.begin(), termIt.end(),
+				            RsRegularExpression::CompareCharIC() ))
 				{
-                    results.push_back(it->second);
+					results.push_back(it.second);
 					break;
 				}
 			}
 		}
-    return 0 ;
+	}
+	return 0;
 }
 
 bool InternalFileHierarchyStorage::check(std::string& error_string) // checks consistency of storage.
@@ -958,8 +1011,9 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
 
 bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
 {
-    std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
+	RS_ERR(s);
-    return false ;
+	print_stacktrace();
+	return false;
 }
 
 // Removes the given subdirectory from the parent node and all its pendign subdirs and files.

libretroshare/src/file_sharing/directory_storage.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -30,6 +32,7 @@
 #include "directory_storage.h"
 #include "dir_hierarchy.h"
 #include "filelist_io.h"
+#include "util/cxx17retrocompat.h"
 
 #ifdef RS_DEEP_FILES_INDEX
 #	include "deep_search/filesindex.hpp"
@@ -67,8 +70,11 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
 
     return *this;
 }
-DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
+DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
-DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
+{ return mStorage->getSubFileIndex(mParentIndex, mFileTabIndex); }
+
+DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
+{ return mStorage->getSubDirIndex(mParentIndex, mDirTabIndex); }
 
 DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
 DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
@@ -139,7 +145,9 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx, const std:
     mChanged = true ;
     return res ;
 }
-bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
+bool DirectoryStorage::updateSubFilesList(
+        const EntryIndex& indx, const std::map<std::string,FileTS>& subfiles,
+        std::map<std::string,FileTS>& new_files )
 {
     RS_STACK_MUTEX(mDirStorageMtx) ;
     bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
@@ -348,7 +356,8 @@ int LocalDirectoryStorage::searchHash(const RsFileHash& hash, RsFileHash& real_h
     return false ;
 }
 
-void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
+void LocalDirectoryStorage::setSharedDirectoryList(
+        const std::list<SharedDirInfo>& lst )
 {
 	std::set<std::string> dirs_with_new_virtualname ;
     bool dirs_with_changed_flags = false ;
@@ -379,7 +388,9 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo
 			virtual_names.insert(candidate_virtual_name) ;
 		}
 
-		// now for each member of the processed list, check if it is an existing shared directory that has been changed. If so, we need to update the dir TS of that directory
+		/* now for each member of the processed list, check if it is an existing
+		 * shared directory that has been changed. If so, we need to update the
+		 * dir TS of that directory */
 
 		std::map<std::string,SharedDirInfo> new_dirs ;
 
@@ -529,7 +540,7 @@ bool LocalDirectoryStorage::updateHash(
 #endif
 
 		ret = (!update_internal_hierarchy) ||
-		        mFileHierarchy->updateHash(index,hash);
+		        mFileHierarchy->updateHash(index, hash);
 	} // RS_STACK_MUTEX(mDirStorageMtx);
 
 #ifdef RS_DEEP_FILES_INDEX
@@ -538,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
 	        fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
 	{
 		DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-		ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
+		ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
 	}
 #endif // def RS_DEEP_FILES_INDEX
 
@@ -617,46 +628,76 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil
     return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
 }
 
-bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
+bool LocalDirectoryStorage::locked_getFileSharingPermissions(
+        const EntryIndex& indx, FileStorageFlags& flags,
+        std::list<RsNodeGroupId>& parent_groups )
 {
-    flags.clear() ;
+	flags.clear();
 	parent_groups.clear();
 
-    std::string base_dir;
+	/* We got a request for root directory no need to do anything more after
+	 * clearing outputs */
+	if(!indx) return true;
 
-    const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
+	using FileStorageNode = InternalFileHierarchyStorage::FileStorageNode;
+	using EntryIndex = DirectoryStorage::EntryIndex;
 
-    if(n == NULL)
+	rs_view_ptr<const FileStorageNode> n = mFileHierarchy->getNode(indx);
-        return false ;
+	if(!n)
-
-    for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
 	{
-        const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
+		RS_ERR("Node for index: ", indx, "not found");
-
+		print_stacktrace();
-        if(e == NULL)
+		return false;
-            break ;
-
-        if(e->parent_index == 0)
-        {
-            base_dir = e->dir_name ;
-            break ;
-        }
-        i = e->parent_index ;
 	}
 
-    if(!base_dir.empty())
+	// Climb down node tree up to root + 1
+	EntryIndex curIndex = indx;
+	while (n->parent_index)
 	{
-        std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
+		curIndex = n->parent_index;
+		n = mFileHierarchy->getNode(curIndex);
+	}
+
+	// Retrieve base name
+	std::string tBaseName;
+	switch (n->type())
+	{
+	// Handle single file shared case
+	case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE:
+		tBaseName = mFileHierarchy->getFileEntry(curIndex)->file_name;
+		break;
+	// Handle shared directory case
+	case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR:
+		tBaseName = mFileHierarchy->getDirEntry(curIndex)->dir_name;
+		break;
+	default:
+		RS_ERR("Got unhandled node type: ", n->type());
+		print_stacktrace();
+		return false;
+	}
+
+	// Use base name to retrieve sharing permissions
+	if(!tBaseName.empty())
+	{
+		auto it = std::as_const(mLocalDirs).find(tBaseName);
 
 		if(it == mLocalDirs.end())
 		{
-            std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
+			RS_ERR( "base name \"", tBaseName,
-            return false ;
+			        "\" for index: ", indx, " not found in shared dir list." );
+			print_stacktrace();
+			return false;
 		}
 
 		flags = it->second.shareflags;
 		parent_groups = it->second.parent_groups;
 	}
+	else
+	{
+		RS_ERR("base name for indx: ", indx, " is empty");
+		print_stacktrace();
+		return false;
+	}
 
 	return true;
 }
@@ -705,38 +746,52 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
    return it->second.virtualname + "/" + res;
 }
 
-bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
+bool LocalDirectoryStorage::serialiseDirEntry(
+        const EntryIndex& indx, RsTlvBinaryData& bindata,
+        const RsPeerId& client_id )
 {
-    RS_STACK_MUTEX(mDirStorageMtx) ;
+	RS_STACK_MUTEX(mDirStorageMtx);
 
-    const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
+	const InternalFileHierarchyStorage::DirEntry* dir =
+	        mFileHierarchy->getDirEntry(indx);
 
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
     std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl;
 #endif
-    if(dir == NULL)
+
+	if(!dir)
 	{
-        std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
+		RS_ERR("Cannot find entry ", indx);
 		return false;
 	}
 
 	// compute list of allowed subdirs
-    std::vector<RsFileHash> allowed_subdirs ;
+	std::vector<RsFileHash> allowed_subdirs;
-    FileStorageFlags node_flags ;
+	FileStorageFlags node_flags;
-    std::list<RsNodeGroupId> node_groups ;
+	std::list<RsNodeGroupId> node_groups;
 
-    // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
+	/* for each subdir, compute the node flags and groups, then ask rsPeers to
+	 * compute the mask that result from these flags for the particular peer
+	 * supplied in parameter */
 
 	for(uint32_t i=0;i<dir->subdirs.size();++i)
-        if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
+		if(indx != 0 || (
+		            locked_getFileSharingPermissions(
+		                dir->subdirs[i], node_flags, node_groups ) &&
+		            ( rsPeers->computePeerPermissionFlags(
+		                  client_id, node_flags, node_groups ) &
+		              RS_FILE_HINTS_BROWSABLE ) ))
 		{
-            RsFileHash hash ;
+			RsFileHash hash;
 			if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
 			{
-              std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
+				RS_ERR( "Cannot get hash from subdir index: ",
+				        dir->subdirs[i], ". Weird bug." );
+				print_stacktrace();
 				return false;
 			}
-            allowed_subdirs.push_back(hash) ;
+			allowed_subdirs.push_back(hash);
+
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
             std::cerr << "  pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl;
 #endif
@@ -746,93 +801,159 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
             std::cerr << "  not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl;
 #endif
 
-    // now count the files that do not have a null hash (meaning the hash has indeed been computed)
+	/* now count the files that do not have a null hash (meaning the hash has
-
+	 * indeed been computed), also in case files are shared singularly (without
-    uint32_t allowed_subfiles = 0 ;
+	 * a shared directory) so they are child of root check browsability
-
+	 * permission */
-    for(uint32_t i=0;i<dir->subfiles.size();++i)
+	uint32_t allowed_subfiles = 0;
+	for(uint32_t i=0; i<dir->subfiles.size(); ++i)
 	{
-        const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
+		const InternalFileHierarchyStorage::FileEntry* file =
-        if(file != NULL && !file->file_hash.isNull())
+		        mFileHierarchy->getFileEntry(dir->subfiles[i]);
-            allowed_subfiles++ ;
+		if(file != nullptr && !file->file_hash.isNull()
+		        && ( indx !=0 || (
+		                 locked_getFileSharingPermissions(
+		                     dir->subfiles[i], node_flags, node_groups ) &&
+		                 rsPeers->computePeerPermissionFlags(
+		                     client_id, node_flags, node_groups ) &
+		                 RS_FILE_HINTS_BROWSABLE ) ))
+			allowed_subfiles++;
 	}
 
-    unsigned char *section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
+	unsigned char* section_data = (unsigned char *)
-
+	        rs_malloc(FL_BASE_TMP_SECTION_SIZE);
-    if(!section_data)
+	if(!section_data) return false;
-        return false ;
 
 	uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
 	uint32_t section_offset = 0;
 
-    // we need to send:
+	/* we need to send:
-    //	- the name of the directory, its TS
+	 *  - the name of the directory, its TS
-    //	- the index entry for each subdir (the updte TS are exchanged at a higher level)
+	 *  - the index entry for each subdir (the updte TS are exchanged at a
-    //	- the file info for each subfile
+	 *    higher level)
-    //
+	 *  - the file info for each subfile */
-    std::string virtual_dir_name = locked_getVirtualDirName(indx) ;
 
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME       ,virtual_dir_name                   )) { free(section_data); return false ;}
+	std::string virtual_dir_name = locked_getVirtualDirName(indx);
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) { free(section_data); return false ;}
+
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS       ,(uint32_t)dir->dir_modtime         )) { free(section_data); return false ;}
+	/* Manual serialization AGAIN! This is terrible and should be ported to the
+	 * new serialization system ASAP! */
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_DIR_NAME, virtual_dir_name ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_RECURS_MODIF_TS,
+	            (uint32_t)dir->dir_most_recent_time ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)dir->dir_modtime ))
+	{ free(section_data); return false;}
 
 	// serialise number of subdirs and number of subfiles
-
+	if(!FileListIO::writeField(
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size()  )) { free(section_data); return false ;}
+	            section_data, section_size, section_offset,
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles        )) { free(section_data); return false ;}
+	            FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subdirs.size() ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subfiles ))
+	{ free(section_data); return false; }
 
 	// serialise subdirs entry indexes
-
+	for(uint32_t i=0; i<allowed_subdirs.size(); ++i)
-    for(uint32_t i=0;i<allowed_subdirs.size();++i)
+		if(!FileListIO::writeField(
-        if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i]  )) { free(section_data); return false ;}
+		            section_data, section_size, section_offset,
+		            FILE_LIST_IO_TAG_ENTRY_INDEX, allowed_subdirs[i] ))
+		{ free(section_data); return false; }
 
 	// serialise directory subfiles, with info for each of them
 
-    unsigned char *file_section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
+	unsigned char* file_section_data =
-
+	        (unsigned char *) rs_malloc(FL_BASE_TMP_SECTION_SIZE);
 	if(!file_section_data)
 	{
 		free(section_data);
-        return false ;
+		return false;
 	}
 
-    uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE ;
+	uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE;
 
-    for(uint32_t i=0;i<dir->subfiles.size();++i)
+	for(uint32_t i=0; i<dir->subfiles.size(); ++i)
 	{
-        uint32_t file_section_offset = 0 ;
+		uint32_t file_section_offset = 0;
 
-        const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
+		const InternalFileHierarchyStorage::FileEntry* file =
+		        mFileHierarchy->getFileEntry(dir->subfiles[i]);
 
-        if(file == NULL || file->file_hash.isNull())
+		if(file == nullptr || file->file_hash.isNull())
 		{
-            std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
+			RS_INFO( "skipping unhashed or Null file entry ",
-            continue ;
+			         dir->subfiles[i], " to get/send file info." );
+			continue;
 		}
 
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME     ,file->file_name   )) { free(section_data);free(file_section_data);return false ;}
+		if(indx == 0)
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE     ,file->file_size   )) { free(section_data);free(file_section_data);return false ;}
+		{
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash   )) { free(section_data);free(file_section_data);return false ;}
+			if(!locked_getFileSharingPermissions(
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS      ,(uint32_t)file->file_modtime)) { free(section_data);free(file_section_data);return false ;}
+			            dir->subfiles[i], node_flags, node_groups ))
+			{
+				RS_ERR( "Failure getting sharing permission for single file: ",
+				        dir->subfiles[i] );
+				print_stacktrace();
+				continue;
+			}
+
+			if(!( rsPeers->computePeerPermissionFlags(
+			          client_id, node_flags, node_groups ) &
+			      RS_FILE_HINTS_BROWSABLE ))
+			{
+				RS_INFO( "Skipping single file shared without browse "
+				         "permission" );
+				continue;
+			}
+		}
+
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_NAME, file->file_name ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_SIZE, file->file_size ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_SHA1_HASH, file->file_hash ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)file->file_modtime ))
+		{ free(section_data); free(file_section_data); return false; }
 
 		// now write the whole string into a single section in the file
-
+		if(!FileListIO::writeField(
-        if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) { free(section_data); free(file_section_data);return false ;}
+		            section_data, section_size, section_offset,
+		            FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,
+		            file_section_data, file_section_offset ))
+		{ free(section_data); free(file_section_data); return false; }
 
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
         std::cerr << "  pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl;
 #endif
 	}
-	free(file_section_data) ;
+	free(file_section_data);
 
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
     std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
 #endif
 
-    bindata.bin_data = realloc(section_data,section_offset) ;	// This discards the possibly unused trailing bytes in the end of section_data
+	// Discards the possibly unused trailing bytes in the end of section_data
-    bindata.bin_len = section_offset ;
+	bindata.bin_data = realloc(section_data,section_offset);
+	bindata.bin_len = section_offset;
 
-    return true ;
+	return true;
 }
 
 

libretroshare/src/file_sharing/directory_updater.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,13 +21,15 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  ******************************************************************************/
+
+#include "util/cxx17retrocompat.h"
 #include "util/folderiterator.h"
 #include "util/rstime.h"
 #include "rsserver/p3face.h"
-
 #include "directory_storage.h"
 #include "directory_updater.h"
 #include "file_sharing_defaults.h"
+#include "util/rsdebuglevel3.h"
 
 //#define DEBUG_LOCAL_DIR_UPDATER 1
 
@@ -123,183 +127,236 @@ bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
 {
 	if(mHashSalt.isNull())
 	{
-        std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
+		RS_ERR("no salt value in LocalDirectoryUpdater");
+		print_stacktrace();
 		return false;
 	}
 
-    mIsChecking = true ;
+	mIsChecking = true;
 
 	RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-    std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
-#endif
 
-    // recursive update algorithm works that way:
+	/* recursive update algorithm works that way:
-    // 	- the external loop starts on the shared directory list and goes through sub-directories
+	 * - the external loop starts on the shared directory list and goes through
-    // 	- at the same time, it updates the local list of shared directories.  A single sweep is performed over the whole directory structure.
+	 *   sub-directories
-    // 	- the information that is costly to compute (the hash) is store externally into a separate structure.
+	 * - at the same time, it updates the local list of shared directories.
-    // 	- doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
+	 *   A single sweep is performed over the whole directory structure.
-    //
+	 * - the information that is costly to compute (the hash) is stored
-    std::list<SharedDirInfo> shared_directory_list ;
+	 *   externally into a separate structure.
+	 * - doing so, changing directory names or moving files between directories
+	 * does not cause a re-hash of the content. */
+	std::list<SharedDirInfo> shared_directory_list;
 	mSharedDirectories->getSharedDirectoryList(shared_directory_list);
+	std::set<std::string> sub_dir_list;
 
-    std::set<std::string> sub_dir_list ;
+	/* Support also single files sharing as it make much more sense on some
+	 * platforms like Android */
+	std::map<std::string, DirectoryStorage::FileTS> singleFilesMap;
 
-    // We re-check that each dir actually exists. It might have been removed from the disk.
+	/* We re-check that each dir actually exists. It might have been removed
+	 * from the disk. Accept also single files not just directories. */
+	for(auto& realDir: std::as_const(shared_directory_list))
+	{
+		const auto& fPath = realDir.filename;
+		if(RsDirUtil::checkDirectory(fPath))
+			sub_dir_list.insert(fPath);
+		else if (RsDirUtil::fileExists(fPath))
+		{
+			rstime_t lastWrite= RsDirUtil::lastWriteTime(fPath);
+			if(time(nullptr) >= lastWrite + MIN_TIME_AFTER_LAST_MODIFICATION)
+			{
+				uint64_t fSize = 0;
+				RsDirUtil::checkFile(fPath,fSize);
 
-    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
+				singleFilesMap[fPath].modtime = lastWrite;
-        if(RsDirUtil::checkDirectory( (*real_dir_it).filename ) )
+				singleFilesMap[fPath].size = fSize;
-		        sub_dir_list.insert( (*real_dir_it).filename ) ;
+			}
+			else
+			{
+				some_files_not_ready = true;
+				RS_INFO( "file: \"", fPath, "\" is "
+				         "probably being written to. Keep it for later");
+			}
+		}
+		else RS_WARN( "Got non existent file \"", fPath,
+		              "\" in shared directories list. Ignored." );
+	}
 
-    // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
+	{
+		const auto tRoot = mSharedDirectories->root();
+		std::map<std::string, DirectoryStorage::FileTS> needsUpdate;
+		mSharedDirectories->updateSubFilesList(
+		            tRoot, singleFilesMap, needsUpdate);
 
-    mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
+		for( DirectoryStorage::FileIterator storedSingleFilesIt(
+		         mSharedDirectories, mSharedDirectories->root() );
+		     storedSingleFilesIt; ++storedSingleFilesIt )
+		{
+			const auto& it = storedSingleFilesIt;
+			RsFileHash hash;
+			if( mHashCache->requestHash(
+			            it.name(), it.size(), it.modtime(), hash,
+			            this, *it ) )
+				mSharedDirectories->updateHash(*it, hash, it.hash() != hash);
+		}
+	}
+
+	/* make sure that entries in stored_dir_it are the same than paths in
+	 * real_dir_it, and in the same order. */
+	mSharedDirectories->updateSubDirectoryList(
+	            mSharedDirectories->root(), sub_dir_list, mHashSalt );
 
 	// now for each of them, go recursively and match both files and dirs
-
+	std::set<std::string> existing_dirs;
-    std::set<std::string> existing_dirs ;
+	for( DirectoryStorage::DirIterator stored_dir_it(
-
+	         mSharedDirectories, mSharedDirectories->root() );
-    for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
+	     stored_dir_it; ++stored_dir_it )
 	{
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+		RS_DBG4("recursing into \"", stored_dir_it.name());
-        std::cerr << "[directory storage]   recursing into " << stored_dir_it.name() << std::endl;
-#endif
-		existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
 
-        recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ;		// here we need to use the list that was stored, instead of the shared dir list, because the two
+		existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
-                                                                            									// are not necessarily in the same order.
+		recursUpdateSharedDir(
+		            stored_dir_it.name(), *stored_dir_it,
+		            existing_dirs, 1, some_files_not_ready );
+		/* here we need to use the list that was stored, instead of the shared
+		 * dir list, because the two are not necessarily in the same order. */
 	}
 
 	RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
-    mIsChecking = false ;
+	mIsChecking = false;
 
-    return true ;
+	return true;
 }
 
-void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
+void LocalDirectoryUpdater::recursUpdateSharedDir(
+        const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,
+        std::set<std::string>& existing_directories, uint32_t current_depth,
+        bool& some_files_not_ready )
 {
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+	RS_DBG4("parsing directory \"", cumulated_path, "\" index: ", indx);
-    std::cerr << "[directory storage]   parsing directory " << cumulated_path << ", index=" << indx << std::endl;
-#endif
 
-    // make sure list of subdirs is the same
+	/* make sure list of subdirs is the same
-    // make sure list of subfiles is the same
+	 * make sure list of subfiles is the same
-    // request all hashes to the hashcache
+	 * request all hashes to the hashcache */
 
-    librs::util::FolderIterator dirIt(cumulated_path,mFollowSymLinks,false);	// disallow symbolic links and files from the future.
+	// disallow symbolic links and files from the future.
+	librs::util::FolderIterator dirIt(cumulated_path, mFollowSymLinks, false);
 
-    rstime_t dir_local_mod_time ;
+	rstime_t dir_local_mod_time;
 	if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
 	{
-        std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
+		RS_ERR("Cannot get local mod time for dir index: ", indx);
+		print_stacktrace();
 		return;
 	}
 
-    rstime_t now = time(NULL) ;
+	rstime_t now = time(nullptr);
-
+	/* the > is because we may have changed the virtual name, and therefore the
-    if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)	// the > is because we may have changed the virtual name, and therefore the TS wont match.
+	 * TS wont match. We only want to detect when the directory has changed on
-																		// we only want to detect when the directory has changed on the disk
+	 * the disk */
+	if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)
 	{
 		// collect subdirs and subfiles
+		std::map<std::string, DirectoryStorage::FileTS> subfiles;
+		std::set<std::string> subdirs;
 
-       std::map<std::string,DirectoryStorage::FileTS> subfiles ;
+		for( ; dirIt.isValid(); dirIt.next() )
-       std::set<std::string> subdirs ;
-
-       for(;dirIt.isValid();dirIt.next())
 			if(filterFile(dirIt.file_name()))
 			{
-			   switch(dirIt.file_type())
+				const auto fType = dirIt.file_type();
+				switch(fType)
 				{
 				case librs::util::FolderIterator::TYPE_FILE:
-
+					if(now >= dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION)
-                   if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
 					{
-					   subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
+						subfiles[dirIt.file_name()].modtime = dirIt.file_modtime();
 						subfiles[dirIt.file_name()].size = dirIt.file_size();
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+						RS_DBG4("adding sub-file \"", dirIt.file_name(), "\"");
-					   std::cerr << "  adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
-#endif
 					}
 					else
 					{
-                       some_files_not_ready = true ;
+						some_files_not_ready = true;
-
+						RS_INFO( "file: \"", dirIt.file_fullpath(), "\" is "
-                       std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being written to. Keeping it for later." << std::endl;
+						         "probably being written to. Keep it for later");
 					}
-
 					break;
-
 				case librs::util::FolderIterator::TYPE_DIR:
 				{
-				   bool dir_is_accepted = true ;
+					bool dir_is_accepted = true;
-
+					/* 64 is here as a safe limit, to make infinite loops
-				   if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64))	// 64 is here as a safe limit, to make loops impossible.
+					 * impossible.
-					   dir_is_accepted = false ;
+					 * TODO: Make it a visible constexpr in the header */
+					if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth)
+					        || (mMaxShareDepth == 0 && current_depth >= 64) )
+						dir_is_accepted = false;
 
 					if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
 					{
-					   std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
+						std::string real_path = RsDirUtil::removeSymLinks(
+						            cumulated_path + "/" + dirIt.file_name() );
 
-					   if(existing_directories.end() != existing_directories.find(real_path))
+						if( existing_directories.end() !=
+						        existing_directories.find(real_path) )
 						{
-						   std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
+							RS_WARN( "Directory: \"", cumulated_path,
-						   dir_is_accepted = false ;
+							         "\" has real path: \"", real_path,
+							         "\" which already belongs to another "
+							         "shared directory. Ignoring" );
+							dir_is_accepted = false;
 						}
-					   else
+						else existing_directories.insert(real_path);
-						   existing_directories.insert(real_path) ;
 					}
 
-				   if(dir_is_accepted)
+					if(dir_is_accepted) subdirs.insert(dirIt.file_name());
-					   subdirs.insert(dirIt.file_name());
+
+					RS_DBG4("adding sub-dir \"", dirIt.file_name(), "\"");
 
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-				   std::cerr << "  adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
-#endif
-			   }
 					break;
+				}
 				default:
-				   std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
+					RS_ERR( "Got Dir entry of unknown type:", fType,
+					        "with path \"", cumulated_path, "/",
+					        dirIt.file_name(), "\"" );
+					print_stacktrace();
+					break;
 				}
 			}
-       // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
 
-       mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
+		/* update folder modificatoin time, which is the only way to detect
+		 * e.g. removed or renamed files. */
+		mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime());
 
 		// update file and dir lists for current directory.
+		mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt);
 
-       mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ;
+		std::map<std::string, DirectoryStorage::FileTS> new_files;
-
+		mSharedDirectories->updateSubFilesList(indx, subfiles, new_files);
-       std::map<std::string,DirectoryStorage::FileTS> new_files ;
-       mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
 
 		// now go through list of subfiles and request the hash to hashcache
-
+		for( DirectoryStorage::FileIterator dit(mSharedDirectories,indx);
-       for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
+		     dit; ++dit )
 		{
-		   // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
+			/* ask about the hash. If not present, ask HashCache.
+			 * If not present, or different, the callback will update it. */
+			RsFileHash hash;
 
-		   RsFileHash hash ;
+			/* mSharedDirectories does two things: store H(F), and
+			 * compute H(H(F)), which is used in FT.
+			 * The later is always needed. */
 
-		   // mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed.
+			if( mHashCache->requestHash(
-
+			            cumulated_path + "/" + dit.name(),
-		   if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit))
+			            dit.size(), dit.modtime(), hash, this, *dit ) )
-			   mSharedDirectories->updateHash(*dit,hash,hash != dit.hash());
+				mSharedDirectories->updateHash(*dit, hash, hash != dit.hash());
 		}
 	}
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-    else
-        std::cerr << "  directory is unchanged. Keeping existing files and subdirs list." << std::endl;
-#endif
 
 	// go through the list of sub-dirs and recursively update
-
+	for( DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories, indx);
-		for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
+	     stored_dir_it; ++stored_dir_it )
-		{
+		recursUpdateSharedDir( cumulated_path + "/" + stored_dir_it.name(),
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+		                       *stored_dir_it, existing_directories,
-			std::cerr << "  recursing into " << stored_dir_it.name() << std::endl;
+		                       current_depth+1, some_files_not_ready );
-#endif
-			recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
-		}
 }
 
 bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@@ -347,7 +404,8 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
 
 bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
 {
-    return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
+	return mSharedDirectories->getEntryType(
+	            DirectoryStorage::EntryIndex(client_param) ) == DIR_TYPE_FILE;
 }
 
 void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)

libretroshare/src/file_sharing/hash_cache.cc

@@ -187,9 +187,10 @@ void HashStorage::threadTick()
 			else
 				rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
 
-            //RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
-            if(rsEvents)
 			{
+				/* Emit deprecated event only for retrocompatibility
+				 * TODO: create a proper event with structured data instead of a
+				 * formatted string */
 				auto ev = std::make_shared<RsSharedDirectoriesEvent>();
 				ev->mEventCode = RsSharedDirectoriesEventCode::HASHING_FILE;
 				ev->mMessage = tmpout;
@@ -198,7 +199,7 @@ void HashStorage::threadTick()
 
 			double seconds_origin = rstime::RsScopeTimer::currentTime() ;
 
-			if(RsDirUtil::getFileHash(job.full_path, hash,size, this))
+			if(RsDirUtil::getFileHash(job.full_path, hash, size, this))
 			{
 				// store the result
 
@@ -218,8 +219,7 @@ void HashStorage::threadTick()
 				mChanged = true ;
 				mTotalHashedSize += size ;
 			}
-			else
+			else RS_ERR("Failure hashing file: ", job.full_path);
-				std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
 
 			mHashingTime += rstime::RsScopeTimer::currentTime() - seconds_origin ;
 			mHashedBytes += size ;
@@ -234,10 +234,17 @@ void HashStorage::threadTick()
             ++mHashCounter ;
         }
 	}
-    // call the client
 
+	// call the client
 	if(!hash.isNull())
 		job.client->hash_callback(job.client_param, job.full_path, hash, size);
+
+	/* Notify we completed hashing a file */
+	auto ev = std::make_shared<RsFileHashingCompletedEvent>();
+	ev->mFilePath = job.full_path;
+	ev->mHashingSpeed = mCurrentHashingSpeed;
+	ev->mFileHash = hash;
+	rsEvents->postEvent(ev);
 }
 
 bool HashStorage::requestHash(const std::string& full_path,uint64_t size,rstime_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)

libretroshare/src/file_sharing/p3filelists.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2018  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -30,7 +32,7 @@
 #include "retroshare/rsids.h"
 #include "retroshare/rspeers.h"
 #include "retroshare/rsinit.h"
-
+#include "util/cxx17retrocompat.h"
 #include "rsserver/p3face.h"
 
 #define P3FILELISTS_DEBUG() std::cerr << time(NULL)    << " : FILE_LISTS : " << __FUNCTION__ << " : "
@@ -1044,31 +1046,36 @@ void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIn
 }
 
 // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
-int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
+int p3FileDatabase::RequestDirDetails(
+        void* ref, DirDetails& d, FileSearchFlags flags ) const
 {
-    RS_STACK_MUTEX(mFLSMtx) ;
+	RS_STACK_MUTEX(mFLSMtx);
 
 	d.children.clear();
 
-    // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
+	/* Case where the pointer is NULL, which means we're at the top of the list
-    // or at the top of our own list of shared directories, depending on the flags.
+	 * of shared directories for all friends (including us) or at the top of our
+	 * own list of shared directories, depending on the flags.
+	 *
+	 * Friend index is used as follows:
+	 *  0		: own id
+	 *  1...n	: other friends
+	 *
+	 *  entry_index: starts at 0.
+	 *
+	 * The point is: we cannot use (0,0) because it encodes to NULL. No existing
+	 * combination should encode to NULL.
+	 * So we need to properly convert the friend index into 0 or into a friend
+	 * tab index in mRemoteDirectories.
+	 *
+	 * We should also check the consistency between flags and the content of ref.
+	 */
 
-    // Friend index is used as follows:
+	if (ref == nullptr)
-    //	0		: own id
-    //  1...n	: other friends
-    //
-    // entry_index: starts at 0.
-    //
-    // The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
-    // So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
-    //
-    // We should also check the consistency between flags and the content of ref.
-
-    if (ref == NULL)
 	{
-        d.ref = NULL ;
+		d.ref = nullptr;
 		d.type = DIR_TYPE_ROOT;
-        d.parent = NULL;
+		d.parent = nullptr;
         d.prow = -1;
         d.name = "root";
         d.hash.clear() ;
@@ -1079,10 +1086,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
         if(flags & RS_FILE_HINTS_LOCAL)
 		{
-            void *p;
+			void *p = nullptr;
 
 			{
-			convertEntryIndexToPointer<sizeof(void*)>(0,0,p);	// root of own directories
+				// root of own directories
+				convertEntryIndexToPointer<sizeof(void*)>(0, 0, p);
 				DirStub stub;
             stub.type = DIR_TYPE_PERSON;
             stub.name = mServCtrl->getOwnId().toStdString();
@@ -1092,9 +1100,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
             if(mExtraFiles->size() > 0)
             {
-            convertEntryIndexToPointer<sizeof(void*)>(0,1,p);	// local shared files from extra list
+				// local shared files from extra list
+				convertEntryIndexToPointer<sizeof(void*)>(0, 1, p);
 				DirStub stub;
-            stub.type = DIR_TYPE_PERSON;						// not totally exact, but used as a trick.
+				// not totally exact, but used as a trick.
+				stub.type = DIR_TYPE_PERSON;
             stub.name = "[Extra List]";
             stub.ref  = p;
 
@@ -1127,15 +1137,16 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     }
 
     uint32_t fi;
-    DirectoryStorage::EntryIndex e ;
+	DirectoryStorage::EntryIndex e;
 
     convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
 
 	// check consistency
-    if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||  (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
+	if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||
+	        (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
 	{
-        P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
+		RS_ERR("Remote request on local index or local request on remote index");
-        return false ;
+		return false;
 	}
 
 	if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list
@@ -1150,25 +1161,28 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     }
 
 
-    DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
+	DirectoryStorage* storage =
+	        (flags & RS_FILE_HINTS_LOCAL) ?
+	            ((DirectoryStorage*) mLocalSharedDirs) :
+	            ((DirectoryStorage*) mRemoteDirectories[fi-1]);
 
-    // Case where the index is the top of a single person. Can be us, or a friend.
+	/* Case where the index is the top of a single person.
-
+	 * Can be us, or a friend. */
-    if(storage==NULL || !storage->extractData(e,d))
+	if(!storage || !storage->extractData(e,d))
 	{
-#ifdef DEBUG_FILE_HIERARCHY
+		RS_WARN( "request on index; ", e, ", for directory ID:",
-        P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
+		         ( (!storage)? ("[NULL]") : (storage->peerId().toStdString()) ),
-#endif
+		         " failed" );
-        return false ;
+		return false;
 	}
 
-    // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
+	/* update indexes. This is a bit hacky, but does the job. The cast to
-    // a pointer into an int.
+	 * intptr_t is the proper way to convert a pointer into an int. */
+	convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref);
 
-    convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref) ;
+	for(uint32_t i=0; i<d.children.size(); ++i)
-
+		convertEntryIndexToPointer<sizeof(void*)>(
-    for(uint32_t i=0;i<d.children.size();++i)
+		            (intptr_t) d.children[i].ref, fi, d.children[i].ref );
-        convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
 
     if(e == 0)	// root
     {
@@ -1178,9 +1192,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     else
     {
         if(d.parent == 0)	 // child of root node
-            d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1);
+			d.prow = (flags & RS_FILE_HINTS_LOCAL) ? 0 : (fi-1);
         else
-            d.prow = storage->parentRow(e) ;
+			d.prow = storage->parentRow(e);
 
         convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.parent,fi,d.parent) ;
     }
@@ -1307,7 +1321,9 @@ uint32_t p3FileDatabase::watchPeriod()
     return mLocalDirWatcher->fileWatchPeriod();
 }
 
-int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
+int p3FileDatabase::SearchKeywords(
+        const std::list<std::string>& keywords, std::list<DirDetails>& results,
+        FileSearchFlags flags, const RsPeerId& client_peer_id )
 {
     if(flags & RS_FILE_HINTS_LOCAL)
     {
@@ -1319,15 +1335,15 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
 
             mLocalSharedDirs->searchTerms(keywords,firesults) ;
 
-            for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
+			for(auto& it: std::as_const(firesults))
 			{
-                void *p=NULL;
+				void *p = nullptr;
-                convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
+				convertEntryIndexToPointer<sizeof(void*)>(it, 0, p);
-				pointers.push_back(p) ;
+				pointers.push_back(p);
 			}
         }
 
-        filterResults(pointers,results,flags,client_peer_id) ;
+		filterResults(pointers, results, flags, client_peer_id);
     }
 
     if(flags & RS_FILE_HINTS_REMOTE)
@@ -1476,7 +1492,9 @@ bool p3FileDatabase::search(
     return false;
 }
 
-int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
+int p3FileDatabase::filterResults(
+        const std::list<void*>& firesults, std::list<DirDetails>& results,
+        FileSearchFlags flags, const RsPeerId& peer_id ) const
 {
     results.clear();
 
@@ -1484,26 +1502,30 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
 
     /* translate/filter results */
 
-    for(std::list<void*>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
+	for(void* rit: std::as_const(firesults))
 	{
-        DirDetails cdetails ;
+		DirDetails cdetails;
 
-        if(!RequestDirDetails (*rit,cdetails,RS_FILE_HINTS_LOCAL))
+		if(!RequestDirDetails(rit, cdetails, RS_FILE_HINTS_LOCAL))
 		{
-            P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << *rit << std::endl;
+			RS_ERR("Cannot retrieve dir details for entry: ", rit);
+			print_stacktrace();
 			continue ;
 		}
-#ifdef DEBUG_P3FILELISTS
+
-        P3FILELISTS_DEBUG() << "Filtering candidate " << *rit << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
+		RS_DBG( "Filtering candidate: ", rit,
-#endif
+		        ", name: ", cdetails.name, ", path: ", cdetails.path,
+		        ", flags: ", cdetails.flags, ", peer: ", peer_id );
 
 		if(!peer_id.isNull())
 		{
-            FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
+			FileSearchFlags permission_flags =
+			        rsPeers->computePeerPermissionFlags(
+			            peer_id, cdetails.flags, cdetails.parent_groups );
 
 			if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
 			{
-                cdetails.id.clear() ;
+				cdetails.id.clear();
 				results.push_back(cdetails);
 #ifdef DEBUG_P3FILELISTS
 				std::cerr << ": kept" << std::endl ;
@@ -1518,7 +1540,19 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
             results.push_back(cdetails);
     }
 
-    return !results.empty() ;
+	for(auto& res: results)
+	{
+		/* Most file will just have file name stored, but single file shared
+		 * without a shared dir will contain full path instead of just the
+		 * name, so purify it to perform the search */
+		if(res.name.find("/") != std::string::npos )
+		{
+			std::string _tDirPath;
+			RsDirUtil::splitDirFromFile(res.name, _tDirPath, res.name);
+		}
+	}
+
+	return !results.empty();
 }
 
 bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath)
@@ -1592,78 +1626,90 @@ void p3FileDatabase::tickSend()
 
 void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
 {
-    RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem;
+	RsFileListsSyncResponseItem* ritem = new RsFileListsSyncResponseItem;
 
 	// look at item TS. If local is newer, send the full directory content.
 	{
-        RS_STACK_MUTEX(mFLSMtx) ;
+		RS_STACK_MUTEX(mFLSMtx);
 
-#ifdef DEBUG_P3FILELISTS
+		RS_DBG( "Received directory sync request from peer ", item->PeerId(),
-        P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
+		        ". hash=", item->entry_hash, ", flags=", item->flags,
-#endif
+		        ", request id: ", item->request_id, ", last known TS: ",
+		        item->last_known_recurs_modf_TS );
 
 		EntryIndex entry_index = DirectoryStorage::NO_INDEX;
-
 		if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
 		{
-#ifdef DEBUG_P3FILELISTS
+			RS_DBG("Cannot find entry index for hash ", item->entry_hash,
-            P3FILELISTS_DEBUG() << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
+			       " cannot respond to sync request." );
-#endif
 			return;
 		}
 
-        uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ;
+		uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index);
-        ritem->PeerId(item->PeerId()) ;
+		ritem->PeerId(item->PeerId());
 		ritem->request_id = item->request_id;
-        ritem->entry_hash = item->entry_hash ;
+		ritem->entry_hash = item->entry_hash;
 
 		std::list<RsNodeGroupId> node_groups;
 		FileStorageFlags node_flags;
 
 		if(entry_type != DIR_TYPE_DIR)
 		{
-#ifdef DEBUG_P3FILELISTS
+			RS_DBG( "Directory does not exist anymore, or is not a directory, "
-            P3FILELISTS_DEBUG() << "  Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
+			        "or permission denied. Answering with proper flags." );
-#endif
+			ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
-            ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
+			        RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
 		}
-        else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
+		else if( entry_index != 0 &&
+		         (!mLocalSharedDirs->getFileSharingPermissions(
+		              entry_index, node_flags,node_groups ) ||
+		          !(rsPeers->computePeerPermissionFlags(
+		                item->PeerId(), node_flags, node_groups ) &
+		            RS_FILE_HINTS_BROWSABLE)) )
 		{
-            P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
+			RS_ERR("cannot get file permissions for entry index: ", entry_index,
-            ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
+			       ", or permission denied." );
+			ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
+			        RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
 		}
 		else
 		{
-            rstime_t local_recurs_max_time ;
+			rstime_t local_recurs_max_time;
-            mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
+			mLocalSharedDirs->getDirectoryRecursModTime(
+			            entry_index, local_recurs_max_time );
 
-            if(item->last_known_recurs_modf_TS != local_recurs_max_time)	// normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
+			/* normally, should be "<", but since we provided the TS it should
+			 * be equal, so != is more robust. */
+			if(item->last_known_recurs_modf_TS != local_recurs_max_time)
 			{
-#ifdef DEBUG_P3FILELISTS
+				RS_DBG( "Directory is more recent than what the friend knows. "
-                P3FILELISTS_DEBUG() << "  Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
+				        "Sending full dir content as response.");
-#endif
 
-                ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
+				ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
+				        RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
 				ritem->last_known_recurs_modf_TS = local_recurs_max_time;
 
-                // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
+				/* We supply the peer id, in order to possibly remove some
-                mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
+				 * subdirs, if entries are not allowed to be seen by this peer.
+				 */
+				mLocalSharedDirs->serialiseDirEntry(
+				            entry_index, ritem->directory_content_data,
+				            item->PeerId() );
 			}
 			else
 			{
-#ifdef DEBUG_P3FILELISTS
+				RS_DBG3( "Directory is up to date w.r.t. what the friend knows."
-                P3FILELISTS_DEBUG() << "  Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
+				         " Sending ACK." );
-#endif
 
-                ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
+				ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
-                ritem->last_known_recurs_modf_TS = local_recurs_max_time ;
+				        RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE;
+				ritem->last_known_recurs_modf_TS = local_recurs_max_time;
 			}
 		}
 	}
 
 	// sends the response.
-
+	splitAndSendItem(ritem);
-    splitAndSendItem(ritem) ;
 }
 
 void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem)

libretroshare/src/ft/ftserver.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2008 by Robert Fernie <retroshare@lunamutt.com>                   *
+ * Copyright (C) 2008  Robert Fernie <retroshare@lunamutt.com>                 *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -876,9 +878,11 @@ int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetai
 {
 	return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
 }
-int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
+int ftServer::SearchKeywords(
+        std::list<std::string> keywords, std::list<DirDetails> &results,
+        FileSearchFlags flags, const RsPeerId& peer_id )
 {
-	return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id);
+	return mFileDatabase->SearchKeywords(keywords, results, flags, peer_id);
 }
 
 int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
@@ -2033,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
 
 	std::vector<DeepFilesSearchResult> dRes;
 	DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-	if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
+	if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
 	{
 		RsFileSearchResultItem resIt;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
   VALIDATE_MAX_WAITING_TIME(60)
 {
     mDataAccess = new RsGxsDataAccess(gds);
-
-    // Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-    // This code is only called because it of deep indexing in channels. But loading
-    // the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
-    // eats memory, as many tests have shown. Not because of leaks, but because new threads are
-    // apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
-    // that RS will use them again.
-    //
-    //    * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
-    //      service (e.g. channels here) should update the index when a new message is received. The question to how old messages
-    //	    are processed is open. I believe that they shouldn't. New users will progressively process them.
-    //
-    //    * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
-    //      unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
-    //      It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
-    //      doesn't counterbalance such a low risk.
-    //
-    if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
-    {
-        std::vector<RsGxsGroupId> grpsToDel;
-        GxsMsgReq msgsToDel;
-
-        RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
-                                             this, *mSerialiser,
-                                             grpsToDel,msgsToDel);
-
-        for(auto& grpId: grpsToDel)
-        {
-            uint32_t token2=0;
-            deleteGroup(token2,grpId);
-        }
-
-        if(!msgsToDel.empty())
-        {
-            uint32_t token1=0;
-            deleteMsgs(token1,msgsToDel);
-        }
-    }
-#endif
 }
 
 void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@@ -362,12 +322,12 @@ void RsGenExchange::tick()
 			{
 				mIntegrityCheck = new RsGxsIntegrityCheck( mDataStore, this,
 				                                           *mSerialiser, mGixs);
-				mIntegrityCheck->start("gxs integrity");
+				std::string thisName = typeid(*this).name();
-				mChecking = true;
+				mChecking = mIntegrityCheck->start("gxs IC4 "+thisName);
 			}
 		}
 
-		if(mIntegrityCheck->isDone())
+		if(mIntegrityCheck->isDone() || !mChecking)
 		{
             std::vector<RsGxsGroupId> grpIds;
             GxsMsgReq msgIds;
@@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
             msgDeleted.push_back(note.msgIds);
     }
 
-    for(const auto& msgreq:msgDeleted)
+	/* Three nested for looks like a performance bomb, but as Cyril says here
-        for(const auto& msgit:msgreq)
+	 * https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
-            for(const auto& msg:msgit.second)
+	 * this should actually not explode at all because it is just one message at
-                mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
+	 * time that get notified */
+	for(const auto& msd : mMsgDeletePublish)
+		for(auto& msgMap : msd.mMsgs)
+			for(auto& msgId : msgMap.second)
+				mNotifications.push_back(
+				            new RsGxsMsgDeletedChange(msgMap.first, msgId) );
 
 	mMsgDeletePublish.clear();
 }

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
     return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
 }
 
-#ifndef RS_DEEP_CHANNEL_INDEX
 static bool termSearch(const std::string& src, const std::string& substring)
 {
 	/* always ignore case */
-	return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
+	return src.end() != std::search(
+	            src.begin(), src.end(), substring.begin(), substring.end(),
+	            RsRegularExpression::CompareCharIC() );
 }
-#endif // ndef RS_DEEP_CHANNEL_INDEX
+
 
 bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
@@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
     return true ;
 }
 
-void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
 {
 	std::set<RsGxsGroupId> groupsToNotifyResults;
 
@@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 
 		for (const RsGxsGroupSummary& gps : group_infos)
 		{
-#ifndef RS_DEEP_CHANNEL_INDEX
+#ifdef TO_REMOVE
+			/* Because of deep search is enabled search results may bring more
+			 * info then we already have also about post that are indexed by
+			 * xapian, so we don't apply this filter anymore. */
+
 			/* Only keep groups that are not locally known, and groups that are
 			 * not already in the mDistantSearchResults structure.
-			 * mDataStore may in some situations allocate an empty group meta data, so it's important
+			 * mDataStore may in some situations allocate an empty group meta
-			 * to test that the group meta is both non null and actually corresponds to the group id we seek. */
+			 * data, so it's important to test that the group meta is both non
-
+			 * null and actually corresponds to the group id we seek. */
 			auto& meta(grpMeta[gps.mGroupId]);
+			if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
+#endif // def TO_REMOVE
 
-			if(meta != nullptr && meta->mGroupId == gps.mGroupId)
-                continue;
-
-#ifdef NXS_NET_DEBUG_9
-            std::cerr << "  group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
-#endif
-
-#else // ndef RS_DEEP_CHANNEL_INDEX
-			/* When deep search is enabled search results may bring more info
-			 * then we already have also about post that are indexed by xapian,
-			 * so we don't apply this filter in this case. */
-#endif
 			const RsGxsGroupId& grpId(gps.mGroupId);
 
 			groupsToNotifyResults.insert(grpId);
@@ -5332,16 +5330,17 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 		mObserver->receiveDistantSearchResults(req, grpId);
 }
 
-void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req,
+        const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
 {
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
 #endif
 	auto it = mSearchRequests.find(req);
-
 	if(mSearchRequests.end() == it)
 	{
-        std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
+		RS_WARN("Received search results for unknown request: ", req);
 		return;
 	}
     RsGxsGroupId grpId = it->second;
@@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
 	mObserver->receiveDistantSearchResults(req, grpId);
 }
 
+std::error_condition RsGxsNetService::distantSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
+{
+	if(!mGxsNetTunnel)
+	{
+		free(searchData);
+		return std::errc::function_not_supported;
+	}
+
+	return mGxsNetTunnel->turtleSearchRequest(
+	            searchData, dataSize, serviceType, requestId );
+}
+
+std::error_condition RsGxsNetService::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG("");
+	return mObserver->handleDistantSearchRequest(
+	            requestData, requestSize, resultData, resultSize );
+}
+
+std::error_condition RsGxsNetService::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	return mObserver->receiveDistantSearchResult(
+	            requestId, resultData, resultSize );
+}
+
 bool RsGxsNetService::search( const std::string& substring,
                               std::list<RsGxsGroupSummary>& group_infos )
 {
 	group_infos.clear();
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-
-#warning TODO: filter deep index search result to non circle-restricted groups.
-//    /!\
-//    /!\   These results should be filtered to only return results coming from a non restricted group!
-//    /!\
-
-	std::vector<DeepChannelsSearchResult> results;
-	DeepChannelsIndex::search(substring, results);
-
-	for(auto dsr : results)
-	{
-		RsUrl rUrl(dsr.mUrl);
-		const auto& uQ(rUrl.query());
-		auto rit = uQ.find("id");
-		if(rit != rUrl.query().end())
-		{
-			RsGroupNetworkStats stats;
-			RsGxsGroupId grpId(rit->second);
-			if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
-			{
-				RsGxsGroupSummary s;
-
-				s.mGroupId = grpId;
-
-				if((rit = uQ.find("name")) != uQ.end())
-					s.mGroupName = rit->second;
-				if((rit = uQ.find("signFlags")) != uQ.end())
-					s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
-				if((rit = uQ.find("publishTs")) != uQ.end())
-					s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
-				if((rit = uQ.find("authorId")) != uQ.end())
-					s.mAuthorId  = RsGxsId(rit->second);
-
-				s.mSearchContext = dsr.mSnippet;
-
-				s.mNumberOfMessages = stats.mMaxVisibleCount;
-				s.mLastMessageTs    = stats.mLastGroupModificationTS;
-				s.mPopularity       = stats.mSuppliers;
-
-				group_infos.push_back(s);
-			}
-		}
-	}
-#else // RS_DEEP_CHANNEL_INDEX
 	RsGxsGrpMetaTemporaryMap grpMetaMap;
 	{
 		RS_STACK_MUTEX(mNxsMutex) ;
@@ -5492,7 +5477,6 @@ bool RsGxsNetService::search( const std::string& substring,
 
 			group_infos.push_back(s);
 		}
-#endif // RS_DEEP_CHANNEL_INDEX
 
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << "  performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;

libretroshare/src/gxs/rsgxsnetservice.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSGXSNETSERVICE_H
+#pragma once
-#define RSGXSNETSERVICE_H
 
 #include <list>
 #include <queue>
@@ -130,18 +131,53 @@ public:
 
     virtual bool msgAutoSync() const override { return mAllowMsgSync; }
     virtual bool grpAutoSync() const override { return mGrpAutoSync; }
-    /*!
-     * \brief Search methods.
-     * 			These four methods are used to request distant search and receive the results.
-     * \param group_id
-     */
-    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
-    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
 
-    virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
+	/// @see RsNetworkExchangeService
+	std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/** Request group data via turtle search
+	 * @param group_id */
+	TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
+
+	/**
+	 * @brief Search for matching groups names over turtle search.
+	 * @deprecated this method is kept mostly for retrocompatibility with older
+	 * peers, newly implemented search functions should instead be based on the
+	 * service generic search.
+	 * @see RsNetworkExchangeService
+	 */
+	RS_DEPRECATED_FOR(distantSearchRequest)
+	TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
+
+	/** @see RsNetworkExchangeService
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(receiveDistantSearchResult)
+	void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        const uint8_t* encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) override;
+
+	/**
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(handleRemoteSearchRequest)
+	virtual bool search( const std::string& substring,
+	                     std::list<RsGxsGroupSummary>& group_infos) override;
     virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
     virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
-    virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
 
     virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@@ -629,5 +665,3 @@ private:
 
     bool mUseMetaCache;
 };
-
-#endif // RSGXSNETSERVICE_H

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -27,6 +29,8 @@
 #include "gxs/rsnxs.h"
 #include "rsgxsnettunnel.h"
 
+/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
+
 //#define DEBUG_RSGXSNETTUNNEL 1
 
 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@@ -36,42 +40,93 @@
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA    =   1;
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH  = 100;
 
-RsGxsDistSync *rsGxsDistSync = NULL;
 
 RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
 {
 	mRandomBias.clear();
 
-	mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20);	// adds some variance in order to avoid doing all this tasks at once across services
+	/* adds some variance in order to avoid doing all this tasks at once across
-	mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
+	 * services */
-	mLastDump = time(NULL) + (RSRandom::random_u32()%20);
+	auto now = time(nullptr);
+	mLastKeepAlive = now + (RsRandom::random_u32()%20);
+	mLastAutoWash = now + (RsRandom::random_u32()%20);
+	mLastDump = now + (RsRandom::random_u32()%20);
 }
 
-//===========================================================================================================================================//
+//============================================================================//
 //                         Transport Items                                    //
-//===========================================================================================================================================//
+//============================================================================//
 
-const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
+enum class RsGxsNetTunnelItemSubtypes : uint8_t
+{
+	VIRTUAL_PEER             = 0x01,
 
+	KEEP_ALIVE               = 0x02,
+
+	RANDOM_BIAS              = 0x03,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
+	SEARCH_SUBSTRING         = 0x04,
+
+	SEARCH_GROUP_REQUEST     = 0x05,
+
+	// SEARCH_GROUP_SUMMARY  = 0x06, removed
+
+	SEARCH_GROUP_DATA        = 0x07,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
+	SEARCH_GROUP_SUMMARY     = 0x08,
+
+	/** Generic search request generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REQUEST   = 0x09,
+
+	/** Generic search reply generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REPLY     = 0x0a
+};
+
+RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
+constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
+        static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
+
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                = 0x01 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  = 0x02 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 = 0x03 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     = 0x04 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
-// const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    = 0x07 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
+// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
 
-class RsGxsNetTunnelItem: public RsItem
+struct RsGxsNetTunnelItem: RsItem
 {
 public:
-	explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
+	explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            static_cast<uint8_t>(subtype) )
+	{
+		/* no priority. All items are encapsulated into generic Turtle items
+		 * anyway. */
+	}
+
+	virtual ~RsGxsNetTunnelItem() = default;
+	virtual void clear() {}
+
+	RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
+	explicit RsGxsNetTunnelItem(uint8_t item_subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            item_subtype )
 	{
 		// no priority. All items are encapsulated into generic Turtle items anyway.
 	}
-
-	virtual ~RsGxsNetTunnelItem() {}
-	virtual void clear() {}
 };
 
 class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@@ -113,7 +168,86 @@ public:
 	Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
 };
 
-class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
+struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReqItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(RsServiceType::NONE), mSearchData(nullptr),
+	    mSearchDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mSearchData; /// Service search request data
+	uint32_t mSearchDataSize; /// Search data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mSearchData);
+		mSearchData = nullptr;
+		mSearchDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReqItem() override { clear(); }
+};
+
+struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReplyItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(RsServiceType::NONE), mReplyData(nullptr),
+	    mReplyDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mReplyData; /// Service search reply data
+	uint32_t mReplyDataSize; /// Search reply data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mReplyData);
+		mReplyData = nullptr;
+		mReplyDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
+};
+
+class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
+RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
 {
 public:
     explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@@ -164,6 +298,7 @@ public:
 		RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
 	}
 };
+
 class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
 {
 public:
@@ -193,28 +328,41 @@ public:
 class RsGxsNetTunnelSerializer: public RsServiceSerializer
 {
 public:
-	RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
+	RsGxsNetTunnelSerializer():
+	    RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
 
 	virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
 	{
 		if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
 		{
-			GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
+			RS_ERR( "received item with wrong service ID ", service);
-			return NULL ;
+			print_stacktrace();
+			return nullptr;
 		}
 
-		switch(item_subtype)
+		switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
 		{
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                : return new RsGxsNetTunnelVirtualPeerItem ;
+		case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  : return new RsGxsNetTunnelKeepAliveItem ;
+			return new RsGxsNetTunnelVirtualPeerItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 : return new RsGxsNetTunnelRandomBiasItem ;
+		case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     : return new RsGxsNetTunnelTurtleSearchSubstringItem;
+			return new RsGxsNetTunnelKeepAliveItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+			return new RsGxsNetTunnelRandomBiasItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
+			return new RsGxsNetTunnelTurtleSearchSubstringItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+			return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
+			return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+			return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+			return new RsGxsServiceTurtleSearchReqItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+			return new RsGxsServiceTurtleSearchReplyItem;
 		default:
-			GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
+			RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
-			return NULL ;
+			return nullptr;
 		}
 	}
 };
@@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
+TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
+        const std::string& match_string,
+        RsNetworkExchangeService* client_service )
 {
     GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
 
@@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
 	search_item.service = client_service->serviceType() ;
 
     uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
-    unsigned char *mem = (unsigned char*)rs_malloc(size) ;
+	uint8_t* mem = rs_malloc<uint8_t>(size);
 
     if(mem == NULL)
         return 0 ;
@@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
+std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
 {
-	GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
+	if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
+		return std::errc::invalid_argument;
 
-	RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
+	RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
+	searchItem.mSearchDataSize = dataSize;
+	searchItem.mSearchData = searchData;
 
-    RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
+	RsGxsNetTunnelSerializer tSerializer;
 
-    if(substring_sr != NULL)
+	uint32_t size = tSerializer.size(&searchItem);
-    {
+	uint8_t* buf = rs_malloc<uint8_t>(size);
-		GXS_NET_TUNNEL_DEBUG() << "  : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
 
-        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
+	tSerializer.serialise(&searchItem, buf, &size);
 
-        std::list<RsGxsGroupSummary> results ;
+	requestId = mTurtle->turtleSearch(buf, size, this);
-        RsNetworkExchangeService *service = nullptr;
+	if(!requestId) return std::errc::result_out_of_range;
 
-        {
+	return std::error_condition();
+}
+
+rs_view_ptr<RsNetworkExchangeService>
+RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
+{
 	RS_STACK_MUTEX(mGxsNetTunnelMtx);
+	auto it = mSearchableServices.find(serviceType);
+	if( it != mSearchableServices.end()) return it->second;
+	return nullptr;
+}
 
-            auto it = mSearchableServices.find(substring_sr->service) ;
+bool RsGxsNetTunnelService::receiveSearchRequest(
+        uint8_t* search_request_data, uint32_t search_request_data_len,
+        uint8_t*& search_result_data, uint32_t& search_result_data_size,
+        uint32_t& max_allowed_hits )
+{
+	/* Must return true only if there are matching results available, false in
+	 * all other cases. @see RsTurleClientService */
 
-            if(it != mSearchableServices.end())
+	RS_DBG3("");
-                service = it->second;
+
+	RsGxsNetTunnelSerializer tSerializer;
+
+	std::unique_ptr<RsItem> item;
+	item.reset(tSerializer.deserialise(
+	               search_request_data, &search_request_data_len ));
+
+	if(!item)
+	{
+		RS_ERR( "Deserialization failed: ",
+		        search_request_data, search_request_data_len, item.get() );
+		print_stacktrace();
+		return false;
 	}
 
-        if(service != nullptr && service->search(substring_sr->substring_match,results))
+	switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
 	{
-			RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
+	case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
-
+	{
-			GXS_NET_TUNNEL_DEBUG() << "  : " << results.size() << " result found. Sending back." << std::endl;
+		if(!search_result_data)
-
+		{
-            search_result_item.service = substring_sr->service ;
+			RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
-            search_result_item.group_infos = results ;
+			        "results!" );
-
+			print_stacktrace();
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			break;
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
-
-            delete item;
-
-			if(search_result_data == NULL)
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
-
-			return true ;
-        }
 		}
 
-    RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
+		auto substring_sr =
-
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
-    if(substring_gr != NULL)
+		if(!substring_sr)
 		{
-		RS_STACK_MUTEX(mGxsNetTunnelMtx);
+			RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
-		auto it = mSearchableServices.find(substring_gr->service) ;
+			         item->PacketSubType(), " but casting failed!");
+			break;
+		}
 
-        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
-
+		std::list<RsGxsGroupSummary> results;
-        unsigned char *encrypted_group_data = NULL ;
+		auto tService = retrievieSearchableServiceLocking(substring_sr->service);
-        uint32_t encrypted_group_data_len = 0 ;
+		if(tService && tService->search(substring_sr->substring_match, results))
-
-        if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
 		{
-			RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
+			RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
+			search_result_item.service = substring_sr->service;
+			search_result_item.group_infos = results;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_item.service = substring_gr->service ;
+			tSerializer.serialise(
-            search_result_item.encrypted_group_data = encrypted_group_data ;
+			            &search_result_item, search_result_data,
+			            &search_result_data_size );
+
+			return true;
+		}
+
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+	{
+		auto *substring_gr =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
+
+		if(!substring_gr)
+		{
+			RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
+			         item->PacketSubType(), " but casting failed!" );
+			break;
+		}
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
+		uint8_t* encrypted_group_data = nullptr;
+		uint32_t encrypted_group_data_len = 0;
+
+		auto tService = retrievieSearchableServiceLocking(substring_gr->service);
+		if(tService && tService->search(
+		            substring_gr->hashed_group_id,
+		            encrypted_group_data, encrypted_group_data_len ))
+		{
+			RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
+			search_result_item.service = substring_gr->service;
+			search_result_item.encrypted_group_data = encrypted_group_data;
 			search_result_item.encrypted_group_data_len = encrypted_group_data_len;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			tSerializer.serialise(
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
+			            &search_result_item,
-
+			            search_result_data, &search_result_data_size );
-			if(search_result_data == NULL)
+			return true;
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
-
-            delete item;
-			return true ;
 		}
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+	{
+		RS_DBG3("SERVICE_SEARCH_REQUEST");
+
+		auto searchItem =
+		        static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
+
+		uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
+		auto sService = retrievieSearchableServiceLocking(sType);
+		if(!sService)
+		{
+			RS_WARN("Got search request for non searchable service: ", sType);
+			break;
 		}
 
-    delete item;
+		RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
-    return false ;
+
+		auto errc = sService->handleDistantSearchRequest(
+		            searchItem->mSearchData, searchItem->mSearchDataSize,
+		            replyItem.mReplyData, replyItem.mReplyDataSize );
+		if(errc)
+		{
+			// Some error has been reported by the searchable service
+			RS_WARN("searchable service: ", sType , " reported: ", errc);
+			break;
+		}
+
+		if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
+		    (replyItem.mReplyData && !replyItem.mReplyDataSize) )
+		{
+			// Inconsistent behaviour from searcheable service
+			RS_ERR( "searchable service: ", sType , " silently failed handling "
+			        "inconsistent result mReplyData: ", replyItem.mReplyData,
+			        " mReplyDataSize: ", replyItem.mReplyDataSize );
+			break;
+		}
+
+		/* Our node have 0 matching results */
+		if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
+			break;
+
+		search_result_data_size = tSerializer.size(&replyItem);
+		search_result_data = rs_malloc<uint8_t>(search_result_data_size);
+
+		tSerializer.serialise(
+		            &replyItem, search_result_data, &search_result_data_size );
+
+		return true;
+	}
+	default:
+		RS_WARN("Got unknown item type: ", item->PacketSubType());
+		break;
+	}
+
+	return false;
 }
 
-void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
+void RsGxsNetTunnelService::receiveSearchResult(
+        TurtleSearchRequestId request_id,
+        uint8_t* search_result_data, uint32_t search_result_data_len )
 {
-    RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
+	RS_DBG3(request_id);
 
-	GXS_NET_TUNNEL_DEBUG() << "  : received search result for search request " << std::hex << request_id << "" << std::endl;
+	std::unique_ptr<RsItem> item;
+	item.reset(RsGxsNetTunnelSerializer().deserialise(
+	               search_result_data,&search_result_data_len ));
 
-    RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
+	auto castFailedWarn = [](const uint8_t subtype)
-
-    if(result_gs != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
+		RS_WARN( "Got item with subtype: ", subtype,
-
+		         " but cast failed!" );
-#ifdef DEBUG_RSGXSNETTUNNEL
+	};
-		for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
-			std::cerr << "   group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
-#endif
-
-		auto it = mSearchableServices.find(result_gs->service) ;
 
+	auto searchableServiceGet = [this](const auto pservice)
+	{
+		auto service = static_cast<uint16_t>(pservice);
+		auto it = mSearchableServices.find(service);
 		if(it == mSearchableServices.end())
 		{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
+			RS_WARN( "got item for service ", service,
-            delete item;
+			         " which is not in the searchable services list." );
-			return ;
+			return static_cast<RsNetworkExchangeService*>(nullptr);
 		}
 
-		it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
+		return it->second;
+	};
 
-		delete item;
+	const auto tSubtype = item->PacketSubType();
-		return ;
+	switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
-	}
-
-    RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
-
-    if(result_gd != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
-
-		auto it = mSearchableServices.find(result_gd->service) ;
-
-		if(it == mSearchableServices.end())
 	{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
+		auto result_gs =
-            delete item;
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
-			return ;
+		            item.get() );
+
+		if(!result_gs)
+		{
+			castFailedWarn(tSubtype);
+			break;
 		}
 
-		it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
+		RS_DBG2( " got result is of type group summary result for service ",
+		         result_gs->service );
 
-        result_gd->encrypted_group_data = NULL ; // prevents deletion
+		auto service = searchableServiceGet(result_gs->service);
-		delete item;
+		if(service)
+			service->receiveTurtleSearchResults(
+			            request_id, result_gs->group_infos );
+		return;
+	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+	{
+		auto result_gd =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
 
-		return ;
+		if(!result_gd)
+		{
+			castFailedWarn(tSubtype);
+			break;
 		}
 
-	GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
+		RS_DBG2("got group data result for service: ", result_gd->service);
+
+		auto service = searchableServiceGet(result_gd->service);
+		if(service)
+			service->receiveTurtleSearchResults(
+			        request_id,
+			        result_gd->encrypted_group_data,
+			        result_gd->encrypted_group_data_len );
+
+		/* Ensure ownershipt is passed down preventing deletion */
+		result_gd->encrypted_group_data = nullptr;
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+	{
+		auto searchReply =
+		        static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
+
+		auto service = searchableServiceGet(searchReply->mServiceType);
+		if(service)
+			service->receiveDistantSearchResult(
+			        request_id,
+			        searchReply->mReplyData,
+			        searchReply->mReplyDataSize );
+
+		/* Ensure memory ownership is passed down preventing deletion */
+		searchReply->mReplyData = nullptr;
+		break;
+	}
+	default:
+		RS_WARN("got item of unknown type: ", item->PacketSubType());
+		break;
+	}
 }
 
-void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias      ) const
+void RsGxsNetTunnelService::getStatistics(
+        std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
+        std::map<RsGxsNetTunnelVirtualPeerId,
+        RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
+        std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
+        turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
 {
     groups = mGroups ;
     virtual_peers = mVirtualPeers ;

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -23,6 +25,7 @@
 #pragma once
 
 #include <map>
+#include <system_error>
 
 #include "turtle/p3turtle.h"
 #include "retroshare/rsgxsdistsync.h"
@@ -100,7 +103,7 @@
 //        and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
 //
 
-class RsGxsNetTunnelItem ;
+struct RsGxsNetTunnelItem;
 class RsNetworkExchangeService ;
 
 class RsGxsNetTunnelService:
@@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
         public RsGxsDistSync
 {
 public:
-	  RsGxsNetTunnelService() ;
+	RsGxsNetTunnelService();
-	  virtual ~RsGxsNetTunnelService() ;
+	~RsGxsNetTunnelService() override;
 
       /*!
        * \brief registerSearchableService
@@ -182,23 +185,37 @@ public:
 	  void dump() const;
 
 	/*!
-       * \brief connectToTurtleRouter
 	* Should be called after allocating a RsGxsNetTunnelService
 	* \param tr turtle router object
 	*/
-	  virtual void connectToTurtleRouter(p3turtle *tr) ;
+	void connectToTurtleRouter(p3turtle *tr) override;
 
-      TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
+	/** Gxs services (channels, forums...) are supposed to use this to request
-      TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
+	 * searches on distant peers */
+	std::error_condition turtleSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId );
 
-      /*!
+	///@see RsTurtleClientService
-       * \brief receiveSearchRequest
+	bool receiveSearchRequest(
-       * 			See RsTurtleClientService::@
+	        unsigned char* search_request_data,
-       */
+	        uint32_t search_request_data_len,
-	  virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
+	        unsigned char*& search_result_data,
-	  virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
+	        uint32_t& search_result_data_len,
+	        uint32_t& max_allowed_hits ) override;
 
-	void threadTick() override; /// @see RsTickingThread
+	///@see RsTurtleClientService
+	virtual void receiveSearchResult(
+	        TurtleSearchRequestId request_id,
+	        unsigned char* search_result_data,
+	        uint32_t search_result_data_len ) override;
+
+	TurtleRequestId turtleGroupRequest(
+	        const RsGxsGroupId& group_id,
+	        RsNetworkExchangeService* client_service );
+
+	/// @see RsTickingThread
+	void threadTick() override;
 
 	  // Overloads p3Config
 
@@ -213,6 +230,11 @@ public:
             std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
 		    Bias20Bytes& bias) const;
 
+	RS_DEPRECATED
+	TurtleRequestId turtleSearchRequest(
+	        const std::string& match_string,
+	        RsNetworkExchangeService* client_service );
+
 protected:
 	  // interaction with turtle router
 
@@ -233,6 +255,8 @@ private:
 	  void sendKeepAlivePackets() ;
 	  void handleIncoming(RsGxsNetTunnelItem *item) ;
 	  void flush_pending_items();
+	rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
+	        uint16_t serviceType );
 
 	  std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ;	// groups on the client and server side
 

libretroshare/src/gxs/rsgxsnotify.h

@@ -97,3 +97,13 @@ private:
     bool mMetaChange;
 };
 
+struct RsGxsMsgDeletedChange : RsGxsNotify
+{
+	RsGxsMsgDeletedChange(
+	        const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
+	    RsGxsNotify(gid), messageId(msgId) {}
+
+	NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
+
+	const RsGxsMessageId messageId;
+};

libretroshare/src/gxs/rsgxsutil.cc

@@ -3,8 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013  Christopher Evi-Parker                                  *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -29,12 +29,6 @@
 #include "pqi/pqihash.h"
 #include "gxs/rsgixs.h"
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-#	include "deep_search/channelsindex.hpp"
-#	include "services/p3gxschannels.h"
-#	include "rsitems/rsgxschannelitems.h"
-#endif
-
 // The goals of this set of methods is to check GXS messages and groups for consistency, mostly
 // re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
 // happen anyway, but we still conduct these test as an extra safety measure.
@@ -198,8 +192,7 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
 
 RsGxsIntegrityCheck::RsGxsIntegrityCheck(
         RsGeneralDataService* const dataService, RsGenExchange* genex,
-    RsSerialType&
+        RsSerialType&, RsGixs* gixs )
-                            , RsGixs* gixs )
   : mDs(dataService), mGenExchangeClient(genex),
     mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
 
@@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
     return true;
 }
 
-bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
+bool RsGxsSinglePassIntegrityCheck::check(
-#ifdef RS_DEEP_CHANNEL_INDEX
+        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-                                , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
+        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
-#endif
-                                , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
 {
 #ifdef DEBUG_GXSUTIL
     GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
 #endif
-#ifdef RS_DEEP_CHANNEL_INDEX
-    bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
-    std::set<RsGxsGroupId> indexedGroups;
-#endif
 
     // first take out all the groups
     std::map<RsGxsGroupId, RsNxsGrp*> grp;
@@ -393,54 +380,13 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
             }
             else
                 msgIds.erase(msgIds.find(grp->grpId));	// could not get them, so group is removed from list.
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-            // This should be moved to p3gxschannels. It is really not the place for this here!
-
-            if( isGxsChannels
-                    && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
-                    && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
-            {
-                RsGxsGrpMetaData meta;
-                meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
-
-                uint32_t blz = grp->grp.bin_len;
-                RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
-                                                      &blz);
-
-                if( RsGxsChannelGroupItem* cgIt =
-                        dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
-                {
-                    RsGxsChannelGroup cg;
-                    cgIt->toChannelGroup(cg, false);
-                    cg.mMeta = meta;
-
-                    indexedGroups.insert(grp->grpId);
-                    DeepChannelsIndex::indexChannelGroup(cg);
         }
 		else
 		{
-                    std::cerr << __PRETTY_FUNCTION__ << " Group: "
+			RS_WARN( "deleting group ", grp->grpId,
-                              << meta.mGroupId.toStdString() << " "
+			         " with wrong hash or null/corrupted meta data. meta=",
-                              << meta.mGroupName
+			         grp->metaData );
-                              << " doesn't seems a channel, please "
-                              << "report to developers"
-                              << std::endl;
-                    print_stacktrace();
-                }
-
-                delete rIt;
-            }
-#endif // def RS_DEEP_CHANNEL_INDEX
-        }
-        else
-        {
-            std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
 			grpsToDel.push_back(grp->grpId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-            if(isGxsChannels)
-                DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
-#endif // def RS_DEEP_CHANNEL_INDEX
 		}
 
         delete grp;
@@ -471,13 +417,7 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
 		for (auto& msgId:msgIdV)
 			if(nxsMsgS.find(msgId) == nxsMsgS.end())
-            {
 				msgsToDel[grpId].insert(msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
-#endif // def  RS_DEEP_CHANNEL_INDEX
-            }
     }
 
     for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
             if(msg->metaData == NULL || currHash != msg->metaData->mHash)
             {
-                std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
+				RS_WARN( "deleting message ", msg->msgId, " in group ",
+				         msg->grpId,
+				         " with wrong hash or null/corrupted meta data. meta=",
+				         static_cast<void*>(msg->metaData) );
                 msgsToDel[msg->grpId].insert(msg->msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(
-                                msg->grpId, msg->msgId );
-#endif // def RS_DEEP_CHANNEL_INDEX
-            }
-            else if (subscribed_groups.count(msg->metaData->mGroupId))
-            {
-#ifdef RS_DEEP_CHANNEL_INDEX
-                // This should be moved to p3gxschannels. It is really not the place for this here!
-
-                if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
-                {
-                    RsGxsMsgMetaData meta;
-                    meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
-
-                    uint32_t blz = msg->msg.bin_len;
-                    RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
-                                                          &blz);
-
-                    if( RsGxsChannelPostItem* cgIt =
-                            dynamic_cast<RsGxsChannelPostItem*>(rIt) )
-                    {
-                        RsGxsChannelPost cg;
-                        cgIt->toChannelPost(cg, false);
-                        cg.mMeta = meta;
-
-                        DeepChannelsIndex::indexChannelPost(cg);
-                    }
-                    else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
-                    else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
-                    else
-                    {
-                        std::cerr << __PRETTY_FUNCTION__ << " Message: "
-                                  << meta.mMsgId.toStdString()
-                                  << " in group: "
-                                  << meta.mGroupId.toStdString() << " "
-                                  << " doesn't seems a channel post, please "
-                                  << "report to developers"
-                                  << std::endl;
-                        print_stacktrace();
-                    }
-
-                    delete rIt;
-                }
-#endif // def RS_DEEP_CHANNEL_INDEX
             }
 
             delete msg;

libretroshare/src/gxs/rsgxsutil.h

@@ -3,8 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013 Christopher Evi-Parker                                   *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
     enum CheckState { CheckStart, CheckChecking };
 
 public:
-
+	RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
-
+	                     RsGenExchange* genex, RsSerialType&,
-    /*!
+	                     RsGixs* gixs );
-     *
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
-    RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
-                         RsGenExchange *genex, RsSerialType&,
-                         RsGixs *gixs);
 
     static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
     bool isDone();
@@ -201,19 +193,9 @@ private:
 class RsGxsSinglePassIntegrityCheck
 {
 public:
-
+	static bool check(
-    /*!
+	        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-     *
+	        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
-    static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
-#ifdef RS_DEEP_CHANNEL_INDEX
-                      , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
-#endif
-                      , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
 };
 
 class GroupUpdate

libretroshare/src/gxs/rsnxs.h

@@ -3,8 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com>         *
+ * Copyright (C) 2011  Robert Fernie <retroshare.project@gmail.com>            *
- * Copyright 2011-2011 by Christopher Evi-Parker                               *
+ * Copyright (C) 2011  Christopher Evi-Parker                                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -20,17 +22,15 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
+#pragma once
-#ifndef RSGNP_H
-#define RSGNP_H
 
 #include <set>
 #include <string>
-#include "util/rstime.h"
 #include <stdlib.h>
 #include <list>
 #include <map>
 
+#include "util/rstime.h"
 #include "services/p3service.h"
 #include "retroshare/rsreputations.h"
 #include "retroshare/rsidentity.h"
@@ -61,9 +61,8 @@
 class RsNetworkExchangeService
 {
 public:
-
+	RsNetworkExchangeService() = default;
-	RsNetworkExchangeService(){ return;}
+	virtual ~RsNetworkExchangeService() = default;
-    virtual ~RsNetworkExchangeService() {}
 
     virtual uint16_t serviceType() const =0;
     /*!
@@ -85,9 +84,24 @@ public:
     virtual bool msgAutoSync() const =0;
     virtual bool grpAutoSync() const =0;
 
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
 	///                    DISTANT SEARCH FUNCTIONS                          ///
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
+
+	/// Trigger remote generic GXS service search
+	virtual std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
+
+	/// Handle remote generic GXS services search requests to specific service
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
+
+	/// Receive remote generic GXS services search result
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
 
     /*!
      * \brief turtleGroupRequest
@@ -121,7 +135,11 @@ public:
 	 * \param req Turtle search request ID associated with this result
 	 * \param encrypted_group_data  Group data
 	 */
-	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
+	RS_DEPRECATED_FOR("receiveDistantSearchResult")
+	virtual void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        rs_owner_ptr<const uint8_t> encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) = 0;
 
     /*!
      * \brief retrieveTurtleSearchResults
@@ -141,7 +159,9 @@ public:
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
     virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
 
+	RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
 	virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
+
 	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
 
     /*!
@@ -306,5 +326,3 @@ public:
 		}
 	}
 };
-
-#endif // RSGNP_H

libretroshare/src/gxs/rsnxsobserver.cpp

@@ -1,9 +1,8 @@
 /*******************************************************************************
- * libretroshare/src/util: rserrno.h                                           *
+ * RetroShare General eXchange System                                          *
  *                                                                             *
- * libretroshare: retroshare core library                                      *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
- *                                                                             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,6 +18,19 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#pragma once
 
-const char* rsErrnoName(int err);
+#include "gxs/rsnxsobserver.h"
+
+const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
+
+std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
+const noexcept
+{
+	switch(static_cast<RsNxsObserverErrorNum>(ev))
+	{
+	case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
+		return std::errc::operation_not_supported;
+	default:
+		return std::error_condition(ev, *this);
+	}
+}

libretroshare/src/gxs/rsnxsobserver.h

@@ -3,7 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher                *
+ * Copyright (C) 2011-2012 Robert Fernie                                       *
+ * Copyright (C) 2011-2012 Christopher Evi-Parker                              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,21 +22,61 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSNXSOBSERVER_H
+#pragma once
-#define RSNXSOBSERVER_H
 
-#include <set>
+#include <system_error>
+#include <vector>
+
+#include "retroshare/rsgxsiface.h"
 #include "rsitems/rsnxsitems.h"
+#include "util/rsdebug.h"
 
-typedef uint32_t TurtleRequestId ;
+typedef uint32_t TurtleRequestId;
+
+enum class RsNxsObserverErrorNum : int32_t
+{
+	NOT_OVERRIDDEN_BY_OBSERVER = 2004,
+};
+
+struct RsNxsObserverErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare NXS Observer"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsNxsObserverErrorNum>(ev))
+		{
+		case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
+			return "Method not overridden by observer";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	std::error_condition default_error_condition(int ev) const noexcept override;
+
+	const static RsNxsObserverErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
+}
+
+/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsJsonApiErrorNum */
+inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsNxsObserverErrorCategory::instance );
+};
 
 class RsNxsObserver
 {
-public:
-
-    RsNxsObserver() {}
-
-
 public:
 
     /*!
@@ -56,6 +99,46 @@ public:
         std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
     }
 
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * requests there.
+	 * @param[in] requestData search query
+	 * @param[in] requestSize search query size
+	 * @param[out] resultData results data storage for a pointer to search
+	 * result reply data or nullptr if no mathing results where found
+	 * @param[out] resultSize storage for results data size or 0 if no matching
+	 * results where found
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		/* Avoid unused paramethers warning this way so doxygen can still parse
+		 * paramethers documentation */
+		(void) requestData; (void) requestSize;
+		(void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * results there.
+	 * @param[in] requestId search query id
+	 * @param[out] resultData results data
+	 * @param[out] resultSize results data size
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		(void) requestId; (void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
     /*!
      * @param grpId group id
      */
@@ -70,6 +153,7 @@ public:
      * @param grpId group id
      */
     virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
-};
 
-#endif // RSNXSOBSERVER_H
+	RsNxsObserver() = default;
+	virtual ~RsNxsObserver() = default;
+};

libretroshare/src/libretroshare.pro

@@ -394,7 +394,6 @@ HEADERS +=	pqi/authssl.h \
 			pqi/pqissl.h \
 			pqi/pqissllistener.h \
 			pqi/pqisslpersongrp.h \
-                        pqi/pqissli2pbob.h \
 			pqi/pqisslproxy.h \
 			pqi/pqistore.h \
 			pqi/pqistreamer.h \
@@ -460,7 +459,7 @@ HEADERS +=	rsitems/rsitem.h \
 			rsitems/rsgxsupdateitems.h \
 			rsitems/rsserviceinfoitems.h \
 
-HEADERS +=  services/autoproxy/p3i2pbob.h \
+HEADERS +=  \
             services/rseventsservice.h \
             services/autoproxy/rsautoproxymonitor.h \
             services/p3msgservice.h \
@@ -514,8 +513,7 @@ HEADERS +=	util/folderiterator.h \
     util/cxx11retrocompat.h \
     util/cxx14retrocompat.h \
     util/cxx17retrocompat.h \
-            util/rsurl.h \
+            util/rsurl.h
-    util/rserrno.h
 
 SOURCES +=	ft/ftchunkmap.cc \
 			ft/ftcontroller.cc \
@@ -564,7 +562,6 @@ SOURCES +=	pqi/authgpg.cc \
 			pqi/pqissl.cc \
 			pqi/pqissllistener.cc \
 			pqi/pqisslpersongrp.cc \
-                        pqi/pqissli2pbob.cpp \
 			pqi/pqisslproxy.cc \
 			pqi/pqistore.cc \
 			pqi/pqistreamer.cc \
@@ -626,7 +623,6 @@ SOURCES +=	serialiser/rsbaseserial.cc \
 
 SOURCES +=  services/autoproxy/rsautoproxymonitor.cc \
     services/rseventsservice.cc \
-            services/autoproxy/p3i2pbob.cc \
             services/p3msgservice.cc \
 			services/p3service.cc \
 			services/p3statusservice.cc \
@@ -646,10 +642,10 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsexpr.cc \
 			util/smallobject.cc \
 			util/rsdir.cc \
-			util/rsmemory.cc \
 			util/rsdiscspace.cc \
 			util/rsnet.cc \
 			util/rsnet_ss.cc \
+			util/rsdnsutils.cc \
 			util/extaddrfinder.cc \
 			util/dnsresolver.cc \
 			util/rsprint.cc \
@@ -660,8 +656,7 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsrecogn.cc \
             util/rstime.cc \
             util/rsurl.cc \
-            util/rsbase64.cc \
+            util/rsbase64.cc
-    util/rserrno.cc
 
 equals(RS_UPNP_LIB, miniupnpc) {
         HEADERS += rs_upnp/upnputil.h rs_upnp/upnphandler_miniupnp.h
@@ -725,7 +720,8 @@ SOURCES += rsitems/rsnxsitems.cc \
 	gxs/gxstokenqueue.cc \
 	gxs/rsgxsnetutils.cc \
 	gxs/rsgxsutil.cc \
-	gxs/rsgxsrequesttypes.cc
+        gxs/rsgxsrequesttypes.cc \
+        gxs/rsnxsobserver.cpp
 
 # gxs tunnels
 HEADERS += gxstunnel/p3gxstunnel.h \
@@ -949,6 +945,14 @@ rs_jsonapi {
     SOURCES += jsonapi/jsonapi.cpp
 }
 
+rs_deep_forums_index {
+    HEADERS *= deep_search/commonutils.hpp
+    SOURCES *= deep_search/commonutils.cpp
+
+    HEADERS += deep_search/forumsindex.hpp
+    SOURCES += deep_search/forumsindex.cpp
+}
+
 rs_deep_channels_index {
     HEADERS *= deep_search/commonutils.hpp
     SOURCES *= deep_search/commonutils.cpp
@@ -1022,6 +1026,44 @@ rs_broadcast_discovery {
     }
 }
 
+rs_sam3 {
+    SOURCES += \
+        services/autoproxy/p3i2psam3.cpp \
+        pqi/pqissli2psam3.cpp \
+
+    HEADERS += \
+        services/autoproxy/p3i2psam3.h \
+        pqi/pqissli2psam3.h \
+}
+
+rs_sam3_libsam3 {
+    DUMMYQMAKECOMPILERINPUT = FORCE
+    libsam3.name = Generating libsam3.
+    libsam3.input = DUMMYQMAKECOMPILERINPUT
+    libsam3.output = $$clean_path($${LIBSAM3_BUILD_PATH}/libsam3.a)
+    libsam3.CONFIG += target_predeps combine
+    libsam3.variable_out = PRE_TARGETDEPS
+    win32-g++:isEmpty(QMAKE_SH) {
+        LIBSAM3_MAKE_PARAMS = CC=gcc
+        libsam3.commands = \
+            cd /D $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/libsam3 || cd . $$escape_expand(\\n\\t) \
+            $(CHK_DIR_EXISTS) $$shell_path($$LIBSAM3_BUILD_PATH) $(MKDIR) $$shell_path($${LIBSAM3_BUILD_PATH}) $$escape_expand(\\n\\t) \
+            $(COPY_DIR) $$shell_path($${LIBSAM3_SRC_PATH}) $$shell_path($${LIBSAM3_BUILD_PATH}) || cd . $$escape_expand(\\n\\t)
+    } else {
+        LIBSAM3_MAKE_PARAMS =
+        libsam3.commands = \
+            cd $${RS_SRC_PATH} && ( \
+            git submodule update --init supportlibs/libsam3 || \
+            true ) && \
+            mkdir -p $${LIBSAM3_BUILD_PATH} && \
+            (cp -r $${LIBSAM3_SRC_PATH}/* $${LIBSAM3_BUILD_PATH} || true) &&
+    }
+    libsam3.commands += \
+        cd $$shell_path($${LIBSAM3_BUILD_PATH}) && \
+        $(MAKE) build $${LIBSAM3_MAKE_PARAMS}
+    QMAKE_EXTRA_COMPILERS += libsam3
+}
+
 ###########################################################################################################
 # OLD CONFIG OPTIONS.
 # Not used much - but might be useful one day.

libretroshare/src/pqi/p3netmgr.cc

@@ -47,14 +47,15 @@ struct RsLog::logInfo p3netmgrzoneInfo = {RsLog::Default, "p3netmgr"};
 
 /* Network setup States */
 
-const uint32_t RS_NET_NEEDS_RESET = 	0x0000;
+constexpr uint32_t RS_NET_NEEDS_RESET = 0x0000;
-const uint32_t RS_NET_UNKNOWN = 	0x0001;
+constexpr uint32_t RS_NET_UNKNOWN =     0x0001;
-const uint32_t RS_NET_UPNP_INIT = 	0x0002;
+constexpr uint32_t RS_NET_UPNP_INIT =   0x0002;
-const uint32_t RS_NET_UPNP_SETUP =  	0x0003;
+constexpr uint32_t RS_NET_UPNP_SETUP =  0x0003;
-const uint32_t RS_NET_EXT_SETUP =  	0x0004;
+constexpr uint32_t RS_NET_EXT_SETUP =   0x0004;
-const uint32_t RS_NET_DONE =    	0x0005;
+constexpr uint32_t RS_NET_DONE =        0x0005;
-const uint32_t RS_NET_LOOPBACK =    	0x0006;
+constexpr uint32_t RS_NET_LOOPBACK =    0x0006;
-//const uint32_t RS_NET_DOWN =    	0x0007;
+//constexpr uint32_t RS_NET_DOWN =        0x0007;
+constexpr uint32_t RS_NET_SHUTDOWN =    0x00FF; //Highest value to not restart UPnP nor ExtAddrFinder
 
 /* Stun modes (TODO) */
 //const uint32_t RS_STUN_DHT =      	0x0001;
@@ -68,6 +69,8 @@ const uint32_t MAX_UPNP_COMPLETE = 	600; /* 10 min... seems to take a while */
 
 //const uint32_t MIN_TIME_BETWEEN_NET_RESET = 		5;
 
+const uint32_t MIN_TIME_EXT_FINDER_UPDATE = 300; /* 5min to check if external IP is changed */
+
 /****
  * #define NETMGR_DEBUG 1
  * #define NETMGR_DEBUG_RESET 1
@@ -107,19 +110,17 @@ void pqiNetStatus::print(std::ostream &out)
 }
 
 
-p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
+p3NetMgrIMPL::p3NetMgrIMPL()
-    mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
+    : mPeerMgr(nullptr), mLinkMgr(nullptr)
-    mDoNotNetCheckUntilTs(0)
+    , mNetMtx("p3NetMgr"), mNetMode(RS_NET_MODE_UDP), mVsDisc(RS_VS_DISC_FULL), mVsDht(RS_VS_DHT_FULL)// default to full.
+    , mNetInitTS(0), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
+    , mUseExtAddrFinder(true), mNetExtAddrFinderTs(0), mDoNotNetCheckUntilTs(0)
 {
 
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
-		mNetMode = RS_NET_MODE_UDP;
-
-		mUseExtAddrFinder = true;
 		mExtAddrFinder = new ExtAddrFinder();
-		mNetInitTS = 0;
 	
 		mNetFlags = pqiNetStatus();
 		mOldNetFlags = pqiNetStatus();
@@ -133,10 +134,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
 		mLocalAddr.ss_family = AF_INET;
 		mExtAddr.ss_family = AF_INET;
 
-		// default to full.
-		mVsDisc = RS_VS_DISC_FULL;
-		mVsDht = RS_VS_DHT_FULL;
-
 	}
 	
 #ifdef NETMGR_DEBUG
@@ -235,6 +232,10 @@ void p3NetMgrIMPL::netReset()
 	rslog(RSL_ALERT, p3netmgrzone, "p3NetMgr::netReset() Called");
 
 	shutdown(); /* blocking shutdown call */
+	{
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
+		mNetStatus = RS_NET_UNKNOWN;
+	}
 
 	// Will initiate a new call for determining the external ip.
 	if (mUseExtAddrFinder)
@@ -242,7 +243,7 @@ void p3NetMgrIMPL::netReset()
 #ifdef NETMGR_DEBUG_RESET
 		std::cerr << "p3NetMgrIMPL::netReset() restarting AddrFinder" << std::endl;
 #endif
-		mExtAddrFinder->reset() ;
+		mExtAddrFinder->reset(true) ;
 	}
 	else
 	{
@@ -265,7 +266,7 @@ void p3NetMgrIMPL::netReset()
 		*      as it calls back to p3ConnMgr.
 		*/
 
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		struct sockaddr_storage iaddr = mLocalAddr;
 		
@@ -283,8 +284,7 @@ void p3NetMgrIMPL::netReset()
 	}
 
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
-        	mNetStatus = RS_NET_UNKNOWN;
 		netStatusReset_locked();
 	}
 
@@ -311,8 +311,8 @@ bool p3NetMgrIMPL::shutdown() /* blocking shutdown call */
 	std::cerr << std::endl;
 #endif
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
-		mNetStatus = RS_NET_UNKNOWN;
+		mNetStatus = RS_NET_SHUTDOWN;
 		mNetInitTS = time(NULL);
 		netStatusReset_locked();
 	}
@@ -347,7 +347,7 @@ void p3NetMgrIMPL::netStartup()
 	 */
 
 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 	mNetInitTS = time(NULL);
 	netStatusReset_locked();
@@ -460,35 +460,51 @@ void p3NetMgrIMPL::netStatusTick()
 
 	uint32_t netStatus = 0;
 	rstime_t   age = 0;
+	bool needExtFinderUpdate = false;
 	{
-		RsStackMutex stack(mNetMtx);   /************** LOCK MUTEX ***************/
+		RS_STACK_MUTEX(mNetMtx);   /************** LOCK MUTEX ***************/
 
 		netStatus = mNetStatus;
 		age = time(NULL) - mNetInitTS;
 
+		needExtFinderUpdate = netStatus == RS_NET_DONE;
+		needExtFinderUpdate &= mNetExtAddrFinderTs < time(nullptr);
+		if(needExtFinderUpdate)
+			mNetExtAddrFinderTs = time(nullptr) + MIN_TIME_EXT_FINDER_UPDATE;
 	}
 
-    if(netStatus <= RS_NET_UPNP_SETUP && mUseExtAddrFinder)
+	if( mUseExtAddrFinder
+	   && ( netStatus <= RS_NET_UPNP_SETUP
+	        || needExtFinderUpdate) )
 	{
-        sockaddr_storage tmpip = mLocalAddr;	// copies local port and correctly inits the IP family
+		sockaddr_storage tmpip;
+		sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-        std::cerr << "Asking ExtAddrFinder for IP. Initializing port with " << sockaddr_storage_port(tmpip) << std::endl;
+		RS_DBG("Asking ExtAddrFinder for IP. Initializing port with ", sockaddr_storage_port(tmpip));
 #endif
 
-        if(mExtAddrFinder->hasValidIP(tmpip) && sockaddr_storage_ipv6_to_ipv4(tmpip) && !sockaddr_storage_same(tmpip,mExtAddr))
+		if(mExtAddrFinder->hasValidIPV4(tmpip))
+		{
+			if(!sockaddr_storage_same(tmpip,mExtAddr))
 			{
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-            std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
+				RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
-#endif
-
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-            std::cerr << "p3NetMgrIMPL::netExtCheck() ";
-            std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
-            std::cerr << std::endl;
 #endif
 				setExtAddress(tmpip);
 			}
 		}
+		else if(mExtAddrFinder->hasValidIPV6(tmpip))
+		{
+			if(!sockaddr_storage_same(tmpip,mExtAddr))
+			{
+				//Only if no IPv4 else, reset connections on setExtAddress()
+#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
+				RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
+#endif
+				setExtAddress(tmpip);
+			}
+		}
+	}
 
 	switch(netStatus)
 	{
@@ -578,7 +594,7 @@ void p3NetMgrIMPL::netDhtInit()
 	
 	uint32_t vs = 0;
 	{
-		RsStackMutex stack(mNetMtx); /*********** LOCKED MUTEX ************/
+		RS_STACK_MUTEX(mNetMtx); /*********** LOCKED MUTEX ************/
 		vs = mVsDht;
 	}
 	
@@ -745,25 +761,16 @@ void p3NetMgrIMPL::netExtCheck()
 			if (mUseExtAddrFinder)
 			{
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-				std::cerr << "p3NetMgrIMPL::netExtCheck() checking ExtAddrFinder" << std::endl;
+				RS_DBG("checking ExtAddrFinder");
 #endif
-                sockaddr_storage tmpip = mLocalAddr;	// copies local port and correctly inits the IP family
+				sockaddr_storage tmpip;
+				sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
 
-                bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
+				// Test for IPv4 first to be compatible with older versions.
-
+				if (mExtAddrFinder->hasValidIPV4(tmpip))
-				if (extFinderOk && sockaddr_storage_ipv6_to_ipv4(tmpip))
 				{
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-					std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
-#endif
 					sockaddr_storage_setport(tmpip, guessNewExtPort());
 
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-					std::cerr << "p3NetMgrIMPL::netExtCheck() ";
-					std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
-					std::cerr << std::endl;
-#endif
-
 					mNetFlags.mExtAddrOk = true;
 
 					address_votes[tmpip].n++ ;
@@ -773,10 +780,22 @@ void p3NetMgrIMPL::netExtCheck()
 					 * (which it is not normally) */
 					mNetFlags.mExtAddrStableOk = true;
 
-					std::cerr << __PRETTY_FUNCTION__ << " ExtAddrFinder "
+					RS_DBG("Reported external IPv4 address ", sockaddr_storage_iptostring(tmpip));
-					          << " reported external address "
+				}
-					          << sockaddr_storage_iptostring(tmpip)
+				else if (mExtAddrFinder->hasValidIPV6(tmpip))
-					          << std::endl;
+				{
+					sockaddr_storage_setport(tmpip, guessNewExtPort());
+
+					mNetFlags.mExtAddrOk = true;
+
+					address_votes[tmpip].n++ ;
+
+					/* XXX HACK TO FIX drbob: ALLOWING
+					 * ExtAddrFinder -> ExtAddrStableOk = true
+					 * (which it is not normally) */
+					mNetFlags.mExtAddrStableOk = true;
+
+					RS_DBG("Reported external IPv6 address ", sockaddr_storage_iptostring(tmpip));
 				}
 			}
 		}
@@ -967,7 +986,7 @@ void p3NetMgrIMPL::netExtCheck()
 
 	if (netSetupDone)
 	{
-		std::cerr << "p3NetMgrIMPL::netExtCheck() netSetupDone" << std::endl;
+		RS_DBG("netSetupDone");
 
 		/* Setup NetStateBox with this info */
 		updateNetStateBox_startup();
@@ -978,12 +997,14 @@ void p3NetMgrIMPL::netExtCheck()
 			mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
 		}
 
-		/* inform DHT about our external address */
+		/* inform DHT about our external IPV4 address, it doesn't support IPv6 for now.*/
+		if(sockaddr_storage_ipv6_to_ipv4(mExtAddr))
+		{
 			RsPeerId fakeId;
 			netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
+		}
 
-		std::cerr << __PRETTY_FUNCTION__ << " Network Setup Complete"
+		RS_INFO("Network Setup Complete");
-		          << std::endl;
 	}
 }
 
@@ -995,6 +1016,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 {
 	bool addrChanged = false;
 	bool validAddr = false;
+	bool needOwnAddrUpdate = false;
 	
 	sockaddr_storage prefAddr;
 	sockaddr_storage oldAddr;
@@ -1067,8 +1089,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 
 	if (!validAddr)
 	{
-		RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
+		RS_ERR("no valid local network address found. Report to developers.");
-		        <<" found. Report to developers." << std::endl;
 		print_stacktrace();
 
 		return false;
@@ -1109,6 +1130,7 @@ bool p3NetMgrIMPL::checkNetAddress()
             sockaddr_storage_setport(mExtAddr, port);	// this accounts for when the port was updated
             addrChanged = true;
 		}
+
 	} // RS_STACK_MUTEX(mNetMtx);
 
     if (addrChanged)
@@ -1123,11 +1145,24 @@ bool p3NetMgrIMPL::checkNetAddress()
             rsEvents->postEvent(ev);
         }
 
-        if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
+        needOwnAddrUpdate = true;
 
         netReset();
     }
 
+	if (mPeerMgr)
+	{
+		// Retrieve last known IP, if none, update own addresse to get current.
+		peerState ps;
+		mPeerMgr->getOwnNetStatus(ps);
+		needOwnAddrUpdate |= ps.ipAddrs.mLocal.mAddrs.empty();
+		needOwnAddrUpdate |= ps.ipAddrs.mExt.mAddrs.empty();
+		if (needOwnAddrUpdate)
+		{
+			mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
+		}
+	}
+
 	return true;
 }
 
@@ -1139,7 +1174,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 /* to allow resets of network stuff */
 void    p3NetMgrIMPL::addNetListener(pqiNetListener *listener)
 {
-        RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+        RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
         mNetListeners.push_back(listener);
 }
 
@@ -1149,7 +1184,7 @@ bool    p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
 {
 	bool changed = false;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		if (!sockaddr_storage_same(mLocalAddr, addr))
 		{
 			changed = true;
@@ -1171,7 +1206,7 @@ bool    p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
 }
 bool    p3NetMgrIMPL::getExtAddress(struct sockaddr_storage& addr)
 {
-        RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+        RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
         if(mNetFlags.mExtAddrOk)
         {
@@ -1186,7 +1221,7 @@ bool    p3NetMgrIMPL::setExtAddress(const struct sockaddr_storage &addr)
 {
 	bool changed = false;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		if (!sockaddr_storage_same(mExtAddr, addr))
 		{
 			changed = true;
@@ -1220,7 +1255,7 @@ bool    p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
 {
 	uint32_t oldNetMode;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		/* only change TRY flags */
 
 		oldNetMode = mNetMode;
@@ -1266,7 +1301,7 @@ bool    p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
 
 bool    p3NetMgrIMPL::setVisState(uint16_t vs_disc, uint16_t vs_dht)
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	mVsDisc = vs_disc;
     mVsDht = vs_dht;
 
@@ -1628,7 +1663,7 @@ void	p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
 	uint32_t netsize = 0, rsnetsize = 0;
 	netAssistConnectStats(netsize, rsnetsize);
 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 	/* quick update of the stuff that can change! */
 	mNetFlags.mDhtOk = dhtOk;
@@ -1652,7 +1687,7 @@ void	p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
 
 bool  p3NetMgrIMPL::getIPServersEnabled() 
 { 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mUseExtAddrFinder;
 }
 
@@ -1661,15 +1696,32 @@ void  p3NetMgrIMPL::getIPServersList(std::list<std::string>& ip_servers)
 	mExtAddrFinder->getIPServersList(ip_servers);
 }
 
+void  p3NetMgrIMPL::getCurrentExtIPList(std::list<std::string>& ip_list)
+{
+	ip_list.clear();
+	sockaddr_storage addr;
+	if(mExtAddrFinder->hasValidIPV4(addr))
+		ip_list.push_back(sockaddr_storage_iptostring(addr));
+	if(mExtAddrFinder->hasValidIPV6(addr))
+		ip_list.push_back(sockaddr_storage_iptostring(addr));
+}
+
 void p3NetMgrIMPL::setIPServersEnabled(bool b)
 {
+	if (mUseExtAddrFinder != b)
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		mExtAddrFinder->reset(true);
+		if (b)
+			mExtAddrFinder->start_request();
+	}
+
+	{
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		mUseExtAddrFinder = b;
 	}
 
 #ifdef NETMGR_DEBUG
-	std::cerr << "p3NetMgr: setIPServers to " << b << std::endl ; 
+	RS_DBG("set mUseExtAddrFinder to ", b);
 #endif
 
 }
@@ -1682,31 +1734,31 @@ void p3NetMgrIMPL::setIPServersEnabled(bool b)
 
 RsNetState p3NetMgrIMPL::getNetStateMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNetStateMode();
 }
 
 RsNetworkMode p3NetMgrIMPL::getNetworkMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNetworkMode();
 }
 
 RsNatTypeMode p3NetMgrIMPL::getNatTypeMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNatTypeMode();
 }
 
 RsNatHoleMode p3NetMgrIMPL::getNatHoleMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNatHoleMode();
 }
 
 RsConnectModes p3NetMgrIMPL::getConnectModes()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getConnectModes();
 }
 
@@ -1730,7 +1782,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
         	/* input network bits */
        		if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
 		{
-			RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+			RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 			mNetStateBox.setAddressStunDht(tmpaddr, isstable);
 			
 #ifdef	NETMGR_DEBUG_STATEBOX
@@ -1749,7 +1801,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
         	/* input network bits */
        		if (mProxyStunner->getExternalAddr(tmpaddr, isstable))
 		{
-			RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+			RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 			mNetStateBox.setAddressStunProxy(tmpaddr, isstable);
 
 #ifdef	NETMGR_DEBUG_STATEBOX
@@ -1768,7 +1820,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
 		bool dhtOn = netAssistConnectEnabled();
 		bool dhtActive = netAssistConnectActive();
 
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		mNetStateBox.setDhtState(dhtOn, dhtActive);
 	}
 
@@ -1777,7 +1829,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
 
 #ifdef	NETMGR_DEBUG_STATEBOX
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
         auto netstate = mNetStateBox.getNetStateMode();
         auto netMode = mNetStateBox.getNetworkMode();
@@ -1823,7 +1875,7 @@ void p3NetMgrIMPL::updateNatSetting()
 	RsNatTypeMode natType = RsNatTypeMode::UNKNOWN;
 	RsNatHoleMode natHole = RsNatHoleMode::UNKNOWN;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		natType = mNetStateBox.getNatTypeMode();
 		natHole = mNetStateBox.getNatHoleMode();
@@ -1925,7 +1977,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 	std::cerr << std::endl;
 #endif
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		/* fill in the data */
 		struct sockaddr_storage tmpip;
@@ -1974,7 +2026,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 		if (mUseExtAddrFinder)
 		{
 			tmpip = mLocalAddr;
-            bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
+			bool extFinderOk = mExtAddrFinder->hasValidIPV4(tmpip);
 
 			if (extFinderOk)
 			{
@@ -2020,7 +2072,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 void p3NetMgrIMPL::updateNetStateBox_reset()
 {
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		mNetStateBox.reset();
 

libretroshare/src/pqi/p3netmgr.h

@@ -120,6 +120,7 @@ virtual bool	shutdown() = 0; /* blocking shutdown call */
 virtual bool  getIPServersEnabled() = 0;
 virtual void  setIPServersEnabled(bool b)  = 0;
 virtual void  getIPServersList(std::list<std::string>& ip_servers)  = 0;
+virtual void  getCurrentExtIPList(std::list<std::string>& ip_list)  = 0;
 
 	// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
 virtual void 	getNetStatus(pqiNetStatus &status) = 0;
@@ -171,6 +172,7 @@ virtual bool	shutdown(); /* blocking shutdown call */
 virtual bool  getIPServersEnabled();
 virtual void  setIPServersEnabled(bool b);
 virtual void  getIPServersList(std::list<std::string>& ip_servers);
+virtual void  getCurrentExtIPList(std::list<std::string>& ip_list);
 
 	// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
 virtual void 	getNetStatus(pqiNetStatus &status);
@@ -313,6 +315,7 @@ void 	netStatusReset_locked();
 	bool     mStatusChanged;
 
 	bool     mUseExtAddrFinder;
+	rstime_t mNetExtAddrFinderTs;
 
 	/* network status flags (read by rsiface) */
 	pqiNetStatus mNetFlags;

libretroshare/src/pqi/pqissl.cc

@@ -1742,6 +1742,13 @@ bool 	pqissl::moretoread(uint32_t usec)
 	{
 		rslog(RSL_ALERT, pqisslzone, 
 			"pqissl::moretoread() Select ERROR!");
+		RS_WARN(strerror(errno));
+
+		if (errno == EBADF) {
+			// happens when SAM is shut down
+			rslog(RSL_ALERT, pqisslzone, "pqissl::moretoread() -> calling reset()");
+			reset_locked();
+		}
 		return 0;
 	}
 

libretroshare/src/pqi/pqissli2pbob.cpp

@@ -1,52 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/pqi: pqissli2pbob.cc                                      *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#include "pqissli2pbob.h"
-
-bool pqissli2pbob::connect_parameter(uint32_t type, const std::string &value)
-{
-	if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
-	{
-		RS_STACK_MUTEX(mSslMtx);
-		// a new line must be appended!
-		mI2pAddr = value + '\n';
-		return true;
-	}
-
-	return pqissl::connect_parameter(type, value);
-}
-
-int pqissli2pbob::Basic_Connection_Complete()
-{
-	int ret;
-
-	if ((ret = pqissl::Basic_Connection_Complete()) != 1)
-	{
-		// basic connection not complete.
-		return ret;
-	}
-
-	// send addr. (new line is already appended)
-	ret = send(sockfd, mI2pAddr.c_str(), mI2pAddr.length(), 0);
-	if (ret != (int)mI2pAddr.length())
-		return -1;
-	return 1;
-}

libretroshare/src/pqi/pqissli2pbob.h

@@ -1,52 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/pqi: pqissli2pbob.h                                       *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#ifndef PQISSLI2PBOB_H
-#define PQISSLI2PBOB_H
-
-#include "pqi/pqissl.h"
-
-/*
- * This class is a minimal varied version of pqissl to work with I2P BOB tunnels.
- * The only difference is that the [.b32].i2p addresses must be sent first.
- *
- * Everything else is untouched.
- */
-
-class pqissli2pbob : public pqissl
-{
-public:
-	pqissli2pbob(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
-	    : pqissl(l, parent, lm) {}
-
-	// NetInterface interface
-public:
-	bool connect_parameter(uint32_t type, const std::string &value);
-
-	// pqissl interface
-protected:
-	int Basic_Connection_Complete();
-
-private:
-	std::string mI2pAddr;
-};
-
-#endif // PQISSLI2PBOB_H

libretroshare/src/pqi/pqissli2psam3.cpp

@@ -0,0 +1,261 @@
+#include "pqissli2psam3.h"
+
+#include <libsam3.h>
+
+RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
+static constexpr int pqiDone  =  1;
+static constexpr int pqiWait  =  0;
+static constexpr int pqiError = -1;
+
+pqissli2psam3::pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
+    : pqissl(l, parent, lm), mState(pqisslSam3State::NONE), mI2pAddrB32(), mI2pAddrLong()
+{
+	RS_DBG4();
+	mConn = nullptr;
+}
+
+bool pqissli2psam3::connect_parameter(uint32_t type, const std::string &value)
+{
+	RS_DBG4();
+
+	if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
+	{
+		RS_DBG1("got addr:", value);
+		RS_STACK_MUTEX(mSslMtx);
+		mI2pAddrB32 = value;
+		return true;
+	}
+
+	return pqissl::connect_parameter(type, value);
+}
+
+int pqissli2psam3::Initiate_Connection()
+{
+	RS_DBG4();
+
+	if(waiting != WAITING_DELAY)
+	{
+		RS_ERR("Already Attempt in Progress!");
+		return pqiError;
+	}
+
+	switch (mState) {
+	case(pqisslSam3State::NONE):
+		RS_DBG2("NONE");
+	{
+		if(mConn) {
+			// how did we end up here?
+			RS_ERR("state is NONE but a connection is existing?!");
+		}
+		mConn = 0;
+		// get SAM session
+		mConn = 0;
+		samSettings ss;
+		ss.session = nullptr;
+		rsAutoProxyMonitor::taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, static_cast<void*>(&ss));
+
+		if (!!ss.session) {
+			RS_DBG3("NONE->DO_LOOKUP");
+			mState = pqisslSam3State::DO_LOOKUP;
+		} else {
+			RS_DBG3("NONE->DO_LOOKUP NOPE", ss.session);
+		}
+	}
+		break;
+	case(pqisslSam3State::DO_LOOKUP):
+		RS_DBG1("DO_LOOKUP");
+
+		if (!mI2pAddrLong.empty()) {
+			// skip lookup, it is highly unlikely/impossible for a public key to change (isn't it?)
+			mState = pqisslSam3State::WAIT_LOOKUP;
+			break;
+		}
+
+	{
+		i2p::address *addr = new i2p::address;
+		addr->clear();
+		addr->base32 = mI2pAddrB32;
+		rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::lookupKey, this, static_cast<void*>(addr));
+	}
+		mState = pqisslSam3State::WAIT_LOOKUP;
+		break;
+	case(pqisslSam3State::DO_CONNECT):
+		RS_DBG2("DO_CONNECT");
+
+	{
+		auto wrapper = new samEstablishConnectionWrapper();
+		wrapper->address.clear();
+		wrapper->address.publicKey = mI2pAddrLong;
+		wrapper->connection = nullptr;
+
+		rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::establishConnection, this, static_cast<void*>(wrapper));
+	}
+		mState = pqisslSam3State::WAIT_CONNECT;
+		break;
+	case(pqisslSam3State::DONE):
+		RS_DBG2("DONE");
+
+		if (setupSocket())
+			return pqiDone;
+		return pqiError;
+
+	/* waiting */
+	case(pqisslSam3State::WAIT_LOOKUP):
+		RS_DBG3("WAIT_LOOKUP");
+		break;
+	case(pqisslSam3State::WAIT_CONNECT):
+		RS_DBG3("WAIT_CONNECT");
+		break;
+	}
+	return pqiWait;
+}
+
+int pqissli2psam3::net_internal_close(int fd)
+{
+	RS_DBG4();
+
+	// sanity check
+	if (mConn && fd != mConn->fd) {
+		// this should never happen!
+		RS_ERR("fd != mConn");
+//		sam3CloseConnection(mConn);
+	}
+
+	// now to the actuall closing
+	int ret = pqissl::net_internal_close(fd);
+
+	rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::closeConnection, this, mConn);
+
+	// finally cleanup
+	mConn = 0;
+	mState = pqisslSam3State::NONE;
+
+	return ret;
+}
+
+void pqissli2psam3::taskFinished(taskTicket *&ticket)
+{
+	RS_DBG4();
+
+	switch (ticket->task) {
+	case autoProxyTask::lookupKey:
+	{
+		auto addr = static_cast<i2p::address*>(ticket->data);
+
+		RS_STACK_MUTEX(mSslMtx);
+		if (ticket->result == autoProxyStatus::ok) {
+			mI2pAddrLong = addr->publicKey;
+			mState = pqisslSam3State::DO_CONNECT;
+		} else {
+			waiting = WAITING_FAIL_INTERFACE;
+		}
+
+		delete addr;
+		ticket->data = nullptr;
+		addr = nullptr;
+	}
+		break;
+	case autoProxyTask::establishConnection:
+	{
+		auto wrapper = static_cast<struct samEstablishConnectionWrapper*>(ticket->data);
+
+		RS_STACK_MUTEX(mSslMtx);
+		if (ticket->result == autoProxyStatus::ok) {
+			mConn = wrapper->connection;
+			mState = pqisslSam3State::DONE;
+		} else {
+			waiting = WAITING_FAIL_INTERFACE;
+		}
+
+		delete wrapper;
+		ticket->data = nullptr;
+		wrapper = nullptr;
+	}
+		break;
+	case autoProxyTask::closeConnection:
+		// nothing to do here
+		break;
+	default:
+		RS_WARN("unkown task", ticket->task);
+	}
+
+	// clean up!
+	delete ticket;
+	ticket = nullptr;
+}
+
+bool pqissli2psam3::setupSocket()
+{
+	/*
+	 * This function contains the generis part from pqissl::Initiate_Connection()
+	 */
+	int err;
+	int osock = mConn->fd;
+
+	err = unix_fcntl_nonblock(osock);
+	if (err < 0)
+	{
+		RS_ERR("Cannot make socket NON-Blocking:", err);
+
+		waiting = WAITING_FAIL_INTERFACE;
+		net_internal_close(osock);
+		return false;
+	}
+
+#ifdef WINDOWS_SYS
+	/* Set TCP buffer size for Windows systems */
+
+	int sockbufsize = 0;
+	int size = sizeof(int);
+
+	err = getsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, &size);
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: Current TCP receive buffer size " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error getting TCP receive buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	sockbufsize = 0;
+
+	err = getsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, &size);
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: Current TCP send buffer size " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error getting TCP send buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	sockbufsize = WINDOWS_TCP_BUFFER_SIZE;
+
+	err = setsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, sizeof(sockbufsize));
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: TCP receive buffer size set to " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error setting TCP receive buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	err = setsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, sizeof(sockbufsize));
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: TCP send buffer size set to " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error setting TCP send buffer size. Error " << err << std::endl;
+	}
+#endif
+#endif // WINDOWS_SYS
+
+
+	mTimeoutTS = time(NULL) + mConnectTimeout;
+	//std::cerr << "Setting Connect Timeout " << mConnectTimeout << " Seconds into Future " << std::endl;
+
+	waiting = WAITING_SOCK_CONNECT;
+	sockfd = osock;
+
+	return true;
+}

libretroshare/src/pqi/pqissli2psam3.h

@@ -0,0 +1,47 @@
+#ifndef PQISSLI2PSAM3_H
+#define PQISSLI2PSAM3_H
+
+#include "pqi/pqissl.h"
+#include "services/autoproxy/rsautoproxymonitor.h"
+#include "services/autoproxy/p3i2psam3.h"
+
+// Use a state machine as the whole pqi code is designed around them and some operation (like lookup) might be blocking
+enum class pqisslSam3State : uint8_t {
+	NONE = 0,
+	DO_LOOKUP,
+	WAIT_LOOKUP,
+	DO_CONNECT,
+	WAIT_CONNECT,
+	DONE
+};
+
+class pqissli2psam3 : public pqissl, public autoProxyCallback
+{
+public:
+	pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm);
+
+	// NetInterface interface
+public:
+	bool connect_parameter(uint32_t type, const std::string &value);
+
+	// pqissl interface
+protected:
+	int Initiate_Connection();
+	int net_internal_close(int fd);
+
+	// autoProxyCallback interface
+public:
+	void taskFinished(taskTicket *&ticket);
+
+private:
+	bool setupSocket();
+
+private:
+	pqisslSam3State mState;
+	std::string mI2pAddrB32;
+	std::string mI2pAddrLong;
+
+	Sam3Connection *mConn;
+};
+
+#endif // PQISSLI2PSAM3_H

libretroshare/src/pqi/pqissllistener.h

@@ -186,8 +186,8 @@ public:
 	virtual int finaliseConnection(int fd, SSL *ssl, const RsPeerId& peerId,
 								   const sockaddr_storage &raddr);
 
+	RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
 private:
 	std::map<RsPeerId, pqissl*> listenaddr;
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(2)
 };

libretroshare/src/pqi/pqisslpersongrp.cc

@@ -46,7 +46,7 @@ static struct RsLog::logInfo pqipersongrpzoneInfo = {RsLog::Default, "pqipersong
 #endif
 
 #include "pqi/pqisslproxy.h"
-#include "pqi/pqissli2pbob.h"
+#include "pqi/pqissli2psam3.h"
 
 pqilistener * pqisslpersongrp::locked_createListener(const struct sockaddr_storage &laddr)
 {
@@ -74,24 +74,26 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
         std::cerr << std::endl;
 #endif
 
-		// Use pqicI2PBOB for I2P
+		// Use pqicI2P for I2P
-		pqiconnect *pqicSOCKSProxy, *pqicI2PBOB;
+		pqiconnect *pqicSOCKSProxy, *pqicI2P;
 		{
 			pqisslproxy  *pqis = new pqisslproxy((pqissllistener *) listener, pqip, mLinkMgr);
 			RsSerialiser *rss  = new RsSerialiser();
 			rss->addSerialType(new RsRawSerialiser());
 			pqicSOCKSProxy = new pqiconnect(pqip, rss, pqis);
 		}
-		if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PBOB))
+#ifdef RS_USE_I2P_SAM3
+		if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PSAM3))
 		{
-			pqissli2pbob *pqis = new pqissli2pbob((pqissllistener *) listener, pqip, mLinkMgr);
+			pqissli2psam3 *pqis = new pqissli2psam3((pqissllistener *) listener, pqip, mLinkMgr);
 			RsSerialiser *rss  = new RsSerialiser();
 			rss->addSerialType(new RsRawSerialiser());
 
-			pqicI2PBOB = new pqiconnect(pqip, rss, pqis);
+			pqicI2P = new pqiconnect(pqip, rss, pqis);
-		} else {
-			pqicI2PBOB = pqicSOCKSProxy;
 		}
+		else
+#endif // def RS_USE_I2P_SAM3
+			pqicI2P = pqicSOCKSProxy;
 
 
 		/* first select type based on peer */
@@ -101,7 +103,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
 			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_TOR_TCP, pqicSOCKSProxy);
 			break;
 		case RS_HIDDEN_TYPE_I2P:
-			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
+			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
 			break;
 		default:
 			/* peer is not a hidden one but we are */
@@ -109,7 +111,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
 			uint32_t typeOwn = mPeerMgr->getHiddenType(AuthSSL::getAuthSSL()->OwnId());
 			switch (typeOwn) {
 			case RS_HIDDEN_TYPE_I2P:
-				pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
+				pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
 				break;
 			default:
 				/* this case shouldn't happen! */

libretroshare/src/retroshare/rsevents.h

@@ -91,7 +91,7 @@ enum class RsEventType : uint32_t
     /// @see RsGxsPostedEvent
     GXS_IDENTITY                                            = 12,
 
-    /// @see RsFiles
+	/// @see RsFiles @deprecated
 	SHARED_DIRECTORIES                                      = 13,
 
     /// @see RsFiles
@@ -103,6 +103,9 @@ enum class RsEventType : uint32_t
 	/// @see rspeers.h
 	NETWORK                                                 = 16,
 
+	/** Emitted to update library clients about file hashing being completed */
+	FILE_HASHING_COMPLETED                                  = 20,
+
 	__MAX /// Used internally, keep last
 };
 

libretroshare/src/retroshare/rsfiles.h

@@ -194,7 +194,8 @@ enum class RsFileTransferEventCode: uint8_t {
     COMPLETED_FILES_REMOVED     = 0x02,	//
 };
 
-struct RsSharedDirectoriesEvent: RsEvent
+struct RS_DEPRECATED_FOR("Packing arbitrary data into an std::string is bad idea")
+RsSharedDirectoriesEvent: RsEvent
 {
 	RsSharedDirectoriesEvent()  : RsEvent(RsEventType::SHARED_DIRECTORIES), mEventCode(RsSharedDirectoriesEventCode::UNKNOWN) {}
 	~RsSharedDirectoriesEvent() override = default;
@@ -212,6 +213,31 @@ struct RsSharedDirectoriesEvent: RsEvent
     std::string mMessage;
 };
 
+struct RsFileHashingCompletedEvent: RsEvent
+{
+	RsFileHashingCompletedEvent():
+	    RsEvent(RsEventType::FILE_HASHING_COMPLETED), mHashingSpeed(0) {}
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+		RS_SERIAL_PROCESS(mFilePath);
+		RS_SERIAL_PROCESS(mFileHash);
+		RS_SERIAL_PROCESS(mHashingSpeed);
+	}
+
+	/// Complete path of the file being hashed
+	std::string mFilePath;
+
+	/// File hash, null if error occurred
+	RsFileHash mFileHash;
+
+	/// Hashing speed in MB/s
+	double mHashingSpeed;
+};
+
 struct RsFileTransferEvent: RsEvent
 {
 	RsFileTransferEvent()  : RsEvent(RsEventType::FILE_TRANSFER), mFileTransferEventCode(RsFileTransferEventCode::UNKNOWN) {}

libretroshare/src/retroshare/rsgxscircles.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
 enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
 {
 	UNKNOWN            = 0,	/// Used to detect uninizialized values.
-	PUBLIC             = 1,	/// Public distribution, based on GxsIds
+	PUBLIC             = 1,	/// Public distribution
 	EXTERNAL           = 2,	/// Restricted to an external circle, based on GxsIds
 
 	NODES_GROUP        = 3,	/// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

libretroshare/src/retroshare/rsgxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -118,6 +118,10 @@ enum class RsForumEventCode: uint8_t
 	SYNC_PARAMETERS_UPDATED  = 0x0a, /// sync and storage times have changed
 	PINNED_POSTS_CHANGED     = 0x0b, /// some posts where pinned or un-pinned
 	DELETED_FORUM            = 0x0c, /// forum was deleted by cleaning
+	DELETED_POST             = 0x0d,  /// Post deleted (usually by cleaning)
+
+	/// Distant search result received
+	DISTANT_SEARCH_RESULT    = 0x0e
 };
 
 struct RsGxsForumEvent: RsEvent
@@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
 		RS_SERIAL_PROCESS(mForumEventCode);
 		RS_SERIAL_PROCESS(mForumGroupId);
 		RS_SERIAL_PROCESS(mForumMsgId);
-		RS_SERIAL_PROCESS(mForumMsgId);
 		RS_SERIAL_PROCESS(mModeratorsAdded);
 		RS_SERIAL_PROCESS(mModeratorsRemoved);
 	}
@@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
 	~RsGxsForumEvent() override;
 };
 
+/** This event is fired once distant search results are received */
+struct RsGxsForumsDistantSearchEvent: RsEvent
+{
+	RsGxsForumsDistantSearchEvent():
+	    RsEvent(RsEventType::GXS_FORUMS),
+	    mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
+
+	RsForumEventCode mForumEventCode;
+	TurtleRequestId mSearchId;
+	std::vector<RsGxsSearchResult> mSearchResults;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mForumEventCode);
+		RS_SERIAL_PROCESS(mSearchId);
+		RS_SERIAL_PROCESS(mSearchResults);
+	}
+};
+
 class RsGxsForums: public RsGxsIfaceHelper
 {
 public:
@@ -385,6 +411,50 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) = 0;
 
+	/**
+	 * @brief Get forum content summaries
+	 * @jsonapi{development}
+	 * @param[in] forumId id of the forum of which the content is requested
+	 * @param[in] contentIds ids of requested contents, if empty summaries of
+	 *	all messages are reqeusted
+	 * @param[out] summaries storage for summaries
+	 * @return success or error details if something failed
+	 */
+	virtual std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) = 0;
+
+	/**
+	 * @brief Search the whole reachable network for matching forums and
+	 *	posts
+	 * @jsonapi{development}
+	 * An @see RsGxsForumsDistantSearchEvent is emitted when matching results
+	 *	arrives from the network
+	 * @param[in] matchString string to search into the forum and posts
+	 * @param[out] searchId storage for search id, useful to track search events
+	 *	and retrieve search results
+	 * @return success or error details
+	 */
+	virtual std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) = 0;
+
+	/**
+	 * @brief Search the local index for matching forums and posts
+	 * @jsonapi{development}
+	 * @param[in] matchString string to search into the index
+	 * @param[out] searchResults storage for searchr esults
+	 * @return success or error details
+	 */
+	virtual std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) = 0;
+
+
+	////////////////////////////////////////////////////////////////////////////
+	/* Following functions are deprecated and should not be considered a stable
+	 * to use API */
+
 	/**
 	 * @brief Create forum. Blocking API.
 	 * @jsonapi{development}

libretroshare/src/retroshare/rsgxsiface.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -31,13 +32,63 @@
 #include "rsitems/rsserviceids.h"
 #include "retroshare/rsevents.h"
 
+/*!
+ * This structure is used to transport GXS search results.
+ * It contains the group information as well as a context string to tell where
+ * the information was found.
+ * Keep it small as to make search responses as light as possible.
+ * It differs from RsGxsGroupSearchResults because it supports also results from
+ * message matches not just groups.
+ */
+struct RsGxsSearchResult : RsSerializable
+{
+	RsGxsSearchResult(): mPublishTs(0) {}
+
+	/** Id of the group which match*/
+	RsGxsGroupId mGroupId;
+
+	/** Title of the group which match */
+	std::string mGroupName;
+
+	/** Optional message id if the search match is against a message */
+	RsGxsMessageId mMsgId;
+
+	/** Optional message title if the search match is against a message */
+	std::string mMsgName;
+
+	/** Author id of the element which matched (group or message) */
+	RsGxsId mAuthorId;
+
+	/** Publish timestamp of the element which matched (group or message) */
+	rstime_t mPublishTs;
+
+	/** A snippet of content around the exact match */
+	std::string mSearchContext;
+
+	/// @see RsSerializable::serial_process
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mGroupId);
+		RS_SERIAL_PROCESS(mGroupName);
+		RS_SERIAL_PROCESS(mMsgId);
+		RS_SERIAL_PROCESS(mMsgName);
+		RS_SERIAL_PROCESS(mAuthorId);
+		RS_SERIAL_PROCESS(mPublishTs);
+		RS_SERIAL_PROCESS(mSearchContext);
+	}
+
+	virtual ~RsGxsSearchResult() = default;
+};
+
 /*!
  * This structure is used to transport group summary information when a GXS
  * service is searched. It contains the group information as well as a context
  * string to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSummary : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSummary : RsSerializable
 {
 	RsGxsGroupSummary() :
 	    mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
  * strings to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSearchResults : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSearchResults : RsSerializable
 {
+	/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
+	 * do we really need both? */
+
 	RsGxsGroupSearchResults()
         : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
     {}
@@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
 	virtual ~RsGxsGroupSearchResults() = default;
 };
 
+
 /*!
  * Stores ids of changed gxs groups and messages.
  * It is used to notify about GXS changes.

libretroshare/src/retroshare/rsgxsifacetypes.h

@@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
 	    mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
 	    mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
 
-    virtual ~RsGroupMetaData() {}
+	virtual ~RsGroupMetaData() = default;
 
     void operator =(const RsGxsGrpMetaData& rGxsMeta);
     RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

libretroshare/src/retroshare/rsinit.h

@@ -195,7 +195,7 @@ public:
 	/*
 	 * Setup Hidden Location;
 	 */
-	static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob);
+	static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p);
 
 	static bool LoadPassword(const std::string& passwd) ;
 

libretroshare/src/retroshare/rspeers.h

@@ -788,6 +788,7 @@ public:
 	virtual bool setProxyServer(const uint32_t type, const std::string &addr, const uint16_t port) = 0;
 
 	virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
+	virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
 	virtual void allowServerIPDetermination(bool) = 0;
 	virtual bool resetOwnExternalAddressList() = 0;
 	virtual bool getAllowServerIPDetermination() = 0 ;

libretroshare/src/retroshare/rstypes.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2004-2006  Robert Fernie <retroshare@lunamutt.com>            *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -135,17 +137,10 @@ struct PeerBandwidthLimits : RsSerializable
 	}
 };
 
-//class SearchRequest // unused stuff.
-//{
-//	public:
-//	int searchId;
-//	RsCertId toId;  /* all zeros for everyone! */
-//	std::list<Condition> tests;
-//};
-
-
 /********************** For FileCache Interface *****************/
 
+/* G10h4ck: Having this kind of stuff on public headers is pretty dangerous for
+ * name space pollution, a C++11 enum class should be used instad ASAP */
 #define DIR_TYPE_UNKNOWN	    0x00
 #define DIR_TYPE_ROOT		0x01
 #define DIR_TYPE_PERSON  	0x02
@@ -258,7 +253,10 @@ struct DirStub : RsSerializable
 
 	uint8_t type;
 	std::string name;
-	void *ref;
+
+	/* G10h4ck do we still need to keep this as void* instead of uint64_t for
+	 * retroshare-gui sake? */
+	void* ref;
 
 	/// @see RsSerializable
 	void serial_process(RsGenericSerializer::SerializeJob j,
@@ -267,29 +265,12 @@ struct DirStub : RsSerializable
 		RS_SERIAL_PROCESS(type);
 		RS_SERIAL_PROCESS(name);
 
-#if defined(__GNUC__) && !defined(__clang__)
+		/* Enforce serialization as uint64_t because void* changes size (usually
-#	pragma GCC diagnostic ignored "-Wstrict-aliasing"
+		 * 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
-#endif // defined(__GNUC__) && !defined(__clang__)
+		 * architectures and make JSON API behave inconsistenly. */
-		// (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
+		uint64_t handle = reinterpret_cast<uint64_t>(ref);
-        // preferable to force type serialization to the correct size rather than letting the compiler choose for us.
-        // /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
-
-		if(sizeof(ref) == 4)
-		{
-			std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
 		RS_SERIAL_PROCESS(handle);
-		}
+		ref = reinterpret_cast<void*>(handle);
-		else if(sizeof(ref) == 8)
-		{
-			std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
-			RS_SERIAL_PROCESS(handle);
-		}
-		else
-			std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#	pragma GCC diagnostic pop
-#endif // defined(__GNUC__) && !defined(__clang__)
 	}
 };
 
@@ -330,9 +311,9 @@ struct DirDetails : RsSerializable
 	void serial_process(RsGenericSerializer::SerializeJob j,
 	                    RsGenericSerializer::SerializeContext& ctx) override
 	{
-		/* Enforce serialization as uint64_t because void* changes size
+		/* Enforce serialization as uint64_t because void* changes size (usually
-		 * depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
+		 * 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
-		 */
+		 * architectures and make JSON API behave inconsistenly. */
 		uint64_t handle = reinterpret_cast<uint64_t>(ref);
 		RS_SERIAL_PROCESS(handle);
 		ref = reinterpret_cast<void*>(handle);

libretroshare/src/rsitems/rsgxsforumitems.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2012  Robert Fernie <retroshare@lunamutt.com>                 *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RS_GXS_FORUM_ITEMS_H
+#pragma once
-#define RS_GXS_FORUM_ITEMS_H
 
 #include <map>
 
@@ -31,7 +32,18 @@
 
 #include "retroshare/rsgxsforums.h"
 
+enum class RsGxsForumsItems : uint8_t
+{
+	GROUP_ITEM     = 0x02,
+	MESSAGE_ITEM   = 0x03,
+	SEARCH_REQUEST = 0x04,
+	SEARCH_REPLY   = 0x05,
+};
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM   = 0x02;
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
 
 class RsGxsForumGroupItem : public RsGxsGrpItem
@@ -61,6 +73,48 @@ public:
 	RsGxsForumMsg mMsg;
 };
 
+struct RsGxsForumsSearchRequest : RsSerializable
+{
+	RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Store search match string
+	std::string mQuery;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mQuery);
+	}
+
+	~RsGxsForumsSearchRequest() override = default;
+};
+
+struct RsGxsForumsSearchReply : RsSerializable
+{
+	RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Results storage
+	std::vector<RsGxsSearchResult> mResults;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mResults);
+	}
+
+	~RsGxsForumsSearchReply() override = default;
+};
+
 class RsGxsForumSerialiser : public RsServiceSerializer
 {
 public:
@@ -69,5 +123,3 @@ public:
 
 	virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
 };
-
-#endif /* RS_GXS_FORUM_ITEMS_H */

libretroshare/src/rsitems/rsitem.h

@@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
 	virtual void serial_process(RsGenericSerializer::SerializeJob,
 	                            RsGenericSerializer::SerializeContext&)// = 0;
 	{
-		std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
+		RS_ERR( "called by an item using new serialization system without "
-		          << "new serialization classes, but not derived! Class is "
+		        "overriding Class is: ", typeid(*this).name() );
-		          << typeid(*this).name() << std::endl;
 		print_stacktrace();
 	}
 

libretroshare/src/rsitems/rsserviceids.h

@@ -28,7 +28,9 @@
 
 enum class RsServiceType : uint16_t
 {
-	NONE                       = 0, /// To detect non-initialized reads
+	/// To detect non-initialized items
+	NONE                       = 0,
+
 	GOSSIP_DISCOVERY           = 0x0011,
 	CHAT                       = 0x0012,
 	MSG                        = 0x0013,
@@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
 	GWEMAIL_MAIL               = 0x0025,
 	SERVICE_CONTROL            = 0x0026,
 	DISTANT_CHAT               = 0x0027,
+
+	/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
 	GXS_TUNNEL                 = 0x0028,
+
 	BANLIST                    = 0x0101,
 	STATUS                     = 0x0102,
     FRIEND_SERVER              = 0x0103,
@@ -59,6 +64,7 @@ enum class RsServiceType : uint16_t
 	POSTED                     = 0x0216,
 	CHANNELS                   = 0x0217,
 	GXSCIRCLE                  = 0x0218,
+
 	/// not gxs, but used with identities.
 	REPUTATION                 = 0x0219,
 	GXS_RECOGN                 = 0x0220,
@@ -69,13 +75,13 @@ enum class RsServiceType : uint16_t
 	CHANNELS_CONFIG            = 0x0317,
 	RTT                        = 0x1011, /// Round Trip Time
 
-
-	/***************** IDS ALLOCATED FOR PLUGINS ******************/
-	// 2000+
 	PLUGIN_ARADO_ID            = 0x2001,
 	PLUGIN_QCHESS_ID           = 0x2002,
 	PLUGIN_FEEDREADER          = 0x2003,
 
+	/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
+	GXS_DISTANT                = 0x2233,
+
 	/// Reserved for packet slicing probes.
 	PACKET_SLICING_PROBE       = 0xAABB,
 

libretroshare/src/rsserver/p3face.h

@@ -43,7 +43,7 @@
 
 class p3heartbeat;
 class p3discovery2;
-class p3I2pBob;
+class p3I2pSam3;
 
 /* GXS Classes - just declare the classes.
    so we don't have to totally recompile to switch */
@@ -161,8 +161,8 @@ public:
 		p3ChatService *chatSrv;
 		p3StatusService *mStatusSrv;
 		p3GxsTunnelService *mGxsTunnels;
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-		p3I2pBob *mI2pBob;
+		p3I2pSam3 *mI2pSam3;
 #endif
 
         // This list contains all threaded services. It will be used to shut them down properly.

libretroshare/src/rsserver/p3peers.cc

@@ -374,7 +374,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
 			sockaddr_storage_ipv6_to_ipv4(it->mAddr);
 			std::string toto;
 			toto += sockaddr_storage_tostring(it->mAddr);
-			rs_sprintf_append(toto, "    %ld sec", time(NULL) - it->mSeenTime);
+			rs_sprintf_append(toto, "    %ld sec loc", time(NULL) - it->mSeenTime);
 			d.ipAddressList.push_back(toto);
 		}
 		for(it = ps.ipAddrs.mExt.mAddrs.begin(); it != ps.ipAddrs.mExt.mAddrs.end(); ++it)
@@ -382,7 +382,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
 			sockaddr_storage_ipv6_to_ipv4(it->mAddr);
 			std::string toto;
 			toto += sockaddr_storage_tostring(it->mAddr);
-			rs_sprintf_append(toto, "    %ld sec", time(NULL) - it->mSeenTime);
+			rs_sprintf_append(toto, "    %ld sec ext", time(NULL) - it->mSeenTime);
 			d.ipAddressList.push_back(toto);
 		}
 	}
@@ -860,8 +860,14 @@ void p3Peers::getIPServersList(std::list<std::string>& ip_servers)
 {
 	mNetMgr->getIPServersList(ip_servers) ;
 }
+void p3Peers::getCurrentExtIPList(std::list<std::string>& ip_list)
+{
+	mNetMgr->getCurrentExtIPList(ip_list) ;
+}
 bool p3Peers::resetOwnExternalAddressList()
 {
+	//TODO Phenom 2021-10-30: Need to call something like mNetMgr->netReset();
+	// to update this addresslist.
 	return mPeerMgr->resetOwnExternalAddressList();
 }
 void p3Peers::allowServerIPDetermination(bool b)
@@ -1272,7 +1278,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
         }
 #else
         sockaddr_storage tLocal;
-        if(sockaddr_storage_inet_pton(tLocal, tDetails.localAddr) && sockaddr_storage_isValidNet(tLocal)  && sockaddr_storage_ipv6_to_ipv4(tLocal) && tDetails.localPort )
+        bool validLoc =   sockaddr_storage_inet_pton(tLocal, tDetails.localAddr)
+                       && sockaddr_storage_isValidNet(tLocal)
+                       && tDetails.localPort;
+        bool isLocIpv4 = sockaddr_storage_ipv6_to_ipv4(tLocal);
+        if(validLoc && isLocIpv4)
         {
             uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tLocal).sin_addr.s_addr;
 
@@ -1290,7 +1300,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
         }
 
         sockaddr_storage tExt;
-        if(sockaddr_storage_inet_pton(tExt, tDetails.extAddr) && sockaddr_storage_isValidNet(tExt)  && sockaddr_storage_ipv6_to_ipv4(tExt) && tDetails.extPort )
+        bool validExt =   sockaddr_storage_inet_pton(tExt, tDetails.extAddr)
+                       && sockaddr_storage_isValidNet(tExt)
+                       && tDetails.extPort;
+        bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
+        if(validExt && isExtIpv4)
         {
             uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tExt).sin_addr.s_addr;
 
@@ -1306,6 +1320,17 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
 
             offset += 4+2;
         }
+        else if(validExt && !isExtIpv4)
+        {
+            // External address is IPv6, save it on LOCATOR
+            sockaddr_storage_setport(tExt,tDetails.extPort);
+            std::string tLocator = sockaddr_storage_tostring(tExt);
+
+            addPacketHeader(RsShortInviteFieldType::LOCATOR, tLocator.size(),buf,offset,buf_size);
+            memcpy(&buf[offset],tLocator.c_str(),tLocator.size());
+
+            offset += tLocator.size();
+        }
 #endif
 
     }
@@ -1457,9 +1482,27 @@ std::string p3Peers::GetRetroshareInvite( const RsPeerId& sslId, RetroshareInvit
 
 	if (getPeerDetails(ssl_id, detail))
 	{
-        if(!(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY) || detail.isHiddenNode)
+		if(   !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
+		   || detail.isHiddenNode)
 			detail.ipAddressList.clear();
 
+		//Check if external address is IPv6, then move it to ipAddressList as RsCertificate only allow 4 numbers.
+		sockaddr_storage tExt;
+		bool validExt =   sockaddr_storage_inet_pton(tExt, detail.extAddr)
+		               && sockaddr_storage_isValidNet(tExt)
+		               && detail.extPort;
+		bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
+
+		if(   !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
+		   && !detail.isHiddenNode
+		   && validExt && !isExtIpv4)
+		{
+			sockaddr_storage_setport(tExt,detail.extPort);
+			detail.ipAddressList.push_front(sockaddr_storage_tostring(tExt) + " "); // Space needed to later parse.
+			detail.extAddr = ""; //Clear it to not trigg error.
+			detail.extPort = 0;
+		}
+
 		unsigned char *mem_block = nullptr;
 		size_t mem_block_size = 0;
 

libretroshare/src/rsserver/p3peers.h

@@ -132,6 +132,7 @@ public:
 	virtual bool isProxyAddress(const uint32_t type, const sockaddr_storage &addr);
 
     virtual void getIPServersList(std::list<std::string>& ip_servers) override;
+    virtual void getCurrentExtIPList(std::list<std::string>& ip_list) override;
     virtual void allowServerIPDetermination(bool) override;
     virtual bool getAllowServerIPDetermination() override;
     virtual bool resetOwnExternalAddressList() override;

libretroshare/src/rsserver/rsinit.cc

@@ -2,7 +2,8 @@
  * libretroshare/src/retroshare: rsinit.cc                                     *
  *                                                                             *
  * Copyright (C) 2004-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2016-2019  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2021       Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -171,7 +172,7 @@ struct RsInitConfig
 		std::string hiddenNodeAddress;
 		uint16_t    hiddenNodePort;
 
-		bool        hiddenNodeI2PBOB;
+		bool        hiddenNodeI2P;
 
 		/* Logging */
 		bool haveLogFile;
@@ -665,13 +666,13 @@ void RsInit::setAutoLogin(bool autoLogin){
 }
 
 /* Setup Hidden Location; */
-void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob)
+void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p)
 {
 	/* parse the bugger (todo) */
 	rsInitConfig->hiddenNodeSet = true;
 	rsInitConfig->hiddenNodeAddress = hiddenaddress;
 	rsInitConfig->hiddenNodePort = port;
-	rsInitConfig->hiddenNodeI2PBOB = useBob;
+	rsInitConfig->hiddenNodeI2P = useI2p;
 }
 
 
@@ -718,7 +719,7 @@ RsGRouter *rsGRouter = NULL ;
 #	include "rs_upnp/upnphandler_miniupnp.h"
 #endif // def RS_USE_LIBUPNP
 
-#include "services/autoproxy/p3i2pbob.h"
+#include "services/autoproxy/p3i2psam3.h"
 #include "services/autoproxy/rsautoproxymonitor.h"
 
 #include "services/p3gxsreputation.h"
@@ -924,9 +925,9 @@ int RsServer::StartupRetroShare()
 	mNetMgr->setManagers(mPeerMgr, mLinkMgr);
 
 	rsAutoProxyMonitor *autoProxy = rsAutoProxyMonitor::instance();
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	mI2pBob = new p3I2pBob(mPeerMgr);
+	mI2pSam3 = new p3I2pSam3(mPeerMgr);
-	autoProxy->addProxy(autoProxyType::I2PBOB, mI2pBob);
+	autoProxy->addProxy(autoProxyType::I2PSAM3, mI2pSam3);
 #endif
 
 	//load all the SSL certs as friends
@@ -1340,22 +1341,26 @@ int RsServer::StartupRetroShare()
     mWiki->setNetworkExchangeService(wiki_ns) ;
 #endif
 
-        /**** Forum GXS service ****/
+	/************************* Forum GXS service ******************************/
 
-        RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
+	RsGeneralDataService* gxsforums_ds = new RsDataService(
-                                                            RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
+	            currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
+	            nullptr, rsInitConfig->gxs_passwd );
 
+	p3GxsForums* mGxsForums = new p3GxsForums(
+	            gxsforums_ds, nullptr, mGxsIdService );
 
-        p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
+	RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
+#ifdef RS_DEEP_FORUMS_INDEX
+	gxsForumsTunnelService = mGxsNetTunnel;
+#endif
 
-        // create GXS photo service
 	RsGxsNetService* gxsforums_ns = new RsGxsNetService(
-                        RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
+	            RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
-                        mGxsForums, mGxsForums->getServiceInfo(),
+	            mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
-			mReputations, mGxsCircles,mGxsIdService,
+	            mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
-		    pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
+	mGxsForums->setNetworkExchangeService(gxsforums_ns);
 
-    mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
 
         /**** Channel GXS service ****/
 
@@ -1602,7 +1607,10 @@ int RsServer::StartupRetroShare()
 	/**************************************************************************/
     // Turtle search for GXS services
 
-    mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
+	mGxsNetTunnel->registerSearchableService(gxschannels_ns);
+#ifdef RS_DEEP_FORUMS_INDEX
+	mGxsNetTunnel->registerSearchableService(gxsforums_ns);
+#endif
 
 	/**************************************************************************/
 
@@ -1659,8 +1667,9 @@ int RsServer::StartupRetroShare()
 	mConfigMgr->addConfiguration("wire.cfg", wire_ns);
 #endif
 #endif //RS_ENABLE_GXS
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pBob);
+	// to make migration easiert, SAM will use BOBs configuration, as they are compatible / the same.
+	mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pSam3);
 #endif
 
 	mPluginsManager->addConfigurations(mConfigMgr) ;
@@ -1713,34 +1722,33 @@ int RsServer::StartupRetroShare()
 	{
 		std::cout << "RsServer::StartupRetroShare setting up hidden locations" << std::endl;
 
-		if (rsInitConfig->hiddenNodeI2PBOB) {
+		if (rsInitConfig->hiddenNodeI2P) {
-			std::cout << "RsServer::StartupRetroShare setting up BOB" << std::endl;
+			std::cout << "RsServer::StartupRetroShare setting up SAMv3" << std::endl;
 
 			// we need a local port!
 			mNetMgr->checkNetAddress();
 
 			// add i2p proxy
-			// bob will use this address
 			sockaddr_storage i2pInstance;
 			sockaddr_storage_ipv4_aton(i2pInstance, rsInitConfig->hiddenNodeAddress.c_str());
 			mPeerMgr->setProxyServerAddress(RS_HIDDEN_TYPE_I2P, i2pInstance);
 
 			std::string addr; // will be set by auto proxy service
-			uint16_t port = rsInitConfig->hiddenNodePort; // unused by bob
+			uint16_t port; // unused by SAM
 
-			bool r = autoProxy->initialSetup(autoProxyType::I2PBOB, addr, port);
+			bool r = autoProxy->initialSetup(autoProxyType::I2PSAM3, addr, port);
 
 			if (r && !addr.empty()) {
 				mPeerMgr->setupHiddenNode(addr, port);
 
-				// now enable bob
+				// now enable SAM
-				bobSettings bs;
+				samSettings ss;
-				autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::getSettings, &bs);
+				autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, &ss);
-				bs.enable = true;
+				ss.enable = true;
-				autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &bs);
+				autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::setSettings, &ss);
 			} else {
 				std::cerr << "RsServer::StartupRetroShare failed to receive keys" << std::endl;
-				/// TODO add notify for failed bob setup
+				/// TODO add notify for failed i2p setup
 			}
 		} else {
 			mPeerMgr->setupHiddenNode(rsInitConfig->hiddenNodeAddress, rsInitConfig->hiddenNodePort);
@@ -1762,19 +1770,17 @@ int RsServer::StartupRetroShare()
 	if (rsInitConfig->hiddenNodeSet) {
 		// newly created location
 		// mNetMgr->checkNetAddress() will setup ports for us
+
+#if 0 // this was used for BOB but is not requires for SAMv3
 		// trigger updates for auto proxy services
 		std::vector<autoProxyType::autoProxyType_enum> types;
-
-		// i2p bob need to rebuild its command map
-		types.push_back(autoProxyType::I2PBOB);
-
 		rsAutoProxyMonitor::taskSync(types, autoProxyTask::reloadConfig);
+#endif
 	}
 
 	/**************************************************************************/
 	/* startup (stuff dependent on Ids/peers is after this point) */
 	/**************************************************************************/
-
 	autoProxy->startAll();
 
 	pqih->init_listener();
@@ -1807,8 +1813,8 @@ int RsServer::StartupRetroShare()
 	/**************************************************************************/
 
 	// auto proxy threads
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	startServiceThread(mI2pBob, "I2P-BOB");
+	startServiceThread(mI2pSam3, "I2P-SAM3");
 #endif
 
 #ifdef RS_ENABLE_GXS

libretroshare/src/services/autoproxy/p3i2pbob.h

@@ -1,256 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/services/autoproxy: p3i2pbob.h                            *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#ifndef P3I2PBOB_H
-#define P3I2PBOB_H
-
-#include <map>
-#include <queue>
-#include <sys/types.h>
-#include "util/rstime.h"
-#ifndef WINDOWS_SYS
-	#include <sys/socket.h>
-#endif
-
-#include "pqi/p3cfgmgr.h"
-#include "services/autoproxy/rsautoproxymonitor.h"
-#include "util/rsthreads.h"
-#include "util/i2pcommon.h"
-
-/*
- * This class implements I2P BOB (BASIC OPEN BRIDGE) communication to allow RS
- * to automatically remote control I2P to setup the needed tunnel.
- * BOB is a simple text-based interface: https://geti2p.net/en/docs/api/bob
- *
- * Note 1:
- *	One tunnel is enough even for hidden locations since it can be used
- *	bidirectional. (In contrast to what RS I2P users had to set up manually.)
- *
- * Note 2:
- *	BOB tunnels are no SOCKS tunnel. Therefore pqissli2pbob implements a simplified
- *	proxy specially for BOB tunnels.
- *
- * Note 3:
- *	BOB needs a unique name as an ID for each tunnel.
- *	We use 'RetroShare-' + 8 random base32 characters.
- *
- * Design:
- *	The service uses three state machines to manage its task:
- *		int stateMachineBOB();
- *			mBOBState
- *		int stateMachineController();
- *			mState
- *			mTask
- *
- * stateMachineBOB:
- *	This state machine manages the low level communication with BOB. It basically has a linked
- *	list (currently a implemented as a std::map) that contains a command and the next
- *	state.
- *	Each high level operation (start up / shut down / get keys) is represented by a
- *	chain of states. E.g. the chain to retrieve new keys:
- *		mCommands[bobState::setnickN] = {setnick, bobState::newkeysN};
- *		mCommands[bobState::newkeysN] = {newkeys, bobState::getkeys};
- *		mCommands[bobState::getkeys]  = {getkeys, bobState::clear};
- *		mCommands[bobState::clear]    = {clear,   bobState::quit};
- *		mCommands[bobState::quit]     = {quit,    bobState::cleared};
- *
- * stateMachineController:
- *	This state machine manages the high level tasks.
- *	It is controlled by mState and mTask.
- *
- *		mTast:
- *			Tracks the high level operation (like start up).
- *			It will keep its value even when a task is done to track
- *			the requested BOB state.
- *			When other operations are performed like a conection check
- *			the last task gets backed up and is later restored again
- *
- *		mState:
- *			This state lives only for one operation an manages the communication
- *			with the BOB instance. This is basically connecting, starting BOB
- *			protocol and disconnecting
- *
- * How a task looks like:
- *	1) RS sets task using the ticket system
- *	2) stateMachineController connects to BOBs control port, sets mBobState to a lists head
- *	3) stateMachineBOB processes command chain
- *	4) stateMachineBOB is done and sets mBobState to cleared signaling that the connection
- *		is cleared and can be closed
- *	5) stateMachineController disconnects from BOBs control port and updates mState
- */
-
-///
-/// \brief The controllerState enum
-/// States for the controller to keep track of what he is currently doing
-enum controllerState {
-	csIdel,
-	csDoConnect,
-	csConnected,
-	csWaitForBob,
-	csDoDisconnect,
-	csDisconnected,
-	csError
-};
-
-///
-/// \brief The controllerTask enum
-/// This state tracks the controllers tast (e.g. setup a BOB tunnel or shut down
-/// an existing one).
-enum controllerTask {
-	ctIdle,
-	ctRunSetUp,
-	ctRunShutDown,
-	ctRunGetKeys,
-	ctRunCheck
-};
-
-///
-/// \brief The bobState enum
-/// One state for each message
-///
-enum bobState {
-	bsCleared,
-	bsSetnickC, // chain head for only client tunnel
-	bsSetnickN, // chain head for getting new (server) keys
-	bsSetnickS, // chain head for client and server tunnel
-	bsGetnick,
-	bsNewkeysC, // part of chain for only client tunnel
-	bsNewkeysN, // part of chain for getting new (server) keys
-	bsGetkeys,
-	bsSetkeys,
-	bsInhost,
-	bsOuthost,
-	bsInport,
-	bsOutport,
-	bsInlength,
-	bsOutlength,
-	bsInvariance,
-	bsOutvariance,
-	bsInquantity,
-	bsOutquantity,
-	bsQuiet,
-	bsStart,
-	bsStop,
-	bsClear,
-	bsList, // chain head for 'list' command
-	bsQuit
-};
-
-///
-/// \brief The bobStateInfo struct
-/// State machine with commands
-/// \todo This could be replaced by a linked list instead of a map
-struct bobStateInfo {
-	std::string command;
-	bobState    nextState;
-};
-
-struct bobSettings : i2p::settings {};
-
-///
-/// \brief The bobStates struct
-/// This container struct is used to pass all states.
-/// Additionally, the tunnel name is included to to show it in the GUI.
-/// The advantage of a struct is that it can be forward declared.
-struct bobStates {
-	bobState bs;
-	controllerState cs;
-	controllerTask ct;
-
-	std::string tunnelName;
-};
-
-class p3PeerMgr;
-
-class p3I2pBob : public RsTickingThread, public p3Config, public autoProxyService
-{
-public:
-	explicit p3I2pBob(p3PeerMgr *peerMgr);
-
-	// autoProxyService interface
-public:
-	bool isEnabled();
-	bool initialSetup(std::string &addr, uint16_t &);
-	void processTaskAsync(taskTicket *ticket);
-	void processTaskSync(taskTicket *ticket);
-
-	void threadTick() override; /// @see RsTickingThread
-
-private:
-	int stateMachineBOB();
-	int stateMachineBOB_locked_failure(const std::string &answer, const bobStateInfo &currentState);
-
-	int stateMachineController();
-	int stateMachineController_locked_idle();
-	int stateMachineController_locked_connected();
-	int stateMachineController_locked_disconnected();
-	int stateMachineController_locked_error();
-
-	// p3Config interface
-protected:
-	RsSerialiser *setupSerialiser();
-	bool saveList(bool &cleanup, std::list<RsItem *> &lst);
-	bool loadList(std::list<RsItem *> &load);
-
-private:
-	// helpers
-	void getBOBSettings(bobSettings *settings);
-	void setBOBSettings(const bobSettings *settings);
-	void getStates(bobStates *bs);
-
-	std::string executeCommand(const std::string &command);
-	bool connectI2P();
-	bool disconnectI2P();
-
-	void finalizeSettings_locked();
-	void updateSettings_locked();
-
-	std::string recv();
-
-	// states for state machines
-	controllerState mState;
-	controllerTask	mTask;
-	// used to store old state when in error state
-	// mStateOld is also used as a flag when an error occured in BOB protocol
-	controllerState mStateOld;
-	// mTaskOld is used to keep the previous task (start up / shut down) when requesting keys or checking the connection
-	controllerTask	mTaskOld;
-	bobSettings mSetting;
-	bobState	mBOBState;
-
-	// used variables
-	p3PeerMgr *mPeerMgr;
-	bool mConfigLoaded;
-	int mSocket;
-	rstime_t mLastProxyCheck;
-	sockaddr_storage mI2PProxyAddr;
-	std::map<bobState, bobStateInfo> mCommands;
-	std::string mErrorMsg;
-	std::string mTunnelName;
-
-	std::queue<taskTicket *> mPending;
-	taskTicket *mProcessing;
-
-	// mutex
-	RsMutex mLock;
-};
-
-#endif // P3I2PBOB_H

libretroshare/src/services/autoproxy/p3i2psam3.cpp

@@ -0,0 +1,798 @@
+#include "p3i2psam3.h"
+
+#include <libsam3.h>
+
+#include "pqi/p3peermgr.h"
+#include "rsitems/rsconfigitems.h"
+
+
+static const std::string kConfigKeySAM3Enable  = "SAM3_ENABLE";
+
+static const std::string kConfigKeyDestPriv    = "DEST_PRIV";
+
+static const std::string kConfigKeyInLength    = "IN_LENGTH";
+static const std::string kConfigKeyInQuantity  = "IN_QUANTITY";
+static const std::string kConfigKeyInVariance  = "IN_VARIANCE";
+static const std::string kConfigKeyInBackupQuantity = "IN_BACKUPQUANTITY";
+
+static const std::string kConfigKeyOutLength   = "OUT_LENGTH";
+static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY";
+static const std::string kConfigKeyOutVariance = "OUT_VARIANCE";
+static const std::string kConfigKeyOutBackupQuantity = "OUT_BACKUPQUANTITY";
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+// used for migration from BOB to SAM
+static const std::string kConfigKeyBOBEnable   = "BOB_ENABLE";
+static const std::string kConfigKeyBOBKey      = "BOB_KEY";
+static const std::string kConfigKeyBOBAddr     = "BOB_ADDR";
+#endif
+
+static constexpr bool   kDefaultSAM3Enable = false;
+
+RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
+static void inline doSleep(std::chrono::duration<long, std::ratio<1,1000>> timeToSleepMS) {
+	std::this_thread::sleep_for(timeToSleepMS);
+}
+
+p3I2pSam3::p3I2pSam3(p3PeerMgr *peerMgr) :
+    mConfigLoaded(false), mPeerMgr(peerMgr), mPending(), mLock("p3i2p-sam3")
+#ifdef RS_USE_I2P_SAM3_LIBSAM3
+  , mLockSam3Access("p3i2p-sam3-access")
+#endif
+{
+	RS_DBG4();
+
+	// set defaults
+	mSetting.initDefault();
+	mSetting.enable  = kDefaultSAM3Enable;
+	mSetting.session = nullptr;
+
+	libsam3_debug = 0;
+}
+
+bool p3I2pSam3::isEnabled()
+{
+	RS_STACK_MUTEX(mLock);
+	return mSetting.enable;
+}
+
+bool p3I2pSam3::initialSetup(std::string &addr, uint16_t &/*port*/)
+{
+	RS_DBG4();
+
+	RS_STACK_MUTEX(mLock);
+
+	if (!mSetting.address.publicKey.empty() || !mSetting.address.privateKey.empty())
+		RS_WARN("overwriting keys!");
+
+	bool success = generateKey(mSetting.address.publicKey, mSetting.address.privateKey);
+
+	if (!success) {
+		RS_WARN("failed to retrieve keys");
+		return false;
+	} else {
+		std::string s, c;
+		i2p::getKeyTypes(mSetting.address.publicKey, s, c);
+		RS_INFO("received key ", s, " ", c);
+		RS_INFO("public key:      ", mSetting.address.publicKey);
+		RS_INFO("private key:     ", mSetting.address.privateKey);
+		RS_INFO("address:         ", i2p::keyToBase32Addr(mSetting.address.publicKey));
+
+		// sanity check
+		auto pub = i2p::publicKeyFromPrivate(mSetting.address.privateKey);
+		RS_INFO("pub key derived: ", pub);
+		RS_INFO("address:         ", i2p::keyToBase32Addr(pub));
+		if (pub != mSetting.address.publicKey) {
+			RS_WARN("public key does not match private key! fixing ...");
+			mSetting.address.publicKey = pub;
+		}
+
+		mSetting.address.base32 = i2p::keyToBase32Addr(mSetting.address.publicKey);
+
+		IndicateConfigChanged();
+	}
+
+	addr = mSetting.address.base32;
+	return true;
+}
+
+void p3I2pSam3::processTaskAsync(taskTicket *ticket)
+{
+	RS_DBG4();
+
+	switch (ticket->task) {
+	case autoProxyTask::stop: [[fallthrough]];
+	case autoProxyTask::start: [[fallthrough]];
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey: [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck: [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
+	{
+		RS_STACK_MUTEX(mLock);
+		mPending.push(ticket);
+	}
+		break;
+	case autoProxyTask::status: [[fallthrough]];
+	case autoProxyTask::getSettings: [[fallthrough]];
+	case autoProxyTask::setSettings: [[fallthrough]];
+	case autoProxyTask::getErrorInfo: [[fallthrough]];
+	case autoProxyTask::reloadConfig:
+		// These are supposed to be sync!
+		RS_DBG("unknown task or sync one!");
+		rsAutoProxyMonitor::taskError(ticket);
+		break;
+	}
+}
+
+void p3I2pSam3::processTaskSync(taskTicket *ticket)
+{
+//	RS_DBG4();
+
+	const bool data = !!ticket->data;
+
+	switch (ticket->task) {
+	case autoProxyTask::status:
+	{
+		samStatus *ss = static_cast<struct samStatus *>(ticket->data);
+		RS_STACK_MUTEX(mLock);
+		ss->state = mState;
+		if (mSetting.session)
+			ss->sessionName = mSetting.session->channel;
+		else
+			ss->sessionName = "none";
+	}
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+
+		break;
+	case autoProxyTask::getSettings:
+		// check if everything needed is set
+		if (!data) {
+			RS_DBG("autoProxyTask::getSettings data is missing");
+			rsAutoProxyMonitor::taskError(ticket);
+			break;
+		}
+
+		// get settings
+	    {
+		    RS_STACK_MUTEX(mLock);
+			*static_cast<struct samSettings *>(ticket->data) = mSetting;
+	    }
+
+		// finish task
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::setSettings:
+		// check if everything needed is set
+		if (!data) {
+			RS_DBG("autoProxyTask::setSettings data is missing");
+			rsAutoProxyMonitor::taskError(ticket);
+			break;
+		}
+
+		// set settings
+	    {
+		    RS_STACK_MUTEX(mLock);
+			mSetting = *static_cast<struct samSettings *>(ticket->data);
+			updateSettings_locked();
+	    }
+
+		// finish task
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::getErrorInfo:
+		*static_cast<std::string *>(ticket->data) = mSetting.session->error;
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::reloadConfig:
+	    {
+		    RS_STACK_MUTEX(mLock);
+			updateSettings_locked();
+	    }
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::stop:
+#if 0 // doesn't seem to work, socket stays "CLOSE_WAIT"
+		// there can be a case where libsam3 will block forever because for some reason it does not detect that the socket it has is dead
+		// as a workaroung kill it from here
+		if (mState == samStatus::samState::connectSession || mState == samStatus::samState::connectForward) {
+			// lock should be held by the main thread
+			if (!mTmpSession) {
+				// now it's getting weird
+				RS_WARN("session is nullptr but mState says it is connecting.");
+				// no break! just ignore for now ...
+			} else {
+				// just close it from here, libsam3 is not thread safe.
+				// a bit of a hack but should do the trick
+//				sam3CloseSession(mSetting.session);
+				sam3tcpDisconnect(mTmpSession->fd);
+				// no break! continue as usual to keep everything in line
+			}
+		}
+#endif
+		[[fallthrough]];
+	case autoProxyTask::start: [[fallthrough]];
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey: [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck: [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
+		// These are supposed to be async!
+		RS_WARN("unknown task or async one!");
+		rsAutoProxyMonitor::taskError(ticket);
+		break;
+	}
+}
+
+void p3I2pSam3::threadTick()
+{
+//	{
+//		RS_STACK_MUTEX(mLock);
+//		Dbg4() << __PRETTY_FUNCTION__ << " mPending: " << mPending.size() << std::endl;
+//	}
+
+	if(mPending.empty()) {
+		// sleep outisde of lock!
+		doSleep(std::chrono::milliseconds(250));
+		return;
+	}
+
+	// get task
+	taskTicket *tt = nullptr;
+	{
+		RS_STACK_MUTEX(mLock);
+		tt = mPending.front();
+		mPending.pop();
+	}
+
+	switch (tt->task) {
+	case autoProxyTask::stop:
+		mState = samStatus::samState::offline;
+		stopForwarding();
+		stopSession();
+		rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::offline);
+		break;
+
+	case autoProxyTask::start:
+	{
+		if (!mSetting.enable) {
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::disabled);
+			break;
+		}
+
+		// create main session
+		mState = samStatus::samState::connectSession;
+		bool ret = startSession();
+		if (!ret) {
+			mState = samStatus::samState::offline;
+
+			rsAutoProxyMonitor::taskError(tt);
+			break;
+		}
+
+		// start forwarding
+		mState = samStatus::samState::connectForward;
+		ret = startForwarding();
+
+		// finish ticket
+		if (ret) {
+			mState = samStatus::samState::online;
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::online);
+		} else {
+			mState = samStatus::samState::offline;
+			rsAutoProxyMonitor::taskError(tt);
+		}
+	}
+		break;
+
+	case autoProxyTask::receiveKey:
+	{
+		i2p::address *addr = static_cast<i2p::address *>(tt->data);
+		if (generateKey(addr->publicKey, addr->privateKey)) {
+			addr->base32 = i2p::keyToBase32Addr(addr->publicKey);
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::ok);
+		} else {
+			rsAutoProxyMonitor::taskError(tt);
+		}
+	}
+		break;
+
+	case autoProxyTask::lookupKey:
+		lookupKey(tt);
+		break;
+
+	case autoProxyTask::proxyStatusCheck:
+	{
+		// TODO better detection of status
+		bool ok;
+		ok = !!mSetting.session->fd;
+		ok &= !!mSetting.session->fwd_fd;
+		*static_cast<bool*>(tt->data) = ok;
+		rsAutoProxyMonitor::taskDone(tt, ok ? autoProxyStatus::ok : autoProxyStatus::error);
+	}
+		break;
+
+	case autoProxyTask::establishConnection:
+		establishConnection(tt);
+		break;
+	case autoProxyTask::closeConnection:
+		closeConnection(tt);
+		break;
+	case autoProxyTask::status: [[fallthrough]];
+	case autoProxyTask::getSettings: [[fallthrough]];
+	case autoProxyTask::setSettings: [[fallthrough]];
+	case autoProxyTask::getErrorInfo: [[fallthrough]];
+	case autoProxyTask::reloadConfig:
+		RS_ERR("unable to handle! This is a bug! task:", tt->task);
+		rsAutoProxyMonitor::taskError(tt);
+		break;
+	}
+	tt = nullptr;
+
+	// give i2p backend some time
+	doSleep(std::chrono::milliseconds(100));
+}
+
+RsSerialiser *p3I2pSam3::setupSerialiser()
+{
+	RsSerialiser* rsSerialiser = new RsSerialiser();
+	rsSerialiser->addSerialType(new RsGeneralConfigSerialiser());
+
+	return rsSerialiser;
+}
+
+#define addKVS(_key, _value) \
+	kv.key = _key;\
+	kv.value = _value;\
+	vitem->tlvkvs.pairs.push_back(kv);
+
+#define addKVSInt(_key, _value) \
+	kv.key = _key;\
+	rs_sprintf(kv.value, "%d", _value);\
+	vitem->tlvkvs.pairs.push_back(kv);
+
+bool p3I2pSam3::saveList(bool &cleanup, std::list<RsItem *> &lst)
+{
+	RS_DBG4();
+
+	cleanup = true;
+	RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet;
+	RsTlvKeyValue kv;
+
+	RS_STACK_MUTEX(mLock);
+	addKVS(kConfigKeySAM3Enable, mSetting.enable ? "TRUE" : "FALSE")
+	addKVS(kConfigKeyDestPriv,   mSetting.address.privateKey);
+
+	addKVSInt(kConfigKeyInLength,    mSetting.inLength)
+	addKVSInt(kConfigKeyInQuantity,  mSetting.inQuantity)
+	addKVSInt(kConfigKeyInVariance,  mSetting.inVariance)
+	addKVSInt(kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
+
+	addKVSInt(kConfigKeyOutLength,   mSetting.outLength)
+	addKVSInt(kConfigKeyOutQuantity, mSetting.outQuantity)
+	addKVSInt(kConfigKeyOutVariance, mSetting.outVariance)
+	addKVSInt(kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+	// these allow SAMv3 users to switch back to BOB
+	// remove after some time
+	addKVS(kConfigKeyBOBEnable, mSetting.enable ? "TRUE" : "FALSE")
+	addKVS(kConfigKeyBOBKey,    mSetting.address.privateKey)
+	addKVS(kConfigKeyBOBAddr,   mSetting.address.base32)
+#endif
+	lst.push_back(vitem);
+	return true;
+}
+
+#undef addKVS
+#undef addKVSUInt
+
+#define getKVSUInt(_kit, _key, _value) \
+	else if (_kit->key == _key) {\
+	    std::istringstream is(_kit->value);\
+	    int tmp;\
+	    is >> tmp;\
+	    _value = (int8_t)tmp;\
+	}
+
+bool p3I2pSam3::loadList(std::list<RsItem *> &load)
+{
+	RS_DBG4();
+
+	std::string priv;
+	priv.clear();
+
+	for(std::list<RsItem*>::const_iterator it = load.begin(); it!=load.end(); ++it) {
+		RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it);
+		if(vitem != NULL) {
+			RS_STACK_MUTEX(mLock);
+			for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) {
+				if      (kit->key == kConfigKeySAM3Enable)
+					mSetting.enable = kit->value == "TRUE";
+				else if (kit->key == kConfigKeyDestPriv)
+					priv = kit->value;
+				getKVSUInt(kit, kConfigKeyInLength,    mSetting.inLength)
+				getKVSUInt(kit, kConfigKeyInQuantity,  mSetting.inQuantity)
+				getKVSUInt(kit, kConfigKeyInVariance,  mSetting.inVariance)
+				getKVSUInt(kit, kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
+
+				getKVSUInt(kit, kConfigKeyOutLength,   mSetting.outLength)
+				getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity)
+				getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance)
+				getKVSUInt(kit, kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+				// import BOB settings
+				else if (kit->key == kConfigKeyBOBEnable)
+				    mSetting.enable = kit->value == "TRUE";
+				else if (kit->key == kConfigKeyBOBKey) {
+					// don't overwirte, just import when not set already!
+					if (priv.empty())
+						priv = kit->value;
+				}
+#endif
+				else
+					RS_INFO("unknown key:", kit->key);
+			}
+		}
+		delete vitem;
+	}
+
+	// get the pub key
+	std::string pub = i2p::publicKeyFromPrivate(priv);
+	if (pub.empty() || priv.empty())
+		RS_DBG("no destination to load");
+	else {
+		RS_STACK_MUTEX(mLock);
+
+		mSetting.address.publicKey = pub;
+		mSetting.address.privateKey = priv;
+		mSetting.address.base32 = i2p::keyToBase32Addr(pub);
+	}
+
+	RS_STACK_MUTEX(mLock);
+	mConfigLoaded = true;
+
+	return true;
+}
+
+#undef getKVSUInt
+
+bool p3I2pSam3::startSession()
+{
+	RS_DBG4();
+
+	constexpr size_t len = 8;
+	const std::string location = RsRandom::alphaNumeric(len);
+	const std::string nick = "RetroShare-" + location;
+
+	std::vector<std::string> params;
+	{
+		RS_STACK_MUTEX(mLock);
+
+		// length
+		params.push_back(i2p::makeOption("inbound.length", mSetting.inLength));
+		params.push_back(i2p::makeOption("outbound.length", mSetting.outLength));
+		// variance
+		params.push_back(i2p::makeOption("inbound.lengthVariance", + mSetting.inVariance));
+		params.push_back(i2p::makeOption("outbound.lengthVariance", + mSetting.outVariance));
+		// quantity
+		params.push_back(i2p::makeOption("inbound.quantity", + mSetting.inQuantity));
+		params.push_back(i2p::makeOption("outbound.quantity", + mSetting.outQuantity));
+		// backup quantity
+		params.push_back(i2p::makeOption("inbound.backupQuantity", + mSetting.inBackupQuantity));
+		params.push_back(i2p::makeOption("outbound.backupQuantity", + mSetting.outBackupQuantity));
+	}
+
+	std::string paramsStr;
+	for (auto &&p : params)
+		paramsStr.append(p + " ");
+	// keep trailing space for easier extending when necessary
+
+	int ret;
+
+	if (mSetting.session) {
+		stopSession();
+	}
+
+	auto session = (Sam3Session*)rs_malloc(sizeof (Sam3Session));
+
+	// add nick
+	paramsStr.append("inbound.nickname=" + nick); // leading space is already there
+
+	{
+		RS_STACK_MUTEX(mLockSam3Access);
+
+		if(!mSetting.address.privateKey.empty()) {
+			RS_DBG3("with destination");
+			ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, mSetting.address.privateKey.c_str(), Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
+		} else {
+			RS_DBG("without destination");
+			ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, SAM3_DESTINATION_TRANSIENT, Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
+		}
+	}
+
+	if (ret != 0) {
+		delete session;
+		session = nullptr;
+		return false;
+	}
+
+#if 0 // this check is useless. For non i2p hidden locations the public key is temporal anyway and for i2p hidden ones, it is part of the (fixed) private key.
+	if (!mSetting.address.publicKey.empty() && mSetting.address.publicKey != session->pubkey)
+		// This should be ok for non hidden locations. This should be a problem for hidden i2p locations...
+		RS_DBG("public key changed! Yet unsure if this is ok or a problem. Should be fine for non i2p hidden locations or clear net.");
+#endif
+	/*
+	 * Note: sam3CreateSession will issue a name looup of "ME" to receive its public key, thus it is always correct.
+	 *       No need to use i2p::publicKeyFromPrivate()
+	 */
+	RS_STACK_MUTEX(mLock);
+	mSetting.session = session;
+	mSetting.address.publicKey = session->pubkey;
+	mSetting.address.base32 = i2p::keyToBase32Addr(session->pubkey);
+	// do not overwrite the private key, if any!!
+
+	RS_DBG1("nick: ", nick, " address: ", mSetting.address.base32);
+	RS_DBG2("  myDestination.pub  ", mSetting.address.publicKey);
+	RS_DBG2("  myDestination.priv ", mSetting.address.privateKey);
+	return true;
+}
+
+bool p3I2pSam3::startForwarding()
+{
+	RS_DBG4();
+
+	if(mSetting.address.privateKey.empty()) {
+		RS_DBG3("no private key set");
+		// IMPORANT: return true here!
+		// since there is no forward session for non hidden nodes, this funtion is successfull by doing nothing
+		return true;
+	}
+
+	if (!mSetting.session) {
+		RS_WARN("no session found!");
+		return false;
+	}
+
+	peerState ps;
+	mPeerMgr->getOwnNetStatus(ps);
+
+	RS_STACK_MUTEX(mLockSam3Access);
+
+	mSetting.session->silent = true;
+	int ret = sam3StreamForward(mSetting.session, sockaddr_storage_iptostring(ps.localaddr).c_str(), sockaddr_storage_port(ps.localaddr));
+	if (ret < 0) {
+		RS_DBG("forward failed, due to", mSetting.session->error);
+		return false;
+	}
+
+	return true;
+}
+
+void p3I2pSam3::stopSession()
+{
+	RS_DBG4();
+
+	{
+		RS_STACK_MUTEX(mLock);
+		if (!mSetting.session)
+			return;
+
+		// swap connections
+		mInvalidConnections = mValidConnections;
+		mValidConnections.clear();
+
+		RS_STACK_MUTEX(mLockSam3Access);
+		sam3CloseSession(mSetting.session);
+		free(mSetting.session);
+
+		mSetting.session = nullptr;
+		mState = samStatus::samState::offline;
+	}
+
+	// At least i2pd doesn't like to instantaniously stop and (re)start a session, wait here just a little bit.
+	// Not ideal but does the trick.
+	// (This happens when using the "restart" button in the settings.)
+	doSleep(std::chrono::seconds(10));
+}
+
+void p3I2pSam3::stopForwarding()
+{
+	// nothing to do here, forwarding is stop when closing the seassion
+}
+
+bool p3I2pSam3::generateKey(std::string &pub, std::string &priv)
+{
+	RS_DBG4();
+
+	pub.clear();
+	priv.clear();
+
+	// The session is only usef for transporting the data
+	Sam3Session ss;
+
+	if (0 > sam3GenerateKeys(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, Sam3SigType::EdDSA_SHA512_Ed25519)) {
+		RS_DBG("got error: ", ss.error);
+		return false;
+	}
+
+	pub = std::string(ss.pubkey);
+	priv = std::string(ss.privkey);
+
+	// sanity check
+	auto p = i2p::publicKeyFromPrivate(priv);
+	if (p != pub) {
+		RS_WARN("public key does not match private key! fixing ...");
+		pub = p;
+	}
+
+	RS_DBG2("publuc key / address ", pub);
+	RS_DBG2("private key ", priv);
+
+	return true;
+}
+
+void p3I2pSam3::lookupKey(taskTicket *ticket)
+{
+	// this can be called independend of the main SAM session!
+
+	auto addr = static_cast<i2p::address*>(ticket->data);
+	if (addr->base32.empty()) {
+		RS_ERR("lookupKey: called with empty address");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	RsThread::async([ticket]()
+	{
+		auto addr = static_cast<i2p::address*>(ticket->data);
+
+		// The session is only usef for transporting the data
+		Sam3Session ss;
+		int ret = sam3NameLookup(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, addr->base32.c_str());
+		if (ret < 0) {
+			// get error
+			RS_DBG("key:       ", addr->base32);
+			RS_DBG("got error: ", ss.error);
+			rsAutoProxyMonitor::taskError(ticket);
+		} else {
+			addr->publicKey = ss.destkey;
+			rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+			RS_DBG1("success");
+		}
+	});
+}
+
+void p3I2pSam3::establishConnection(taskTicket *ticket)
+{
+	if (mState != samStatus::samState::online || !mSetting.session) {
+		RS_WARN("no session found!");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	samEstablishConnectionWrapper *wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
+	if (wrapper->address.publicKey.empty()) {
+		RS_ERR("no public key given");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	RsThread::async([ticket, this]() {
+		auto wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
+
+		struct Sam3Connection *connection;
+		{
+			auto l = this->mLockSam3Access;
+			RS_STACK_MUTEX(l);
+			mSetting.session->silent = false;
+			connection = sam3StreamConnect(this->mSetting.session, wrapper->address.publicKey.c_str());
+		}
+
+		if (!connection) {
+			// get error
+			RS_DBG("got error:", this->mSetting.session->error);
+			rsAutoProxyMonitor::taskError(ticket);
+		} else {
+			wrapper->connection = connection;
+			{
+				auto l = this->mLockSam3Access;
+				RS_STACK_MUTEX(l);
+				this->mValidConnections.push_back(connection);
+			}
+			RS_DBG1("success");
+			rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		}
+	});
+}
+
+void p3I2pSam3::closeConnection(taskTicket *ticket)
+{
+	Sam3Connection *conn = static_cast<Sam3Connection*>(ticket->data);
+
+	if (mState == samStatus::samState::offline || !mSetting.session) {
+		// no session found, sam was likely stopped
+		RS_DBG2("no session found");
+
+		auto it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
+		if (it != mInvalidConnections.end()) {
+			// this is the expected case
+			mInvalidConnections.erase(it);
+		} else {
+			// this is unexpected but not a big deal, just warn
+			RS_WARN("cannot find connection in mInvalidConnections");
+
+			it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
+			if (it != mValidConnections.end()) {
+				mValidConnections.erase(it);
+
+				// now it is getting even weirder, still not a big deal, just warn
+				RS_WARN("found connection in mValidConnections");
+			}
+		}
+
+		// when libsam3 has already handled closing of the connection - which should be the case here - the memory has been freed already (-> pointer is invalid)
+		conn = nullptr;
+	} else {
+		RS_STACK_MUTEX(mLock);
+
+		bool callClose = true;
+		// search in current connections
+		auto it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
+		if (it != mValidConnections.end()) {
+			RS_DBG2("found valid connection");
+			mValidConnections.erase(it);
+		} else {
+			// search in old connections
+			it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
+			if (it != mInvalidConnections.end()) {
+				// old connection, just ignore. *should* be freed already
+				mInvalidConnections.erase(it);
+
+				RS_DBG2("found old (invalid) connection");
+
+				callClose = false;
+				conn = nullptr;
+			} else {
+				// weird
+				RS_WARN("could'n find connection!");
+
+				// best thing we can do here
+				callClose = false;
+				conn = nullptr;
+			}
+		}
+
+		if (callClose) {
+			RS_DBG2("closing connection");
+
+			RS_STACK_MUTEX(mLockSam3Access);
+			sam3CloseConnection(conn);
+			conn = nullptr; // freed by above call
+		}
+	}
+
+	if (conn) {
+		free(conn);
+		conn = nullptr;
+	}
+
+	ticket->data = nullptr;
+	rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+	return;
+}
+
+void p3I2pSam3::updateSettings_locked()
+{
+	RS_DBG4();
+	IndicateConfigChanged();
+
+#if 0 // TODO recreat session when active, can we just recreat it?
+	if (mSs) {
+		stopSession();
+		startSession();
+	}
+#endif
+}

libretroshare/src/services/autoproxy/p3i2psam3.h

@@ -0,0 +1,110 @@
+#ifndef P3I2PSAM3_H
+#define P3I2PSAM3_H
+
+#include <queue>
+#include <list>
+
+#include "services/autoproxy/rsautoproxymonitor.h"
+#include "pqi/p3cfgmgr.h"
+#include "util/i2pcommon.h"
+#include "util/rsthreads.h"
+
+/*
+ * This class implements I2P SAMv3 (Simple Anonymous Messaging) to allow RS
+ * to automatically setup tunnel to and from I2P.
+ * SAMv3 is a simple text-based interface: https://geti2p.net/de/docs/api/samv3
+ *
+ * For the actual SAM commands / low level stuff libsam3 (https://github.com/i2p/libsam3)
+ * is used with some minor adjustments, for exmaple, the FORWARD session is always silent.
+ *
+ * SAM in a nutshell works like this:
+ * 1) setup main/control session which configures everything (destination ID, tunnel number, hops number, and so on)
+ * 2) setup a forward session, so that I2P will establish a connection to RS for each incoming connection to our i2p destination
+ * 3a) query/lookup the destination (public key) for a given i2p address
+ * 3b) connect to the given destination
+ *
+ * An established connection (both incoming or outgoing) are then handed over to RS.
+ * The lifetime of a session (and its subordinates connections) is bound to their tcp socket. When the socket closes, the session is closed, too.
+ *
+ */
+
+class p3PeerMgr;
+
+class Sam3Session;
+class Sam3Connection;
+
+typedef Sam3Session samSession;
+
+struct samSettings : i2p::settings {
+	samSession *session;
+};
+
+struct samEstablishConnectionWrapper {
+	i2p::address address;
+	Sam3Connection *connection;
+};
+
+struct samStatus {
+	std::string sessionName;
+	enum samState {
+		offline,
+		connectSession,
+		connectForward,
+		online
+	} state; // the name is kinda redundant ...
+};
+
+class p3I2pSam3 : public RsTickingThread, public p3Config, public autoProxyService
+{
+public:
+	p3I2pSam3(p3PeerMgr *peerMgr);
+
+	// autoProxyService interface
+public:
+	bool isEnabled();
+	bool initialSetup(std::string &addr, uint16_t &port);
+	void processTaskAsync(taskTicket *ticket);
+	void processTaskSync(taskTicket *ticket);
+
+	// RsTickingThread interface
+public:
+	void threadTick();	/// @see RsTickingThread
+
+	// p3Config interface
+protected:
+	RsSerialiser *setupSerialiser();
+	bool saveList(bool &cleanup, std::list<RsItem *> &);
+	bool loadList(std::list<RsItem *> &load);
+
+private:
+	bool startSession();
+	bool startForwarding();
+	void stopSession();
+	void stopForwarding();
+
+	bool generateKey(std::string &pub, std::string &priv);
+	void lookupKey(taskTicket *ticket);
+	void establishConnection(taskTicket *ticket);
+	void closeConnection(taskTicket *ticket);
+	void updateSettings_locked();
+
+	bool mConfigLoaded;
+
+	samSettings mSetting;
+	p3PeerMgr *mPeerMgr;
+	std::queue<taskTicket *> mPending;
+
+	// Used to report the state to the gui
+	// (Since the create session call/will can block and there is no easy way from outside the main thread to see
+	// what is going on, it is easier to store the current state in an extra variable independen from the main thread)
+	samStatus::samState mState;
+
+	// used to keep track of connections, libsam3 does it internally but it can be unreliable since pointers are shared
+	std::list<Sam3Connection *> mValidConnections, mInvalidConnections;
+
+	// mutex
+	RsMutex mLock;
+	RsMutex mLockSam3Access; // libsam3 is not thread safe! (except for key lookup)
+};
+
+#endif // P3I2PSAM3_H

libretroshare/src/services/autoproxy/rsautoproxymonitor.cc

@@ -329,14 +329,22 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType
 
 bool rsAutoProxyMonitor::isAsyncTask(autoProxyTask::autoProxyTask_enum t)
 {
+	// Explicit list all values, so that missing ones will be detected by the compiler.
 	switch (t) {
-	case autoProxyTask::start:
+	case autoProxyTask::start:      [[fallthrough]];
-	case autoProxyTask::stop:
+	case autoProxyTask::stop:       [[fallthrough]];
-	case autoProxyTask::receiveKey:
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey:  [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
 		return true;
-		break;
+	case autoProxyTask::status:           [[fallthrough]];
-	default:
+	case autoProxyTask::getSettings:      [[fallthrough]];
-		break;
+	case autoProxyTask::setSettings:      [[fallthrough]];
+	case autoProxyTask::getErrorInfo:     [[fallthrough]];
+	case autoProxyTask::reloadConfig:     [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck:
+		return false;
 	}
 	return false;
 }

libretroshare/src/services/autoproxy/rsautoproxymonitor.h

@@ -31,7 +31,8 @@ class autoProxyCallback;
 
 namespace autoProxyType {
     enum autoProxyType_enum {
-		I2PBOB
+//		I2PBOB,
+		I2PSAM3
 	};
 }
 
@@ -41,7 +42,10 @@ namespace autoProxyTask {
 		start,               ///< start up proxy
 		stop,                ///< shut down proxy
 		receiveKey,          ///< renew proxy key (if any)
+		lookupKey,           ///< look up a base32 addr
 		proxyStatusCheck,    ///< use to check if the proxy is still running
+		establishConnection, ///< create a connection to a given public key or base32 address
+		closeConnection,	 ///< closes a connection
 		/* sync tasks */
 		status,       ///< get status from auto proxy
 		getSettings,  ///< get setting from auto proxy

libretroshare/src/services/p3gxsforums.cc

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
     RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
                    RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
     RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
-    mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
+    mGenActive(false), mGenCount(0),
+    mKnownForumsMutex("GXS forums known forums timestamp cache")
+#ifdef RS_DEEP_FORUMS_INDEX
+    , mDeepIndex(DeepForumsIndex::dbDefaultPath())
+#endif
 {
 	// Test Data disabled in Repo.
 	//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@@ -190,160 +194,199 @@ RsSerialiser* p3GxsForums::setupSerialiser()
 	return rss;
 }
 
-void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
+void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
 {
-#ifdef GXSFORUMS_DEBUG
+	RS_DBG2(changes.size(), " changes to notify");
-	std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
+
+	for(RsGxsNotify* gxsChange: changes)
+	{
+		// Let the compiler delete the change for us
+		std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
+
+		switch(gxsChange->getType())
+		{
+		case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
+		case RsGxsNotify::TYPE_PUBLISHED:
+		{
+			auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
+
+			if(msgChange) /* Message received */
+			{
+				uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
+				switch(static_cast<RsGxsForumsItems>(msgSubtype))
+				{
+				case RsGxsForumsItems::MESSAGE_ITEM:
+				{
+					auto newForumMessageItem =
+					        dynamic_cast<RsGxsForumMsgItem*>(
+					            msgChange->mNewMsgItem );
+
+					if(!newForumMessageItem)
+					{
+						RS_ERR("Received message change with mNewMsgItem type "
+						       "mismatching or null");
+						print_stacktrace();
+						return;
+					}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+					RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
+					tmpPost.mMeta = newForumMessageItem->meta;
+					mDeepIndex.indexForumPost(tmpPost);
 #endif
-
-	std::vector<RsGxsNotify *>::iterator it;
-	for(it = changes.begin(); it != changes.end(); ++it)
-	{
-		RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
-
-		if (msgChange)
-		{
-			if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
-				if (rsEvents)
-				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
 					ev->mForumMsgId = msgChange->mMsgId;
 					ev->mForumGroupId = msgChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
 					rsEvents->postEvent(ev);
-				}
-
-#ifdef NOT_USED_YET
-			if (!msgChange->metaChange())
-			{
-#ifdef GXSCHANNELS_DEBUG
-				std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
-				std::cerr << std::endl;
-#endif
-
-				std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
-				for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
-				{
-#ifdef GXSCHANNELS_DEBUG
-					std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
-					std::cerr << std::endl;
-#endif
-					bool enabled = false;
-					if (autoDownloadEnabled(mit->first, enabled) && enabled)
-					{
-#ifdef GXSCHANNELS_DEBUG
-						std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
-						std::cerr << std::endl;
-#endif
-
-						/* problem is most of these will be comments and votes,
-						 * should make it occasional - every 5mins / 10minutes TODO */
-						unprocessedGroups.push_back(mit->first);
-					}
-				}
-			}
-#endif
-		}
-		else
-		{
-			if (rsEvents)
-			{
-				RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
-				if (grpChange)
-				{
-					switch (grpChange->getType())
-					{
-					case RsGxsNotify::TYPE_PROCESSED:	// happens when the group is subscribed
-					{
-							auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
-							ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
-							rsEvents->postEvent(ev);
-					}
 					break;
-
-                    case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
-                    {
-                            auto ev = std::make_shared<RsGxsForumEvent>();
-                            ev->mForumGroupId = grpChange->mGroupId;
-                            ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
-                            rsEvents->postEvent(ev);
 				}
+				default:
+					RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
 					break;
+				}
+			}
 
-                    case RsGxsNotify::TYPE_PUBLISHED:
+			auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-					case RsGxsNotify::TYPE_RECEIVED_NEW:
+			if(groupChange) /* Group received */
 			{
-						/* group received */
-
 				bool unknown;
 				{
 					RS_STACK_MUTEX(mKnownForumsMutex);
-                            unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
+					unknown = ( mKnownForums.find(gxsChange->mGroupId)
-                            mKnownForums[grpChange->mGroupId] = time(nullptr);
+					            == mKnownForums.end() );
+					mKnownForums[gxsChange->mGroupId] = time(nullptr);
 					IndicateConfigChanged();
 				}
 
 				if(unknown)
 				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
+					ev->mForumGroupId = gxsChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
 					rsEvents->postEvent(ev);
 				}
-						else
+
-							RsInfo() << __PRETTY_FUNCTION__
+#ifdef RS_DEEP_FORUMS_INDEX
-							         << " Not notifying already known forum "
+				uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
-							         << grpChange->mGroupId << std::endl;
+				switch(static_cast<RsGxsForumsItems>(itemType))
+				{
+				case RsGxsForumsItems::GROUP_ITEM:
+				{
+					auto newForumGroupItem =
+					        static_cast<RsGxsForumGroupItem*>(
+					            groupChange->mNewGroupItem );
+					mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
+					break;
+				}
+				default:
+					RS_WARN("Got unknown gxs group subtype: ", itemType);
+					break;
+				}
+#endif // def RS_DEEP_FORUMS_INDEX
+
 			}
 			break;
-
+		}
-                    case RsGxsNotify::TYPE_GROUP_DELETED:
+		case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-                        ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
+		{
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_MESSAGE_DELETED:
+		{
+			auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
+			if(!delChange)
+			{
+				RS_ERR( "Got mismatching notification type: ",
+				        gxsChange->getType() );
+				print_stacktrace();
+				break;
+			}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumPostFromIndex(
+			            delChange->mGroupId, delChange->messageId);
+#endif
+
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumEventCode = RsForumEventCode::DELETED_POST;
+			ev->mForumGroupId = delChange->mGroupId;
+			ev->mForumMsgId = delChange->messageId;
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_DELETED:
+		{
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
+#endif
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
 			rsEvents->postEvent(ev);
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_STATISTICS_CHANGED:
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-						ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
 			rsEvents->postEvent(ev);
 
 			RS_STACK_MUTEX(mKnownForumsMutex);
-                        mKnownForums[grpChange->mGroupId] = time(nullptr);
+			mKnownForums[gxsChange->mGroupId] = time(nullptr);
 			IndicateConfigChanged();
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_UPDATED:
 		{
-						// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
+			/* Happens when the group data has changed. In this case we need to
+			 * analyse the old and new group in order to detect possible
+			 * notifications for clients */
 
-						RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-						RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
 
-						if(old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
+			RsGxsForumGroupItem* old_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			RsGxsForumGroupItem* new_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
+
+			if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
 			{
-                            RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
+				RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
-							delete grpChange;
+				        "mNewGroup not of type RsGxsForumGroupItem or NULL. "
-							continue;
+				        "This is inconsistent!");
+				print_stacktrace();
+				break;
 			}
 
-						// First of all, we check if there is a difference between the old and new list of moderators
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
+#endif
+
+			/* First of all, we check if there is a difference between the old
+			 * and new list of moderators */
 
 			std::list<RsGxsId> added_mods, removed_mods;
-
 			for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
-							if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == old_forum_grp_item->mGroup.mAdminList.ids.end() )
 					added_mods.push_back(gxs_id);
 
 			for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
-							if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == new_forum_grp_item->mGroup.mAdminList.ids.end() )
 					removed_mods.push_back(gxs_id);
 
 			if(!added_mods.empty() || !removed_mods.empty())
@@ -359,24 +402,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			}
 
 			// check the list of pinned posts
-
 			std::list<RsGxsMessageId> added_pins, removed_pins;
 
 			for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					added_pins.push_back(msg_id);
 
 			for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					removed_pins.push_back(msg_id);
 
 			if(!added_pins.empty() || !removed_pins.empty())
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
-
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
-
 				rsEvents->postEvent(ev);
 			}
 
@@ -384,30 +426,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			        || old_forum_grp_item->meta.mGroupName  != new_forum_grp_item->meta.mGroupName
 			        || old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
 			        || old_forum_grp_item->meta.mAuthorId   != new_forum_grp_item->meta.mAuthorId
-                          || old_forum_grp_item->meta.mCircleId      != new_forum_grp_item->meta.mCircleId
+			        || old_forum_grp_item->meta.mCircleId   != new_forum_grp_item->meta.mCircleId )
-                             )
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
 				rsEvents->postEvent(ev);
 			}
-					}
-                        break;
 
+			break;
+		}
 
 		default:
-                        RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
+			RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
+			        " Currently not handled." );
 			break;
 		}
 	}
-			}
-		}
-
-		/* shouldn't need to worry about groups - as they need to be subscribed to */
-
-        delete *it;
-	}
 }
 
 void	p3GxsForums::service_tick()
@@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
 	forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
 	        | GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
 
+	/* This flag have always this value even for circle restricted forums due to
+	 * how GXS distribute/verify groups */
 	forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
 
 	forum.mMeta.mCircleId.clear();
@@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
 	        id == mMeta.mAuthorId;
 }
 
+std::error_condition p3GxsForums::getContentSummaries(
+        const RsGxsGroupId& forumId,
+        const std::set<RsGxsMessageId>& contentIds,
+        std::vector<RsMsgMetaData>& summaries )
+{
+	uint32_t token;
+	RsTokReqOptions opts;
+	opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
+
+	GxsMsgReq msgReq;
+	msgReq[forumId] = contentIds;
+
+
+	if(!requestMsgInfo(token, opts, msgReq))
+	{
+		RS_ERR("requestMsgInfo failed");
+		return std::errc::invalid_argument;
+	}
+
+	switch(waitToken(token, std::chrono::seconds(5)))
+	{
+	case RsTokenService::COMPLETE:
+	{
+		GxsMsgMetaMap metaMap;
+		if(!RsGenExchange::getMsgMeta(token, metaMap))
+			return std::errc::result_out_of_range;
+		summaries = metaMap[forumId];
+		return std::error_condition();
+	}
+	case RsTokenService::PARTIAL: // [[fallthrough]];
+	case RsTokenService::PENDING:
+		return std::errc::timed_out;
+	default:
+		return std::errc::not_supported;
+	}
+}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+std::error_condition p3GxsForums::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG1("");
+
+	RsGxsForumsSearchRequest request;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(request.mType) );
+		return std::errc::bad_message;
+	}
+
+	RsGxsForumsSearchReply reply;
+	auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
+	if(mErr || reply.mResults.empty()) return mErr;
+
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(reply);
+		resultSize = ctx.mOffset;
+	}
+
+	resultData = rs_malloc<uint8_t>(resultSize);
+	RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+	RsGenericSerializer::SerializeJob j =
+	        RsGenericSerializer::SerializeJob::SERIALIZE;
+	RS_SERIAL_PROCESS(reply);
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string& matchString, TurtleRequestId& searchId )
+{
+	RsGxsForumsSearchRequest request;
+	request.mQuery = matchString;
+
+	uint32_t requestSize;
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(request);
+		requestSize = ctx.mOffset;
+	}
+
+	std::error_condition ec;
+	auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
+	if(!requestData) return ec;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	return netService()->distantSearchRequest(
+	            requestData, requestSize,
+	            static_cast<RsServiceType>(serviceType()), searchId );
+}
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string& matchString,
+        std::vector<RsGxsSearchResult>& searchResults )
+{ return prepareSearchResults(matchString, false, searchResults); }
+
+std::error_condition p3GxsForums::prepareSearchResults(
+        const std::string& matchString, bool publicOnly,
+        std::vector<RsGxsSearchResult>& searchResults )
+{
+	std::vector<DeepForumsSearchResult> results;
+	auto mErr = mDeepIndex.search(matchString, results);
+	if(mErr) return mErr;
+
+	searchResults.clear();
+	for(auto uRes: results)
+	{
+		RsUrl resUrl(uRes.mUrl);
+		const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
+		if(!forumIdStr)
+		{
+			RS_ERR( "Forum URL retrieved from deep index miss ID. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::address_not_available;
+		}
+
+		std::vector<RsGxsForumGroup> forumsInfo;
+		RsGxsGroupId forumId(*forumIdStr);
+		if(forumId.isNull())
+		{
+			RS_ERR( "Forum ID retrieved from deep index is invalid. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::bad_address;
+		}
+
+		if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
+		        forumsInfo.empty() )
+		{
+			RS_ERR( "Forum just parsed from deep index link not found. "
+			        "Should never happen! ", forumId, " ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::identifier_removed;
+		}
+
+		RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
+
+		// Avoid leaking sensitive information to unkown peers
+		if( publicOnly &&
+		        ( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
+		          RsGxsCircleType::PUBLIC ) ) continue;
+
+		RsGxsSearchResult res;
+		res.mGroupId = forumId;
+		res.mGroupName = fMeta.mGroupName;
+		res.mAuthorId = fMeta.mAuthorId;
+		res.mPublishTs = fMeta.mPublishTs;
+		res.mSearchContext = uRes.mSnippet;
+
+		auto postIdStr =
+		        resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
+		if(postIdStr)
+		{
+			RsGxsMessageId msgId(*postIdStr);
+			if(msgId.isNull())
+			{
+				RS_ERR( "Post just parsed from deep index link is invalid. "
+				        "Should never happen! ", postIdStr, " ", uRes.mUrl );
+				print_stacktrace();
+				return std::errc::bad_address;
+			}
+
+			std::vector<RsMsgMetaData> msgSummaries;
+			auto errc = getContentSummaries(
+			            forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
+			if(errc) return errc;
+
+			if(msgSummaries.size() != 1)
+			{
+				RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
+				        "should never happen!" );
+				return std::errc::result_out_of_range;
+			}
+
+			RsMsgMetaData& msgMeta(msgSummaries[0]);
+			res.mMsgId = msgMeta.mMsgId;
+			res.mMsgName = msgMeta.mMsgName;
+			res.mAuthorId = msgMeta.mAuthorId;
+		}
+
+		RS_DBG4(res);
+		searchResults.push_back(res);
+	}
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RsGxsForumsSearchReply reply;
+	{
+		RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(reply);
+	}
+	free(resultData);
+
+	if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(reply.mType) );
+		return std::errc::bad_message;
+	}
+
+	auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
+	event->mSearchId = requestId;
+	event->mSearchResults = reply.mResults;
+	rsEvents->postEvent(event);
+	return std::error_condition();
+}
+
+#else  // def RS_DEEP_FORUMS_INDEX
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string&, TurtleRequestId& )
+{ return std::errc::function_not_supported; }
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string&,
+        std::vector<RsGxsSearchResult>& )
+{ return std::errc::function_not_supported; }
+
+#endif // def RS_DEEP_FORUMS_INDEX
+
 /*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
         "retroshare:///forums";
 /*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

libretroshare/src/services/p3gxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -32,6 +32,10 @@
 #include "util/rstickevent.h"
 #include "util/rsdebug.h"
 
+#ifdef RS_DEEP_FORUMS_INDEX
+#include "deep_search/forumsindex.hpp"
+#endif
+
 
 class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
 	public RsTickEvent	/* only needed for testing - remove after */
@@ -144,6 +148,33 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) override;
 
+	/// @see RsGxsForums
+	std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) override;
+
+	/// @see RsGxsForums
+	std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) override;
+
+	/// @see RsGxsForums
+	std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) override;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	/// @see RsNxsObserver
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNxsObserver
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+#endif
+
     /// implementation of rsGxsGorums
     ///
 	bool getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups) override;
@@ -155,6 +186,17 @@ public:
 
 	bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
 
+protected:
+#ifdef RS_DEEP_FORUMS_INDEX
+	/** Internal usage
+	 * @param[in] publicOnly if true is passed only results pertaining to
+	 * publicly shared forums are returned
+	 */
+	std::error_condition prepareSearchResults(
+	        const std::string& matchString, bool publicOnly,
+	        std::vector<RsGxsSearchResult>& searchResults );
+#endif //def RS_DEEP_FORUMS_INDEX
+
 private:
 
 static uint32_t forumsAuthenPolicy();
@@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
     std::map<RsGxsGroupId,rstime_t> mKnownForums ;
 	
 	RsMutex mKnownForumsMutex;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	DeepForumsIndex mDeepIndex;
+#endif
 };

libretroshare/src/turtle/p3turtle.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2009-2018 by Cyril Soler <csoler@users.sourceforge.net>           *
+ * Copyright (C) 2009-2018  Cyril Soler <csoler@users.sourceforge.net>         *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -26,6 +28,10 @@
 #include <stdexcept>
 #include <stdlib.h>
 #include <assert.h>
+#include <iostream>
+#include <cerrno>
+#include <cmath>
+#include <cstdio>
 
 #include "rsserver/p3face.h"
 #include "crypto/rscrypto.h"
@@ -39,13 +45,7 @@
 #include "ft/ftcontroller.h"
 
 #include "p3turtle.h"
-
+#include "util/cxx17retrocompat.h"
-#include <iostream>
-#include <errno.h>
-#include <cmath>
-
-#include <stdio.h>
-
 #include "util/rsdebug.h"
 #include "util/rsprint.h"
 #include "util/rsrandom.h"
@@ -1975,7 +1975,8 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
 //
 
 
-void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
+void RsTurtleStringSearchRequestItem::search(
+        std::list<TurtleFileInfo>& result ) const
 {
 	/* call to core */
 	std::list<DirDetails> initialResults;
@@ -1988,17 +1989,19 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
 	std::cerr << "Performing rsFiles->search()" << std::endl ;
 #endif
 	// now, search!
-    rsFiles->SearchKeywords(words, initialResults,RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE,PeerId());
+	rsFiles->SearchKeywords(
+	            words, initialResults,
+	            RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE, PeerId() );
 
 #ifdef P3TURTLE_DEBUG
 	std::cerr << initialResults.size() << " matches found." << std::endl ;
 #endif
 	result.clear() ;
 
-	for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
+	for(auto& it: std::as_const(initialResults))
 	{
 		// retain only file type
-		if (it->type == DIR_TYPE_DIR)
+		if (it.type == DIR_TYPE_DIR)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Skipping directory " << it->name << std::endl ;
@@ -2006,12 +2009,12 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
 			continue;
 		}
 
-		TurtleFileInfo i ;
+		TurtleFileInfo i;
-		i.hash = it->hash ;
+		i.hash = it.hash;
-        i.size = it->size ;
+		i.size = it.size;
-		i.name = it->name ;
+		i.name = it.name;
 
-		result.push_back(i) ;
+		result.push_back(i);
 	}
 }
 void RsTurtleRegExpSearchRequestItem::search(std::list<TurtleFileInfo>& result) const

libretroshare/src/turtle/turtleclientservice.h

@@ -19,23 +19,25 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
-// This class is the parent class for any service that will use the turtle router to distribute its packets.
-// Typical representative clients include:
-//
-// 	p3ChatService:		opens tunnels to distant peers for chatting
-// 	ftServer:			searches and open tunnels to distant sources for file transfer
-//
 #pragma once
 
 #include <string>
 #include <stdlib.h>
-#include <serialiser/rsserial.h>
+
-#include <turtle/rsturtleitem.h>
+#include "serialiser/rsserial.h"
+#include "turtle/rsturtleitem.h"
+#include "util/rsdebug.h"
 
 struct RsItem;
 class p3turtle ;
 
+/** This class is the parent class for any service that will use the turtle
+ * router to distribute its packets.
+ * Typical representative clients include:
+ * 	p3ChatService: opens tunnels to distant peers for chatting
+ * 	ftServer:      searches and open tunnels to distant sources for file
+ * 	               transfer
+ */
 class RsTurtleClientService
 {
 	public:
@@ -88,27 +90,32 @@ class RsTurtleClientService
 		}
 
 	/*!
-         * \brief receiveSearchRequest
+	* This method is called by the turtle router to notify the client of a
-         * 			This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
+	* search request in the form generic data.
-         * 			result contains the serialised generic result returned by the client.
+	* The returned result contains the serialised generic result returned by the
-         *
+	* client service.
 	* The turtle router keeps the memory ownership over search_request_data
-         *
 	* \param search_request_data generic serialized search data
 	* \param search_request_data_len  length of the serialized search data
 	* \param search_result_data generic serialized search result data
 	* \param search_result_data_len length of the serialized search result data
-         * \param max_allowed_hits         max number of hits allowed to be sent back and forwarded
+	* \param max_allowed_hits max number of hits allowed to be sent back and
-         *
+	* forwarded
-         * \return true if the search is successful.
+	* \return true if matching results are available, false otherwise.
 	*/
-    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
+	virtual bool receiveSearchRequest(
-                                          uint32_t /*search_request_data_len*/,
+	        unsigned char *search_request_data, uint32_t search_request_data_len,
-                                          unsigned char *& /*search_result_data*/,
+	        unsigned char *& search_result_data, uint32_t& search_result_data_len,
-                                          uint32_t& /*search_result_data_len*/,
+	        uint32_t& max_allows_hits )
-                                          uint32_t& /* max_allows_hits */)
 	{
-			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+		/* Suppress unused warning this way and not commenting the param names
+		 * so doxygen match documentation against params */
+		(void) search_request_data; (void) search_request_data_len;
+		(void) search_result_data; (void) search_result_data_len;
+		(void) max_allows_hits;
+
+		RS_WARN( "Received search request from turtle router, but the client "
+		         "is not handling it!" );
 		return false;
 	}
 

libretroshare/src/use_libretroshare.pri

@@ -90,6 +90,15 @@ rs_broadcast_discovery {
     win32-g++|win32-clang-g++:dLibs *= wsock32
 }
 
+rs_sam3_libsam3 {
+    LIBSAM3_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/libsam3/)
+    LIBSAM3_BUILD_PATH=$$clean_path($${RS_BUILD_PATH}/supportlibs/libsam3/)
+    INCLUDEPATH *= $$clean_path($${LIBSAM3_SRC_PATH}/src/libsam3/)
+    DEPENDPATH *= $$clean_path($${LIBSAM3_BUILD_PATH})
+    QMAKE_LIBDIR *= $$clean_path($${LIBSAM3_BUILD_PATH})
+    LIBS *= -L$$clean_path($${LIBSAM3_BUILD_PATH}) -lsam3
+}
+
 static {
     sLibs *= $$mLibs
 } else {

libretroshare/src/util/dnsresolver.cc

@@ -41,8 +41,6 @@
 const rstime_t MAX_TIME_BEFORE_RETRY 	=	300 ; /* seconds before retrying an ip address */
 const rstime_t MAX_KEEP_DNS_ENTRY 		= 3600 ; /* seconds during which a DNS entry is considered valid */
 
-static const std::string ADDR_AGENT  = "Mozilla/5.0";
-
 void *solveDNSEntries(void *p)
 {
 	bool more_to_go = true ;

libretroshare/src/util/extaddrfinder.cc

@@ -19,9 +19,13 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
+
+//#define EXTADDRSEARCH_DEBUG
+
 #include "extaddrfinder.h"
 
 #include "pqi/pqinetwork.h"
+#include "rsdebug.h"
 #include "util/rsstring.h"
 #include "util/rsmemory.h"
 
@@ -38,286 +42,237 @@
 #include <stdio.h>
 #include "util/rstime.h"
 
+#include <map>
+
 const uint32_t MAX_IP_STORE = 300; /* seconds ip address timeout */
 
-//#define EXTADDRSEARCH_DEBUG
+class ZeroInt
-
-static const std::string ADDR_AGENT  = "Mozilla/5.0";
-
-static std::string scan_ip(const std::string& text)
 {
-	std::set<unsigned char> digits ;
+public:
-	digits.insert('0') ; digits.insert('3') ; digits.insert('6') ;
+	ZeroInt() : n(0) {}
-	digits.insert('1') ; digits.insert('4') ; digits.insert('7') ;
+	uint32_t n ;
-	digits.insert('2') ; digits.insert('5') ; digits.insert('8') ;
+};
-	digits.insert('9') ; 
 
-	for(int i=0;i<(int)text.size();++i)
+void ExtAddrFinder::run()
-	{
-		while(i < (int)text.size() && digits.find(text[i])==digits.end()) ++i ;
-
-		if(i>=(int)text.size())
-			return "" ;
-
-		unsigned int a,b,c,d ;
-
-		if(sscanf(text.c_str()+i,"%u.%u.%u.%u",&a,&b,&c,&d) != 4)
-			continue ;
-
-		if(a < 256 && b<256 && c<256 && d<256)
-		{
-			std::string s ;
-			rs_sprintf(s, "%u.%u.%u.%u", a, b, c, d) ;
-			return s;
-		}
-	}
-	return "" ;
-}
-
-static void getPage(const std::string& server_name,std::string& page)
-{
-	page = "" ;
-	int sockfd,n=0;                   // socket descriptor
-	struct sockaddr_in serveur;       // server's parameters
-	memset(&serveur.sin_zero, 0, sizeof(serveur.sin_zero));
-
-	char buf[1024];
-	char request[1024];
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cout << "ExtAddrFinder: connecting to " << server_name << std::endl ;
-#endif
-	// socket creation
-
-	sockfd = unix_socket(PF_INET,SOCK_STREAM,0);
-	if (sockfd < 0)
-	{
-		std::cerr << "ExtAddrFinder: Failed to create socket" << std::endl;
-		return ;
-	}
-
-	serveur.sin_family = AF_INET;
-
-	// get server's ipv4 adress
-
-    	in_addr in ;
-        
-	if(!rsGetHostByName(server_name.c_str(),in))  /* l'hôte n'existe pas */
-	{
-		std::cerr << "ExtAddrFinder: Unknown host " << server_name << std::endl;
-		unix_close(sockfd);
-		return ;
-	}
-	serveur.sin_addr = in ;
-	serveur.sin_port = htons(80);
-
-#ifdef EXTADDRSEARCH_DEBUG
-	printf("Connection attempt\n");
-#endif
-    	std::cerr << "ExtAddrFinder: resolved hostname " << server_name << " to " << rs_inet_ntoa(in) << std::endl;
-
-	sockaddr_storage server;
-	sockaddr_storage_setipv4(server, &serveur);
-	sockaddr_storage_setport(server, 80);
-	if(unix_connect(sockfd, server) == -1)
-	{
-		std::cerr << "ExtAddrFinder: Connection error to " << server_name << std::endl ;
-		unix_close(sockfd);
-		return ;
-	}
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Connection established to " << server_name << std::endl ;
-#endif
-
-	// envoi 
-	if(snprintf( request, 
-			1024,
-			"GET / HTTP/1.0\r\n"
-			"Host: %s:%d\r\n"
-			"Connection: Close\r\n"
-			"\r\n", 
-			server_name.c_str(), 80) > 1020)
-	{
-		std::cerr << "ExtAddrFinder: buffer overrun. The server name \"" << server_name << "\" is too long. This is quite unexpected." << std::endl;
-		unix_close(sockfd);
-		return ;
-	}
-
-	if(send(sockfd,request,strlen(request),0)== -1)
-	{
-		std::cerr << "ExtAddrFinder: Could not send request to " << server_name << std::endl ;
-		unix_close(sockfd);
-		return ;
-	}
-	// recéption 
-
-	while((n = recv(sockfd, buf, sizeof buf - 1, 0)) > 0)
-	{
-		buf[n] = '\0';
-		page += std::string(buf,n) ;
-	}
-	// fermeture de la socket
-
-	unix_close(sockfd);
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Got full page from " << server_name << std::endl ;
-#endif
-}
-
-
-void* doExtAddrSearch(void *p)
 {
 	
 	std::vector<std::string> res ;
 
-	ExtAddrFinder *af = (ExtAddrFinder*)p ;
+	for(auto& it : _ip_servers)
-
-	for(std::list<std::string>::const_iterator it(af->_ip_servers.begin());it!=af->_ip_servers.end();++it)
 	{
-		std::string page ;
+		std::string ip = "";
-
+		rsGetHostByNameSpecDNS(it,"myip.opendns.com",ip,2);
-		getPage(*it,page) ;
-		std::string ip = scan_ip(page) ;
-
 		if(ip != "")
 			res.push_back(ip) ;
 #ifdef EXTADDRSEARCH_DEBUG
-		std::cout << "ip found through " << *it << ": \"" << ip << "\"" << std::endl ;
+		RS_DBG("ip found through DNS ", it, ": \"", ip, "\"");
 #endif
 	}
 
 	if(res.empty())
 	{
-		// thread safe copy results.
+		reset();
-		//
+		return ;
+	}
+
+	std::map<sockaddr_storage,ZeroInt> addrV4_votes;
+	std::map<sockaddr_storage,ZeroInt> addrV6_votes;
+	std::string all_addrV4_Found;
+	std::string all_addrV6_Found;
+
+	for(auto curRes : res)
 	{
-			RsStackMutex mtx(af->mAddrMtx) ;
+		sockaddr_storage addr;
-
+		sockaddr_storage_clear(addr);
-			af->mFound = false ;
+		bool validIP =   sockaddr_storage_inet_pton(addr, curRes)
-			af->mFoundTS = time(NULL) ;
+		              && sockaddr_storage_isValidNet(addr);
-			af->mSearching = false ;
+		bool isIPv4 = sockaddr_storage_ipv6_to_ipv4(addr);
-		}
+		if( validIP && isIPv4 )
-		return NULL ;
-	}
-
-	sort(res.begin(),res.end()) ; // eliminates outliers.
-
-
-
-	if(!sockaddr_storage_ipv4_aton(af->mAddr, res[res.size()/2].c_str()))
 		{
-		std::cerr << "ExtAddrFinder: Could not convert " << res[res.size()/2] << " into an address." << std::endl ;
+			addr.ss_family = AF_INET;
+			addrV4_votes[addr].n++ ;
+			all_addrV4_Found += sockaddr_storage_tostring(addr) + "\n";
+		}
+		else if( validIP && !isIPv4)
 		{
-			RsStackMutex mtx(af->mAddrMtx) ;
+			addr.ss_family = AF_INET6;
-			af->mFound = false ;
+			addrV6_votes[addr].n++ ;
-			af->mFoundTS = time(NULL) ;
+			all_addrV6_Found += sockaddr_storage_tostring(addr) + "\n";
-			af->mSearching = false ;
 		}
-		return NULL ;
+		else
+			RS_ERR("Invalid addresse reported: ", curRes) ;
+
 	}
 
+	if( (0 == addrV4_votes.size()) && (0 == addrV6_votes.size()) )
+	{
+		RS_ERR("Could not find any external address.");
+		reset();
+		return ;
+	}
+
+	if( 1 < addrV4_votes.size() )
+		RS_ERR("Multiple external IPv4 addresses reported: "
+		      , all_addrV4_Found ) ;
+
+	if( 1 < addrV6_votes.size() )
+		RS_ERR("Multiple external IPv6 addresses reported: "
+		      , all_addrV6_Found ) ;
+
 	{
-		RsStackMutex mtx(af->mAddrMtx) ;
+		RS_STACK_MUTEX(mAddrMtx);
-		af->mFound = true ;
+
-		af->mFoundTS = time(NULL) ;
+		mSearching = false ;
-		af->mSearching = false ;
+		mFoundTS = time(NULL) ;
+
+		// Only save more reported address if not only once.
+		uint32_t admax = 0 ;
+		sockaddr_storage_clear(mAddrV4);
+		for (auto it : addrV4_votes)
+			if (admax < it.second.n)
+			{
+				mAddrV4 = it.first ;
+				mFoundV4 = true ;
+				admax = it.second.n ;
 			}
 
-	return NULL ;
+		admax = 0 ;
+		sockaddr_storage_clear(mAddrV6);
+		for (auto it : addrV6_votes)
+			if (admax < it.second.n)
+			{
+				mAddrV6 = it.first ;
+				mFoundV6 = true ;
+				admax = it.second.n ;
+			}
+
+	}
+
+	return ;
 }
 
-
 void ExtAddrFinder::start_request()
 {
-	void *data = (void *)this;
+	if (!isRunning())
-	pthread_t tid ;
+		start("ExtAddrFinder");
-    
-    	if(! pthread_create(&tid, 0, &doExtAddrSearch, data))
-		pthread_detach(tid); /* so memory is reclaimed in linux */
-        else
-            	std::cerr << "(EE) Could not start ExtAddrFinder thread." << std::endl;
 }
 
-bool ExtAddrFinder::hasValidIP(struct sockaddr_storage &addr)
+bool ExtAddrFinder::hasValidIPV4(struct sockaddr_storage &addr)
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Getting ip." << std::endl ;
+	RS_DBG("Getting ip.");
 #endif
 
 	{
-		RsStackMutex mut(mAddrMtx) ;
+		RS_STACK_MUTEX(mAddrMtx) ;
-		if(mFound)
+		if(mFoundV4)
 		{
 #ifdef EXTADDRSEARCH_DEBUG
-			std::cerr << "ExtAddrFinder: Has stored ip: responding with this ip." << std::endl ;
+			RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV4)) ;
 #endif
-            sockaddr_storage_copyip(addr,mAddr);	// just copy the IP so we dont erase the port.
+			sockaddr_storage_copyip(addr,mAddrV4);	// just copy the IP so we dont erase the port.
 		}
 	}
-	rstime_t delta;
+
+	testTimeOut();
+
+	RS_STACK_MUTEX(mAddrMtx) ;
+	return mFoundV4;
+}
+
+bool ExtAddrFinder::hasValidIPV6(struct sockaddr_storage &addr)
+{
+#ifdef EXTADDRSEARCH_DEBUG
+	RS_DBG("Getting ip.");
+#endif
+
 	{
-		RsStackMutex mut(mAddrMtx) ;
+		RS_STACK_MUTEX(mAddrMtx) ;
-		//timeout the current ip
+		if(mFoundV6)
-		delta = time(NULL) - mFoundTS;
+		{
+#ifdef EXTADDRSEARCH_DEBUG
+			RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV6)) ;
+#endif
+			sockaddr_storage_copyip(addr,mAddrV6);	// just copy the IP so we dont erase the port.
 		}
-	if((uint32_t)delta > MAX_IP_STORE) {//launch a research
+	}
+
+	testTimeOut();
+
+	RS_STACK_MUTEX(mAddrMtx) ;
+	return mFoundV6;
+}
+
+void ExtAddrFinder::testTimeOut()
+{
+	bool timeOut;
+	{
+		RS_STACK_MUTEX(mAddrMtx) ;
+		//timeout the current ip
+		timeOut = (mFoundTS + MAX_IP_STORE < time(NULL));
+	}
+	if(timeOut || mFirstTime) {//launch a research
 		if( mAddrMtx.trylock())
 		{
 			if(!mSearching)
 			{
 #ifdef EXTADDRSEARCH_DEBUG
-				std::cerr << "ExtAddrFinder: No stored ip: Initiating new search." << std::endl ;
+				RS_DBG("No stored ip: Initiating new search.");
 #endif
 				mSearching = true ;
 				start_request() ;
 			}
 #ifdef EXTADDRSEARCH_DEBUG
 			else
-				std::cerr << "ExtAddrFinder: Already searching." << std::endl ;
+				RS_DBG("Already searching.");
 #endif
+			mFirstTime = false;
 			mAddrMtx.unlock();
 		}
 #ifdef EXTADDRSEARCH_DEBUG
 		else
-			std::cerr << "ExtAddrFinder: (Note) Could not acquire lock. Busy." << std::endl ;
+			RS_DBG("(Note) Could not acquire lock. Busy.");
 #endif
 	}
-
-	RsStackMutex mut(mAddrMtx) ;
-	return mFound ;
 }
 
-void ExtAddrFinder::reset()
+void ExtAddrFinder::reset(bool firstTime /*=false*/)
 {
-	RsStackMutex mut(mAddrMtx) ;
+#ifdef EXTADDRSEARCH_DEBUG
+	RS_DBG("firstTime=", firstTime);
+#endif
+	RS_STACK_MUTEX(mAddrMtx) ;
 
-	mFound = false ;
 	mSearching = false ;
-	mFoundTS = time(NULL) - MAX_IP_STORE;
+	mFoundV4 = false ;
+	mFoundV6 = false ;
+	mFirstTime = firstTime;
+	mFoundTS = time(nullptr);
+	sockaddr_storage_clear(mAddrV4);
+	sockaddr_storage_clear(mAddrV6);
 }
 
 ExtAddrFinder::~ExtAddrFinder()
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Deleting ExtAddrFinder." << std::endl ;
+	RS_DBG("Deleting ExtAddrFinder.");
 #endif
-
 }
 
 ExtAddrFinder::ExtAddrFinder() : mAddrMtx("ExtAddrFinder")
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Creating new ExtAddrFinder." << std::endl ;
+	RS_DBG("Creating new ExtAddrFinder.");
 #endif
-	RsStackMutex mut(mAddrMtx) ;
+	reset( true );
 
-	mFound = false;
+//https://unix.stackexchange.com/questions/22615/how-can-i-get-my-external-ip-address-in-a-shell-script
-	mSearching = false;
+	//Enter direct ip so local DNS cannot change it.
-	mFoundTS = time(NULL) - MAX_IP_STORE;
+	//DNS servers must recognize "myip.opendns.com"
-	sockaddr_storage_clear(mAddr);
+	_ip_servers.push_back(std::string( "208.67.222.222" )) ;//resolver1.opendns.com
-
+	_ip_servers.push_back(std::string( "208.67.220.220" )) ;//resolver2.opendns.com
-	_ip_servers.push_back(std::string( "checkip.dyndns.org" )) ;
+	_ip_servers.push_back(std::string( "208.67.222.220" )) ;//resolver3.opendns.com
-	_ip_servers.push_back(std::string( "www.myip.dk"   )) ;
+	_ip_servers.push_back(std::string( "208.67.220.222" )) ;//resolver4.opendns.com
-	_ip_servers.push_back(std::string( "showip.net"         )) ;
+	_ip_servers.push_back(std::string( "2620:119:35::35" )) ;//resolver1.opendns.com
-	_ip_servers.push_back(std::string( "www.displaymyip.com")) ;
+	_ip_servers.push_back(std::string( "2620:119:53::53" )) ;//resolver2.opendns.com
 }
-

libretroshare/src/util/extaddrfinder.h

@@ -30,26 +30,31 @@
 
 struct sockaddr ;
 
-class ExtAddrFinder
+class ExtAddrFinder: public RsThread
 {
 	public:
 		ExtAddrFinder() ;
 		~ExtAddrFinder() ;
 
-		bool hasValidIP(struct sockaddr_storage &addr) ;
+		bool hasValidIPV4(struct sockaddr_storage &addr) ;
+		bool hasValidIPV6(struct sockaddr_storage &addr) ;
 		void getIPServersList(std::list<std::string>& ip_servers) { ip_servers = _ip_servers ; }
 
 		void start_request() ;
 
-		void reset() ;
+		void reset(bool firstTime = false) ;
 
 	private:
-		friend void* doExtAddrSearch(void *p) ;
+		virtual void run();
+		void testTimeOut();
 
-		RsMutex mAddrMtx ;
+		RsMutex mAddrMtx;
+		bool mSearching;
+		bool mFoundV4;
+		bool mFoundV6;
+		bool mFirstTime;
 		rstime_t mFoundTS;
-		struct sockaddr_storage mAddr;
+		struct sockaddr_storage mAddrV4;
-		bool mFound ;
+		struct sockaddr_storage mAddrV6;
-		bool mSearching ;
+		std::list<std::string> _ip_servers;
-		std::list<std::string> _ip_servers ;
 };

libretroshare/src/util/folderiterator.cc

@@ -38,8 +38,11 @@
 namespace librs { namespace util {
 
 
-FolderIterator::FolderIterator(const std::string& folderName, bool allow_symlinks, bool allow_files_from_the_future)
+FolderIterator::FolderIterator(
-    : mFolderName(folderName),mAllowSymLinks(allow_symlinks),mAllowFilesFromTheFuture(allow_files_from_the_future)
+        const std::string& folderName, bool allow_symlinks,
+        bool allow_files_from_the_future ):
+    mFolderName(folderName), mAllowSymLinks(allow_symlinks),
+    mAllowFilesFromTheFuture(allow_files_from_the_future)
 {
     is_open = false ;
     validity = false ;

libretroshare/src/util/folderiterator.h

@@ -44,7 +44,9 @@ namespace librs { namespace util {
 class FolderIterator
 {
 public:
-    FolderIterator(const std::string& folderName,bool allow_symlinks,bool allow_files_from_the_future = true);
+	FolderIterator(
+	        const std::string& folderName, bool allow_symlinks,
+	        bool allow_files_from_the_future = true );
 	~FolderIterator();
 
     enum { TYPE_UNKNOWN = 0x00,

libretroshare/src/util/i2pcommon.cpp

@@ -50,8 +50,10 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	 * https://geti2p.net/spec/common-structures#keysandcert
 	 * https://geti2p.net/spec/common-structures#certificate
 	 */
-	if (priv.length() < 884) // base64 ( = 663 bytes = KeyCert + priv Keys)
+	if (priv.length() < privKeyMinLenth_b64) {
+		RS_WARN("key to short!");
 		return std::string();
+	}
 
 	// creat a copy to work on, need to convert it to standard base64
 	auto priv_copy(priv);
@@ -70,13 +72,14 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	uint8_t certType = 0;
 	uint16_t len = 0;
 	uint16_t signingKeyType = 0;
-	uint16_t cryptKey = 0;
+	uint16_t cryptKeyType = 0;
 
 	// only used for easy break
 	do {
 		try {
 			// jump to certificate
 			p += publicKeyLen;
+
 			// try to read type and length
 			certType = *p++;
 			len = readTwoBytesBE(p);
@@ -87,7 +90,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
 				/*
 				 * CertType.Null
 				 * type null is followed by 0x00 0x00 <END>
-				 * so has to be 0!
+				 * so len has to be 0!
 				 */
 				RS_DBG("cert is CertType.Null");
 				publicKeyLen += 3; // add 0x00 0x00 0x00
@@ -119,7 +122,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
 			// likely 7
 			signingKeyType = readTwoBytesBE(p);
 
-			RS_DBG("signing pubkey type ", certType);
+			RS_DBG("signing pubkey type ", signingKeyType);
 			if (signingKeyType >= 3 && signingKeyType <= 6) {
 				RS_DBG("signing pubkey type ", certType, " has oversize");
 				// calculate oversize
@@ -137,18 +140,18 @@ std::string publicKeyFromPrivate(std::string const &priv)
 					return std::string();
 				}
 
-				publicKeyLen += values.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted
+				publicKeyLen += values.first - 128; // 128 = default DSA key length = the space that can be used before the key must be splitted
 			}
 
 			// Crypto Public Key
 			// likely 0
-			cryptKey = readTwoBytesBE(p);
+			cryptKeyType = readTwoBytesBE(p);
-			RS_DBG("crypto pubkey type ", cryptKey);
+			RS_DBG("crypto pubkey type ", cryptKeyType);
 			// info: these are all smaller than the default 256 bytes, so no oversize calculation is needed
 
 			break;
 		}  catch (const std::out_of_range &e) {
-			RS_DBG("hit exception! ", e.what());
+			RS_DBG("hit an exception! ", e.what());
 			return std::string();
 		}
 	} while(false);
@@ -160,4 +163,107 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	return pub;
 }
 
+bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey)
+{
+	if (key.length() < pubKeyMinLenth_b64) {
+		RS_WARN("key to short!");
+		return false;
+	}
+
+	// creat a copy to work on, need to convert it to standard base64
+	auto key_copy(key);
+	std::replace(key_copy.begin(), key_copy.end(), '~', '/');
+	// replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too
+	// std::replace(copy.begin(), copy.end(), '-', '+');
+
+	// get raw data
+	std::vector<uint8_t> data;
+	RsBase64::decode(key_copy, data);
+
+	auto p = data.cbegin();
+
+	constexpr size_t publicKeyLen = 256 + 128; // default length (bytes)
+	uint8_t certType = 0;
+	uint16_t signingKeyType = 0;
+	uint16_t cryptKeyType = 0;
+
+	// try to read types
+	try {
+		// jump to certificate
+		p += publicKeyLen;
+
+		// try to read type and skip length
+		certType = *p++;
+		p += 2;
+
+		// only 0 and 5 are used / valid at this point
+		// check for == 0
+		if (certType == static_cast<typename std::underlying_type<CertType>::type>(CertType::Null)) {
+			RS_DBG("cert is CertType.Null");
+
+			signingKey = "DSA_SHA1";
+			cryptoKey = "ElGamal";
+			return true;
+		}
+
+		// check for != 5
+		if (certType != static_cast<typename std::underlying_type<CertType>::type>(CertType::Key)) {
+			// unsupported
+			RS_DBG("cert type ", certType, " is unsupported");
+			return false;
+		}
+
+		RS_DBG("cert is CertType.Key");
+
+		// Signing Public Key
+		// likely 7
+		signingKeyType = readTwoBytesBE(p);
+		RS_DBG("signing pubkey type ", signingKeyType);
+
+		// Crypto Public Key
+		// likely 0
+		cryptKeyType = readTwoBytesBE(p);
+		RS_DBG("crypto pubkey type ", cryptKeyType);
+	}  catch (const std::out_of_range &e) {
+		RS_DBG("hit an exception! ", e.what());
+		return false;
+	}
+
+	// now convert to string (this would be easier with c++17)
+#define HELPER(a, b, c) \
+	case static_cast<typename std::underlying_type<a>::type>(a::c): \
+	    b = #c; \
+	    break;
+
+	switch (signingKeyType) {
+	HELPER(SigningKeyType, signingKey, DSA_SHA1)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA256_P256)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA384_P384)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA512_P521)
+	HELPER(SigningKeyType, signingKey, RSA_SHA256_2048)
+	HELPER(SigningKeyType, signingKey, RSA_SHA384_3072)
+	HELPER(SigningKeyType, signingKey, RSA_SHA512_4096)
+	HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519)
+	HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519ph)
+	HELPER(SigningKeyType, signingKey, RedDSA_SHA512_Ed25519)
+	default:
+	    RsWarn("unkown signing key type:", signingKeyType);
+	    return false;
+	}
+
+	switch (cryptKeyType) {
+	HELPER(CryptoKeyType, cryptoKey, ElGamal)
+	HELPER(CryptoKeyType, cryptoKey, P256)
+	HELPER(CryptoKeyType, cryptoKey, P384)
+	HELPER(CryptoKeyType, cryptoKey, P521)
+	HELPER(CryptoKeyType, cryptoKey, X25519)
+	default:
+	    RsWarn("unkown crypto key type:", cryptKeyType);
+	    return false;
+	}
+#undef HELPER
+
+	return true;
+}
+
 } // namespace i2p

libretroshare/src/util/i2pcommon.h

@@ -186,6 +186,39 @@ static const std::array<std::pair<uint16_t, uint16_t>, 12> signingKeyLengths {
 	/*SigningKeyType::RedDSA_SHA512_Ed25519  */ std::make_pair<uint16_t, uint16_t>( 32,  32),
 };
 
+/*
+ * Key length infos:
+ *
+ * BOB private key
+ * len b64: 884
+ * len pln: 663
+ *
+ * BOB public key / destination
+ * len b64: 516
+ * len pln: 387
+ *
+ * SAMv3 private key
+ * len b64: 908
+ * len pln: 679
+ *
+ * SAMv3 public key
+ * len b64: 516
+ * len pln: 387
+ *
+ * Example:
+ * in bytes, public key only
+ *   384 (Key) + 3 (Null certificate) = 387 bytes
+ *   384 (Key) + 7 (key  certificate) = 391 bytes
+ *
+ * in bytes public + private key
+ *   384 (Key) + 3 (Null certificate) + 256 (ElGamal) + 20 (DSA_SHA1) = 663 bytes
+ *   384 (Key) + 7 (key  certificate) + 256 (ElGamal) + 32 (EdDSA_SHA512_Ed25519) = 679 bytes
+ */
+constexpr size_t pubKeyMinLenth_b64 = 516;
+constexpr size_t pubKeyMinLenth_bin = 387;
+constexpr size_t privKeyMinLenth_b64 = 884;
+constexpr size_t privKeyMinLenth_bin = 663;
+
 /**
  * @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs
  * @param lhs option to set
@@ -208,6 +241,15 @@ std::string keyToBase32Addr(const std::string &key);
  */
 std::string  publicKeyFromPrivate(const std::string &priv);
 
+/**
+ * @brief getKeyTypes returns the name of the utilized algorithms used by the key
+ * @param key public key
+ * @param signingKey name of the signing key, e.g. DSA_SHA1
+ * @param cryptoKey name of the crpyto key, e.g. ElGamal
+ * @return true on success, false otherwise
+ */
+bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey);
+
 } // namespace i2p
 
 #endif // I2PCOMMON_H

libretroshare/src/util/rsdebug.cc

@@ -3,9 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright (C) 2004-2008 by Robert Fernie   <retroshare@lunamutt.com>        *
+ * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2020-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -22,6 +22,8 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <iomanip>
+
 #include "util/rsdebug.h"
 
 std::ostream &operator<<(std::ostream& out, const std::error_condition& err)
@@ -36,6 +38,54 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
 	        " not available in category: " + categoryName;
 }
 
+std::error_condition rs_errno_to_condition(int errno_code)
+{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
+
+
+std::ostream& hex_dump(std::ostream& os, const void *buffer,
+                       std::size_t bufsize, bool showPrintableChars /*= true*/)
+{
+	if (buffer == nullptr) {
+		return os;
+	}
+	auto oldFormat = os.flags();
+	auto oldFillChar = os.fill();
+	constexpr std::size_t maxline{8};
+	// create a place to store text version of string
+	char renderString[maxline+1];
+	char *rsptr{renderString};
+	// convenience cast
+	const unsigned char *buf{reinterpret_cast<const unsigned char *>(buffer)};
+
+	for (std::size_t linecount=maxline; bufsize; --bufsize, ++buf) {
+		os << std::setw(2) << std::setfill('0') << std::hex
+		   << static_cast<unsigned>(*buf) << ' ';
+		*rsptr++ = std::isprint(*buf) ? *buf : '.';
+		if (--linecount == 0) {
+			*rsptr++ = '\0';  // terminate string
+			if (showPrintableChars) {
+				os << " | " << renderString;
+			}
+			os << '\n';
+			rsptr = renderString;
+			linecount = std::min(maxline, bufsize);
+		}
+	}
+	// emit newline if we haven't already
+	if (rsptr != renderString) {
+		if (showPrintableChars) {
+			for (*rsptr++ = '\0'; rsptr != &renderString[maxline+1]; ++rsptr) {
+				os << "   ";
+			}
+			os << " | " << renderString;
+		}
+		os << '\n';
+	}
+
+	os.fill(oldFillChar);
+	os.flags(oldFormat);
+	return os;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
@@ -208,3 +258,7 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
 		lineCount++;
 	}
 }
+/// All the lines before are DEPRECATED!!
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////

libretroshare/src/util/rsdebug.h

@@ -2,8 +2,8 @@
  * RetroShare debugging utilities                                              *
  *                                                                             *
  * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
  * the message around */
 std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
 
+/** Convert C errno codes to modern C++11 std::error_condition, this is quite
+ * useful to use toghether with C functions used around the code like `malloc`,
+ * `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
+ */
+std::error_condition rs_errno_to_condition(int errno_code);
+
 
 template <RsLoggerCategories CATEGORY>
 struct t_RsLogger : std::ostringstream
@@ -204,6 +210,53 @@ struct RsNoDbg
 };
 
 
+//From https://codereview.stackexchange.com/a/165162
+/**
+ * @brief hex_dump: Send Hexadecimal Dump to stream
+ * @param os: Output Stream
+ * @param buffer: Buffer to send
+ * @param bufsize: Buffer's size
+ * @param showPrintableChars: If must send printable Char too
+ * @return
+ * basic string:
+ * 61 62 63 64 65 66 31 32  | abcdef12
+ * 33 34 35 36 00 7a 79 78  | 3456.zyx
+ * 77 76 75 39 38 37 36 35  | wvu98765
+ * 34 45 64 77 61 72 64 00  | 4Edward.
+ *
+ * wide string:
+ * 41 00 00 00 20 00 00 00  | A... ...
+ * 77 00 00 00 69 00 00 00  | w...i...
+ * 64 00 00 00 65 00 00 00  | d...e...
+ * 20 00 00 00 73 00 00 00  |  ...s...
+ * 74 00 00 00 72 00 00 00  | t...r...
+ * 69 00 00 00 6e 00 00 00  | i...n...
+ * 67 00 00 00 2e 00 00 00  | g.......
+ *
+ * a double
+ * 49 92 24 49 92 24 09 40  | I.$I.$.@
+ */
+std::ostream& hex_dump(std::ostream& os, const void *buffer,
+                       std::size_t bufsize, bool showPrintableChars = true);
+
+/**
+ * @brief The hexDump struct
+ * Enable to print dump calling like that:
+ * const char test[] = "abcdef123456\0zyxwvu987654Edward";
+ * RsDbg()<<hexDump(test, sizeof(test))<<std::endl;
+ */
+struct hexDump {
+	const void *buffer;
+	std::size_t bufsize;
+	hexDump(const void *buf, std::size_t bufsz) : buffer{buf}, bufsize{bufsz} {}
+	friend std::ostream &operator<<(std::ostream &out, const hexDump &hd) {
+		return hex_dump(out, hd.buffer, hd.bufsize, true);
+	}
+};
+
+
+
+
 
 ////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
@@ -312,3 +365,8 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
 #define PQL_DEBUG_ALERT RSL_DEBUG_ALERT 
 #define PQL_DEBUG_BASIC	RSL_DEBUG_BASIC
 #define PQL_DEBUG_ALL 	RSL_DEBUG_ALL
+
+/// All the lines before are DEPRECATED!!
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////

libretroshare/src/util/rsdir.cc

@@ -458,6 +458,36 @@ bool	RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d
 	return true;
 }
 
+rstime_t RsDirUtil::lastWriteTime(
+        const std::string& path,
+        std::error_condition& errc )
+{
+	if(!fileExists(path))
+	{
+		errc = std::errc::no_such_file_or_directory;
+		return 0;
+	}
+
+#ifdef WINDOWS_SYS
+	struct _stati64 buf;
+	std::wstring wPath;
+	librs::util::ConvertUtf8ToUtf16(path, wPath);
+	if ( 0 == _wstati64(wPath.c_str(), &buf))
+#else
+	struct stat64 buf;
+	if ( 0 == stat64(path.c_str(), &buf))
+#endif
+	{
+		/* errc output param is guaranted to be meaningful only if an error
+		 * happens so is not strictly necessary but we clean it anyway just
+		 * in case */
+		errc = std::error_condition();
+		return buf.st_mtime;
+	}
+
+	errc = std::error_condition(errno, std::generic_category());
+	return 0;
+}
 
 bool RsDirUtil::checkDirectory(const std::string& dir)
 {

libretroshare/src/util/rsdir.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2004-2007  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2020-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -27,11 +27,13 @@
 #include <string>
 #include <list>
 #include <set>
-#include <stdint.h>
+#include <cstdint>
+#include <system_error>
 
 class RsThread;
 
-#include <retroshare/rstypes.h>
+#include "retroshare/rstypes.h"
+#include "util/rsmemory.h"
 
 #ifndef WINDOWS_SYS
 typedef int rs_lock_handle_t;
@@ -79,10 +81,11 @@ const char *scanf_string_for_uint(int bytes) ;
 
 int     	breakupDirList(const std::string& path, std::list<std::string> &subdirs);
 
-// Splits the path into parent directory and file. File can be empty if full_path is a dir ending with '/'
+/** Splits the path into parent directory and file. File can be empty if
-// if full_path does not contain a directory, then dir will be "." and file will be full_path.
+ * full_path is a dir ending with '/' if full_path does not contain a directory,
-
+ * then dir will be "." and file will be full_path */
-bool        splitDirFromFile(const std::string& full_path,std::string& dir, std::string& file);
+bool splitDirFromFile( const std::string& full_path,
+                       std::string& dir, std::string& file );
 
 bool 		copyFile(const std::string& source,const std::string& dest);
 
@@ -92,6 +95,17 @@ bool moveFile(const std::string& source, const std::string& dest);
 bool 		removeFile(const std::string& file);
 bool 		fileExists(const std::string& file);
 bool    	checkFile(const std::string& filename,uint64_t& file_size,bool disallow_empty_file = false);
+
+/**
+ * @brief Retrieve file last modification time
+ * @param path path of the file
+ * @param errc optional storage for error details
+ * @return 0 on error, file modification time represented as unix epoch otherwise.
+ */
+rstime_t lastWriteTime(
+        const std::string& path,
+        std::error_condition& errc = RS_DEFAULT_STORAGE_PARAM(std::error_condition) );
+
 bool    	checkDirectory(const std::string& dir);
 bool    	checkCreateDirectory(const std::string& dir);
 

libretroshare/src/util/rsdnsutils.cc

@@ -0,0 +1,361 @@
+/*******************************************************************************
+ * libretroshare/src/util: rsdnsutils.cc                                       *
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright 2021 Phenom <retrosharephenom@gmail.com>                          *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+//#define DEBUG_SPEC_DNS 1
+
+#include "util/rsnet.h"
+
+#include "util/rsdebug.h"
+#include "util/rsthreads.h"
+#include "util/rsstring.h"
+
+#ifdef WINDOWS_SYS
+#else
+#include <netdb.h>
+#endif
+
+//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
+constexpr uint16_t DNSC_IN    = 1; //Internet (IN)
+//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
+constexpr uint16_t DNST_A     = 1; //Ipv4 address
+constexpr uint16_t DNST_AAAA  =28; //Ipv6 address
+
+///Need to pack as we use sizeof them. (avoid padding)
+#pragma pack(1)
+//DNS header structure
+struct DNS_HEADER
+{
+	unsigned short id; // identification number
+
+	//Header flags https://www.bind9.net/dns-header-flags
+	unsigned char rd :1;    //bit 07 Recursion Desired, indicates if the client means a recursive query
+	unsigned char tc :1;    //bit 06 TrunCation, indicates that this message was truncated due to excessive length
+	unsigned char aa :1;    //bit 05 Authoritative Answer, in a response, indicates if the DNS server is authoritative for the queried hostname
+	unsigned char opcode :4;//bit 01-04 The type can be QUERY (standard query, 0), IQUERY (inverse query, 1), or STATUS (server status request, 2)
+	unsigned char qr :1;    //bit 00 Indicates if the message is a query (0) or a reply (1)
+
+	unsigned char rcode :4; //bit 12-15 Response Code can be NOERROR (0), FORMERR (1, Format error), SERVFAIL (2), NXDOMAIN (3, Nonexistent domain), etc.
+	unsigned char cd :1;    //bit 11 Checking Disabled    [RFC 4035][RFC 6840][RFC Errata 4927] used by DNSSEC
+	unsigned char ad :1;    //bit 10 Authentic Data       [RFC 4035][RFC 6840][RFC Errata 4924] used by DNSSEC
+	unsigned char z :1;     //bit 09 Zero, reserved for future use
+	unsigned char ra :1;    //bit 08 Recursion Available  [RFC 1035] in a response, indicates if the replying DNS server supports recursion
+
+	unsigned short q_count; // number of question entries
+	unsigned short ans_count; // number of answer entries
+	unsigned short auth_count; // number of authority resource records entries
+	unsigned short add_count; // number of additional resource record entries
+};
+//// OpCode text https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-5
+//static const char *opcodetext[] = { "QUERY",      "IQUERY",     "STATUS",
+//                                    "RESERVED3",  "NOTIFY",     "UPDATE",
+//                                    "STATEFUL",   "RESERVED7",  "RESERVED8",
+//                                    "RESERVED9",  "RESERVED10", "RESERVED11",
+//                                    "RESERVED12", "RESERVED13", "RESERVED14",
+//                                    "RESERVED15" };
+//// RCode text https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6
+//static const char *rcodetext[] = { "NOERROR",    "FORMERR",    "SERVFAIL",
+//                                   "NXDOMAIN",   "NOTIMP",     "REFUSED",
+//                                   "YXDOMAIN",   "YXRRSET",    "NXRRSET",
+//                                   "NOTAUTH",    "NOTZONE",    "DSOTYPENI",
+//                                   "RESERVED12", "RESERVED13", "RESERVED14",
+//                                   "RESERVED15", "BADVERS",    "BADKEY",
+//                                   "BADTIME",    "BADMODE",    "BADNAME",
+//                                   "BADALG",     "BADTRUNC",   "BADCOOKIE"};
+
+//Constant sized fields of query structure
+//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml
+struct QUESTION
+{
+	unsigned short qtype;     //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
+	unsigned short qclass;    //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
+};
+
+//Constant sized fields of the resource record structure
+struct RR_DATA
+{
+    unsigned short rtype;   //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
+    unsigned short rclass;  //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
+    unsigned int rttl;      //Time To Live is the number of seconds left before the information expires. (32bits integer, so maximum 140 years)
+    unsigned short data_len;//Lenght of following data
+};
+#pragma pack()
+
+bool rsGetHostByNameSpecDNS(const std::string& servername, const std::string& hostname, std::string& returned_addr, int timeout_s /*= -1*/)
+{
+#ifdef DEBUG_SPEC_DNS
+	RS_DBG("servername=", servername, " hostname=", hostname);
+#endif
+
+	if (strlen(servername.c_str()) > 256)
+	{
+		RS_ERR("servername is too long > 256 chars: ", servername);
+		return false;
+	}
+	if (strlen(hostname.c_str()) > 256)
+	{
+		RS_ERR("hostname is too long > 256 chars: ", hostname);
+		return false;
+	}
+
+	sockaddr_storage serverAddr;
+	bool validServer =   sockaddr_storage_inet_pton(serverAddr, servername)
+	                  && sockaddr_storage_isValidNet(serverAddr);
+	bool isIPV4 = validServer && sockaddr_storage_ipv6_to_ipv4(serverAddr);
+
+	if (!validServer)
+	{
+		in_addr in ;
+		if (!rsGetHostByName(servername, in))
+		{
+			RS_ERR("servername is on an unknow format: ", servername);
+			return false;
+		}
+
+		validServer =   sockaddr_storage_inet_pton(serverAddr, rs_inet_ntoa(in))
+		             && sockaddr_storage_isValidNet(serverAddr);
+		isIPV4 = validServer && sockaddr_storage_ipv6_to_ipv4(serverAddr);
+
+		if (!validServer)
+		{
+			RS_ERR("rsGetHostByName return bad answer: ", rs_inet_ntoa(in));
+			return false;
+		}
+	}
+
+	sockaddr_storage_setport( serverAddr, 53);
+
+	//Set the DNS structure to standard queries
+	unsigned char buf[65536];
+	struct DNS_HEADER* dns = (struct DNS_HEADER *)&buf;
+	dns->id = static_cast<unsigned short>(htons(getpid())); //Transaction Id
+	//dns flags = 0x0100 Standard Query
+	dns->qr = 0;     //Query/Response: Message is a query
+	dns->opcode = 0; //OpCode: Standard query
+	dns->aa = 0;     //Authoritative: Server is not an authority for domain
+	dns->tc = 0;     //TrunCated: Message is not truncated
+	dns->rd = 1;     //Recursion Desired: Do query recursively
+	dns->ra = 0;     //Recursion Available: Server cannot do recursive queries
+	dns->z = 0;      //Z: reserved
+	dns->ad = 0;     //Authentic Data: Answer/authority portion was not authenticated by the server
+	dns->cd = 0;     //Checking Disabled: Unacceptable
+	dns->rcode = 0;  //Response Code: No error
+
+	dns->q_count = htons(1); //1 Question
+	dns->ans_count = 0;      //0 Answer
+	dns->auth_count = 0;     //0 Authority RRs
+	dns->add_count = 0;      //0 Additional RRs
+	size_t curSendSize = sizeof(struct DNS_HEADER);
+
+	//Point to the query server name portion
+	unsigned char* qname =static_cast<unsigned char*>(&buf[curSendSize]);
+	//First byte is Label Type: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-10
+	qname[0] = 0x04; //One Label with Normal label lower 6 bits is the length of the label
+	memcpy(&qname[1],hostname.c_str(),strlen(hostname.c_str()));
+	size_t qnameSize = strlen((const char*)qname);
+	// Format Hostname like www.google.com to 3www6google3com
+	{
+		size_t last = qnameSize;
+		for(size_t i = qnameSize-1 ; i > 0 ; i--)
+			if(qname[i]=='.')
+			{
+				qname[i]=last-i-1;
+				last = i;
+			}
+	}
+	curSendSize += qnameSize +1; //With \0 terminator
+
+	//Point to the query constant portion
+	struct QUESTION* qinfo =(struct QUESTION*)&buf[curSendSize];
+	qinfo->qtype  = htons(isIPV4 ? DNST_A : DNST_AAAA);  //Type: A / AAAA(Host Address)
+	qinfo->qclass = htons(DNSC_IN); //Class: IN
+	curSendSize += sizeof(struct QUESTION);
+
+#ifdef DEBUG_SPEC_DNS
+	RS_DBG("Sending Packet:\n", hexDump(buf, curSendSize));
+#endif
+	int s = socket(serverAddr.ss_family , SOCK_DGRAM , IPPROTO_UDP); //UDP packet for DNS queries
+	if(s<0)
+	{
+		RS_ERR("Could not open socket.");
+		return false;
+	}
+
+	if (timeout_s > -1)
+		rs_setSockTimeout(s, true, timeout_s);
+
+	ssize_t send_size = sendto( s, (char*)buf, curSendSize, 0
+	                          , (struct sockaddr*)&serverAddr
+	                          , isIPV4 ? sizeof(sockaddr_in)
+	                                   : sizeof(sockaddr_in6)
+	                          );
+	if( send_size < 0)
+	{
+		RS_ERR("Send Failed with size = ", send_size);
+		close(s);
+		return false;
+	}
+
+#ifdef DEBUG_SPEC_DNS
+	RS_DBG("Waiting answer...");
+#endif
+	//****************************************************************************************//
+	//---                          Receive the answer                                      ---//
+	//****************************************************************************************//
+	socklen_t sa_size = static_cast<socklen_t>(isIPV4 ? sizeof(sockaddr_in)
+	                                                  : sizeof(sockaddr_in6)
+	                                          );
+	ssize_t rec_size=recvfrom( s,(char*)buf , 65536 , 0
+	                         , (struct sockaddr*)&serverAddr
+	                         , &sa_size
+	                         );
+	close(s); // No more need of this socket, close it.
+	if(rec_size <= 0)
+	{
+		RS_ERR("Receive Failed");
+		return false;
+	}
+#ifdef DEBUG_SPEC_DNS
+	RS_DBG("Received:\n", hexDump(buf, rec_size));
+#endif
+
+
+	if (rec_size< static_cast<ssize_t>(sizeof(struct DNS_HEADER)) )
+	{
+		RS_ERR("Request received too small to get DNSHeader.");
+		return false;
+	}
+#ifdef DEBUG_SPEC_DNS
+	//Point to the header portion
+	dns = (struct DNS_HEADER*) buf;
+	RS_DBG("The response contains :\n"
+	       ,ntohs(dns->q_count)    , " Questions.\n"
+	       ,ntohs(dns->ans_count)  , " Answers.\n"
+	       ,ntohs(dns->auth_count) , " Authoritative Servers.\n"
+	       ,ntohs(dns->add_count)  , " Additional records.");
+#endif
+	size_t curRecSize = sizeof(struct DNS_HEADER);
+
+
+	if (rec_size< static_cast<ssize_t>(curRecSize + 1 + sizeof(struct QUESTION)) )
+	{
+		RS_ERR("Request received too small to get Question return.");
+		return false;
+	}
+	//Point to the query portion
+	unsigned char* qnameRecv =static_cast<unsigned char*>(&buf[curRecSize]);
+	if (memcmp(qname,qnameRecv,qnameSize + 1 + sizeof(struct QUESTION)) )
+	{
+		RS_ERR("Request received different from that sent.");
+		return false;
+	}
+	curRecSize += qnameSize + 1 + sizeof(struct QUESTION);
+
+	if (rec_size< static_cast<ssize_t>(curRecSize + 2) )
+	{
+		RS_ERR("Request received too small to get Answer return.");
+		return false;
+	}
+	//Point to the Answer portion
+	unsigned char* reader = &buf[curRecSize];
+
+	size_t rLabelSize=0;
+	if((reader[0]&0xC0) == 0)
+	{
+		//Normal label lower 6 bits is the length of the label
+		rLabelSize=(reader[0]&~(0xC0)*256) + reader[1];
+	}
+	else if ((reader[0]&0xC0) == 0xC0)
+	{
+		//Compressed label the lower 6 bits and the 8 bits from next octet form a pointer to the compression target.
+		//Don't need to read it, maybe the same as in Query
+		rLabelSize=0;
+	}
+	else
+	{
+		RS_ERR("Answer received with unmanaged label format.");
+		return false;
+	}
+	curRecSize += 2 + rLabelSize;
+
+	if (rec_size< static_cast<ssize_t>(curRecSize + sizeof(struct RR_DATA)) )
+	{
+		RS_ERR("Request received too small to get Data return.");
+		return false;
+	}
+	//Point to the query portion
+	struct RR_DATA* rec_data = (struct RR_DATA *)&buf[curRecSize];
+	if (rec_data->rtype!=qinfo->qtype)
+	{
+		RS_ERR("Answer's type received different from query sent.");
+		return false;
+	}
+	if (rec_data->rclass!=qinfo->qclass)
+	{
+		RS_ERR("Answer's class received different from query sent.");
+		return false;
+	}
+	curRecSize += sizeof(struct RR_DATA);
+
+	if (rec_size< static_cast<ssize_t>(curRecSize + ntohs(rec_data->data_len)) )
+	{
+		RS_ERR("Request received too small to get Full Data return.");
+		return false;
+	}
+
+	//Retrieve Address
+	if(ntohs(rec_data->data_len)==4)
+	{
+		if (isIPV4)
+		{
+			in_addr ipv4Add;
+			ipv4Add.s_addr=*(in_addr_t*)&buf[curRecSize];
+#ifdef DEBUG_SPEC_DNS
+			RS_DBG("Retrieve address: ", rs_inet_ntoa(ipv4Add));
+#endif
+			returned_addr = rs_inet_ntoa(ipv4Add);
+			return true;
+		}
+	}
+	else if(ntohs(rec_data->data_len)==16)
+	{
+		if (!isIPV4)
+		{
+			in6_addr ipv6Add;
+			ipv6Add =*(in6_addr*)&buf[curRecSize];
+
+			struct sockaddr_storage ss;
+			sockaddr_storage_clear(ss);
+			sockaddr_in6 addr_ipv6;
+			addr_ipv6.sin6_addr = ipv6Add;
+			sockaddr_storage_setipv6(ss,&addr_ipv6);
+
+#ifdef DEBUG_SPEC_DNS
+			RS_DBG("Retrieve address: ", sockaddr_storage_iptostring(ss).c_str());
+#endif
+			returned_addr = sockaddr_storage_iptostring(ss);
+			return true;
+		}
+	}
+
+	RS_ERR("Retrieve unmanaged data size=", ntohs(rec_data->data_len));
+	return false;
+}

libretroshare/src/util/rserrno.cc

@@ -1,54 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/util: rserrno.cc                                          *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-
-#include <cerrno>
-
-#define RS_INTERNAL_ERRNO_CASE(e) case e: return #e
-
-const char* rsErrnoName(int err)
-{
-	switch (err)
-	{
-	RS_INTERNAL_ERRNO_CASE(EINVAL);
-	RS_INTERNAL_ERRNO_CASE(EBUSY);
-	RS_INTERNAL_ERRNO_CASE(EAGAIN);
-	RS_INTERNAL_ERRNO_CASE(EDEADLK);
-	RS_INTERNAL_ERRNO_CASE(EPERM);
-	RS_INTERNAL_ERRNO_CASE(EBADF);
-	RS_INTERNAL_ERRNO_CASE(EFAULT);
-	RS_INTERNAL_ERRNO_CASE(ENOTSOCK);
-	RS_INTERNAL_ERRNO_CASE(EISCONN);
-	RS_INTERNAL_ERRNO_CASE(ECONNREFUSED);
-	RS_INTERNAL_ERRNO_CASE(ETIMEDOUT);
-	RS_INTERNAL_ERRNO_CASE(ENETUNREACH);
-	RS_INTERNAL_ERRNO_CASE(EADDRINUSE);
-	RS_INTERNAL_ERRNO_CASE(EINPROGRESS);
-	RS_INTERNAL_ERRNO_CASE(EALREADY);
-	RS_INTERNAL_ERRNO_CASE(ENOTCONN);
-	RS_INTERNAL_ERRNO_CASE(EPIPE);
-	RS_INTERNAL_ERRNO_CASE(ECONNRESET);
-	RS_INTERNAL_ERRNO_CASE(EHOSTUNREACH);
-	RS_INTERNAL_ERRNO_CASE(EADDRNOTAVAIL);
-	}
-
-	return "rsErrnoName UNKNOWN ERROR CODE";
-}

libretroshare/src/util/rsmemory.cc

@@ -1,53 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/util: rsmemory.cc                                         *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net>           *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#include "util/rsmemory.h"
-
-void *rs_malloc(size_t size) 
-{
-    static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything!
-    
-    if(size == 0)
-    {
-        std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl;
-	print_stacktrace() ;
-    	return NULL ;
-    }
-    
-    if(size > SAFE_MEMALLOC_THRESHOLD)
-    {
-        std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
-	print_stacktrace() ;
-    	return NULL ;
-    }
-    
-    void *mem = malloc(size) ;
-    
-    if(mem == NULL)
-    {
-        std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
-	print_stacktrace() ;
-    	return NULL ;
-    }
-    
-    return mem ;
-}
-

libretroshare/src/util/rsmemory.h

@@ -3,8 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012 Cyril Soler <csoler@users.sourceforge.net>                   *
+ * Copyright (C) 2012  Cyril Soler <csoler@users.sourceforge.net>              *
- * Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net>                *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -25,8 +26,10 @@
 #include <cstdlib>
 #include <iostream>
 #include <memory>
+#include <system_error>
 
 #include "util/stacktrace.h"
+#include "util/rsdebug.h"
 
 /**
  * @brief Shorthand macro to declare optional functions output parameters
@@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
  * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
 template<typename T> using rs_owner_ptr = T*;
 
-void *rs_malloc(size_t size) ;
+
+/// 1Gb should be enough for everything!
+static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
+
+/** Comfortable templated safer malloc, just use it specifing the type of the
+ * pointer to be returned without need of ugly casting the returned pointer
+ * `uint8_t* ptr = rs_malloc<uint8_t>(40);`
+ * @param[in] size number of bytes to allocate
+ * @param[out] ec optional storage for error details. Value is meaningful only
+ *	whem nullptr is returned.
+ * @return nullptr on error, pointer to the allocated chuck of memory on success
+ */
+template<typename T = void> rs_owner_ptr<T> rs_malloc(
+        size_t size,
+        rs_view_ptr<std::error_condition> ec = nullptr )
+{
+	if(size == 0)
+	{
+		if(!ec)
+		{
+			RS_ERR("A chunk of size 0 was requested");
+			print_stacktrace();
+			exit(static_cast<int>(std::errc::invalid_argument));
+		}
+
+		*ec = std::errc::invalid_argument;
+		return nullptr;
+	}
+
+	if(size > SAFE_MEMALLOC_THRESHOLD)
+	{
+		if(!ec)
+		{
+			RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
+			        " was requested" );
+			exit(static_cast<int>(std::errc::argument_out_of_domain));
+		}
+
+		*ec = std::errc::argument_out_of_domain;
+		return nullptr;
+	}
+
+	void* mem = malloc(size);
+	if(!mem)
+	{
+		if(!ec)
+		{
+			RS_ERR( "Allocation failed for a chunk of ", size,
+			        " bytes with: ", errno);
+			print_stacktrace();
+			exit(errno);
+		}
+
+		*ec = rs_errno_to_condition(errno);
+		return nullptr;
+	}
+
+	return static_cast<rs_owner_ptr<T>>(mem);
+}
+
 
 /** @deprecated use std::unique_ptr instead
 // This is a scope guard to release the memory block when going of of the current scope.
@@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
 //
 // }	// mem gets freed automatically
 */
-class RsTemporaryMemory
+class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
 {
 public:
 	explicit RsTemporaryMemory(size_t s)

libretroshare/src/util/rsnet.cc

@@ -174,3 +174,17 @@ std::string rs_inet_ntoa(struct in_addr in)
 	rs_sprintf(str, "%u.%u.%u.%u", (int) bytes[0], (int) bytes[1], (int) bytes[2], (int) bytes[3]);
 	return str;
 }
+
+int rs_setSockTimeout( int sockfd, bool forReceive /*= true*/
+                     , int timeout_Sec /*= 0*/, int timeout_uSec /*= 0*/)
+{
+#ifdef WINDOWS_SYS
+		DWORD timeout = timeout_Sec * 1000 + timeout_uSec;
+#else
+		struct timeval timeout;
+		timeout.tv_sec = timeout_Sec;
+		timeout.tv_usec = timeout_uSec;
+#endif
+		return setsockopt( sockfd, SOL_SOCKET, forReceive ? SO_RCVTIMEO : SO_SNDTIMEO
+		                 , (const char*)&timeout, sizeof timeout);
+}

libretroshare/src/util/rsnet.h

@@ -84,6 +84,18 @@ bool isExternalNet(const struct in_addr *addr);
 // uses a re-entrant version of gethostbyname
 bool rsGetHostByName(const std::string& hostname, in_addr& returned_addr) ;
 
+/**
+ * @brief Get hostName address using specific DNS server.
+ * Using it allow to direct ask our Address to IP, so no need to have a DNS (IPv4 or IPv6).
+ * If we ask to a IPv6 DNS Server, it respond for our IPv6 address.
+ * @param servername: Address or name of DNS Server.
+ * @param hostname: HosteName to get IP ("myip.opendns.com" to get own).
+ * @param returned_addr: returned IP of hostname.
+ * @param timeout_s: Timeout in sec to wait server response.
+ * @return True in success.
+ */
+bool rsGetHostByNameSpecDNS(const std::string& servername, const std::string& hostname, std::string& returned_addr, int timeout_s = -1);
+
 std::ostream& operator<<(std::ostream& o, const sockaddr_in&);
 std::ostream& operator<<(std::ostream& o, const sockaddr_storage&);
 
@@ -159,4 +171,14 @@ bool sockaddr_storage_inet_ntop(const sockaddr_storage &addr, std::string &dst);
 int rs_setsockopt( int sockfd, int level, int optname,
                    const uint8_t *optval, uint32_t optlen );
 
+/**
+ * @brief Set socket Timeout.
+ * @param sockfd: The socket to manage.
+ * @param forReceive: True for Receive, False for Send.
+ * @param timeout_Sec: Timeout second part.
+ * @param timeout_uSec: Timeout micro second part.
+ * @return 0 on success, -1 for errors.
+ */
+int rs_setSockTimeout( int sockfd, bool forReceive = true, int timeout_Sec = 0, int timeout_uSec = 0);
+
 #endif /* RS_UNIVERSAL_NETWORK_HEADER */

libretroshare/src/util/rsnet_ss.cc

@@ -233,8 +233,7 @@ bool sockaddr_storage_setport(struct sockaddr_storage &addr, uint16_t port)
 bool sockaddr_storage_setipv4(struct sockaddr_storage &addr, const sockaddr_in *addr_ipv4)
 {
 #ifdef SS_DEBUG
-	std::cerr << "sockaddr_storage_setipv4()";
+	RS_ERR();
-	std::cerr << std::endl;
 #endif
 
 	sockaddr_storage_clear(addr);
@@ -249,7 +248,9 @@ bool sockaddr_storage_setipv4(struct sockaddr_storage &addr, const sockaddr_in *
 
 bool sockaddr_storage_setipv6(struct sockaddr_storage &addr, const sockaddr_in6 *addr_ipv6)
 {
-	std::cerr << "sockaddr_storage_setipv6()" << std::endl;
+#ifdef SS_DEBUG
+	RS_ERR();
+#endif
 
 	sockaddr_storage_clear(addr);
 	struct sockaddr_in6 *ipv6_ptr = to_ipv6_ptr(addr);
@@ -262,6 +263,7 @@ bool sockaddr_storage_setipv6(struct sockaddr_storage &addr, const sockaddr_in6
 }
 
 #ifdef WINDOWS_SYS
+#ifndef InetPtonA
 int inet_pton(int af, const char *src, void *dst)
 {
 	sockaddr_storage ss;
@@ -288,6 +290,7 @@ int inet_pton(int af, const char *src, void *dst)
 	return 0;
 }
 #endif
+#endif
 
 bool sockaddr_storage_inet_pton( sockaddr_storage &addr,
                                  const std::string& ipStr )

libretroshare/src/util/rsthreads.cc

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2004-2007  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -25,7 +25,6 @@
 #include "rsthreads.h"
 
 #include "util/rsdebug.h"
-#include "util/rserrno.h"
 
 #include <chrono>
 #include <ctime>
@@ -92,11 +91,12 @@ void RsThread::resetTid()
 #ifdef WINDOWS_SYS
 	memset (&mTid, 0, sizeof(mTid));
 #else
-	mTid = 0;
+	mTid = pthread_t(); //Thread identifiers should be considered opaque and can be null.
 #endif
 }
 
-RsThread::RsThread() : mHasStopped(true), mShouldStop(false), mLastTid()
+RsThread::RsThread() : mInitMtx("RsThread"), mHasStopped(true),
+    mShouldStop(false), mLastTid()
 #ifdef RS_THREAD_FORCE_STOP
   , mStopTimeout(0)
 #endif
@@ -115,19 +115,24 @@ void RsThread::askForStop()
 
 void RsThread::wrapRun()
 {
+	{RS_STACK_MUTEX(mInitMtx);} // Waiting Init done.
 	run();
 	resetTid();
 	mHasStopped = true;
 }
 
 void RsThread::fullstop()
+{
+	askForStop();
+	waitWhileStopping();
+}
+
+void RsThread::waitWhileStopping()
 {
 #ifdef RS_THREAD_FORCE_STOP
 	const rstime_t stopRequTS = time(nullptr);
 #endif
 
-	askForStop();
-
 	const pthread_t callerTid = pthread_self();
 	if(pthread_equal(mTid, callerTid))
 	{
@@ -169,7 +174,8 @@ void RsThread::fullstop()
 			{
 				RsErr() << __PRETTY_FUNCTION__ << " pthread_cancel("
 				        << std::hex << mTid << std::dec <<") returned "
-				        << terr << " " << rsErrnoName(terr) << std::endl;
+				        << terr << " " << rs_errno_to_condition(terr)
+				        << std::endl;
 				print_stacktrace();
 			}
 
@@ -184,23 +190,15 @@ bool RsThread::start(const std::string& threadName)
 	// Atomically check if the thread was already started and set it as running
 	if(mHasStopped.exchange(false))
 	{
+		RS_STACK_MUTEX(mInitMtx); // Block thread starting to run
+
 		mShouldStop = false;
 		int pError = pthread_create(
 		            &mTid, nullptr, &rsthread_init, static_cast<void*>(this) );
 		if(pError)
 		{
-			RsErr() << __PRETTY_FUNCTION__ << " pthread_create could not create"
+			RS_ERR( "pthread_create could not create new thread: ", threadName,
-			        << " new thread: " << threadName << " pError: " << pError
+			        rs_errno_to_condition(pError) );
-			        << std::endl;
-			mHasStopped = true;
-			print_stacktrace();
-			return false;
-		}
-		if(!mTid)
-		{
-			RsErr() << __PRETTY_FUNCTION__ << " pthread_create could not create"
-			        << " new thread: " << threadName << " mTid: " << mTid
-			        << std::endl;
 			mHasStopped = true;
 			print_stacktrace();
 			return false;
@@ -281,7 +279,7 @@ void RsMutex::lock()
 	if( err != 0)
 	{
 		RsErr() << __PRETTY_FUNCTION__ << "pthread_mutex_lock returned: "
-		        << rsErrnoName(err)
+		        << rs_errno_to_condition(err)
 #ifdef RS_MUTEX_DEBUG
 		        << " name: " << name()
 #endif
@@ -325,7 +323,11 @@ RsThread::~RsThread()
 		        << "likely to crash because of this." << std::endl;
 		print_stacktrace();
 
-		fullstop();
+		/* Last resort attempt to stop the thread in a less pathological state.
+		 * Don't call fullstop() as it rely on virtual methods that at this
+		 * point are not anymore the one from inerithing classes causing
+		 * compilers to output a warning */
+		waitWhileStopping();
 	}
 }
 

libretroshare/src/util/rsthreads.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2004-2006  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -105,7 +105,7 @@ class RsStackMutex
 {
 public:
 
-	RsStackMutex(RsMutex &mtx) : mMtx(mtx)
+	explicit RsStackMutex(RsMutex& mtx): mMtx(mtx)
 	{
 		mMtx.lock();
 #ifdef RS_MUTEX_DEBUG
@@ -261,6 +261,13 @@ private:
 	/** Call @see run() setting the appropriate flags around it*/
 	void wrapRun();
 
+	/** Wait the thread while it is stopping */
+	void waitWhileStopping();
+
+	/** To be sure Init (pthread_setname_np) is done before continue thread.
+	 * Else can finish before and crash. */
+	RsMutex mInitMtx;
+
 	/// True if thread is stopped, false otherwise
 	std::atomic<bool> mHasStopped;
 

libretroshare/src/util/rstime.h

@@ -3,7 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Cyril Soler <csoler@users.sourceforge.net>           *
+ * Copyright 2013  Cyril Soler <csoler@users.sourceforge.net>                  *
+ * Copyright 2018  Gioacchino Mazzurco <gio@altermundi.net>                    *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *

plugins/Common/retroshare_plugin.pri

@@ -103,7 +103,8 @@ win32 {
 	for(bin, BIN_DIR):LIBS += -L"$$bin"
 	LIBS += -lpthread
 
-	QMAKE_LFLAGS += -Wl,--end-group
+	# Do not add for MinGW
+	!isEmpty(QMAKE_SH): QMAKE_LFLAGS += -Wl,--end-group
 }
 
 macx {

plugins/FeedReader/gui/AddFeedDialog.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>715</width>
+    <width>1068</width>
-    <height>605</height>
+    <height>880</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -37,14 +37,14 @@
     <widget class="HeaderFrame" name="headerFrame"/>
    </item>
    <item>
-    <widget class="QFrame" name="frame">
+    <widget class="QFrame" name="gradFrame">
      <property name="frameShape">
       <enum>QFrame::NoFrame</enum>
      </property>
      <property name="frameShadow">
       <enum>QFrame::Raised</enum>
      </property>
-     <layout class="QGridLayout" name="gridLayout_7">
+     <layout class="QGridLayout" name="gradFrame_GL">
       <item row="3" column="0">
        <widget class="QGroupBox" name="typeGroupBox">
         <property name="title">
@@ -73,7 +73,7 @@
             </widget>
            </item>
            <item>
-            <widget class="QComboBox" name="forumComboBox"/>
+            <widget class="RSComboBox" name="forumComboBox"/>
            </item>
           </layout>
          </item>
@@ -306,6 +306,14 @@
         </item>
         <item row="1" column="0">
          <widget class="QLabel" name="nameLabel">
+          <property name="font">
+           <font>
+            <pointsize>11</pointsize>
+            <weight>75</weight>
+            <italic>true</italic>
+            <bold>true</bold>
+           </font>
+          </property>
           <property name="text">
            <string>Name:</string>
           </property>
@@ -405,6 +413,11 @@
    <header>gui/common/HeaderFrame.h</header>
    <container>1</container>
   </customwidget>
+  <customwidget>
+   <class>RSComboBox</class>
+   <extends>QComboBox</extends>
+   <header>gui/common/RSComboBox.h</header>
+  </customwidget>
  </customwidgets>
  <tabstops>
   <tabstop>urlLineEdit</tabstop>

plugins/FeedReader/gui/FeedReaderDialog.ui

@@ -86,7 +86,14 @@
            </widget>
           </item>
           <item>
-           <widget class="StyledLabel" name="titleBarLabel">
+           <widget class="QLabel" name="titleBarLabel">
+            <property name="font">
+             <font>
+              <pointsize>12</pointsize>
+              <weight>75</weight>
+              <bold>true</bold>
+             </font>
+            </property>
             <property name="text">
              <string>Feeds</string>
             </property>
@@ -178,11 +185,6 @@
   </layout>
  </widget>
  <customwidgets>
-  <customwidget>
-   <class>StyledLabel</class>
-   <extends>QLabel</extends>
-   <header>gui/common/StyledLabel.h</header>
-  </customwidget>
   <customwidget>
    <class>RSTreeWidget</class>
    <extends>QTreeWidget</extends>