Merge remote-tracking branch 'upstream/master' into v0.6-TorControl

.gitignore

@@ -15,3 +15,5 @@ Thumbs.db
 *.pro.user
 .kdev4
 *.kdev4
+
+!supportlibs/libsam3/Makefile

.gitlab-ci.yml

@@ -3,30 +3,46 @@ image: docker:stable
 services:
   - docker:stable-dind
 
+stages:
+  - build
+  - test
+
 workflow:
   rules:
     - if: $CI_MERGE_REQUEST_ID
     - if: $CI_COMMIT_BRANCH
 
-build-and-test:
+variables:
+  UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
+        
+build-ubuntu-test-image:
+    stage: build
     script:
         - > 
-          if [ -n "$CI_MERGE_REQUEST_ID" ]; then
+          docker login "$CI_REGISTRY"
-          REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
+          --username "$CI_REGISTRY_USER"
-          REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
+          --password "$CI_REGISTRY_PASSWORD"
-          else
-          REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
-          REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
-          fi ;
-          export REPO_ARGS ;
-          echo REPO_ARGS=$REPO_ARGS ;
         - mkdir Dockercontext
         - >
-          docker build -t retroshare:testing $REPO_ARGS
+          docker build
+          -t $UBUNTU_TESTING_IMAGE_TAG
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
           --file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
           Dockercontext
+        - docker push $UBUNTU_TESTING_IMAGE_TAG
+        - echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
+
+test-ubuntu:
+    stage: test
+    script:
         - > 
-          docker run --name retroshare --detach --tty retroshare:testing
+          docker login "$CI_REGISTRY"
+          --username "$CI_REGISTRY_USER"
+          --password "$CI_REGISTRY_PASSWORD"
+        - echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
+        - docker pull $UBUNTU_TESTING_IMAGE_TAG
+        - >
+          docker run --name retroshare --detach --tty $UBUNTU_TESTING_IMAGE_TAG
           retroshare-service --jsonApiPort 9092
         - apk add jq
         - >
@@ -51,3 +67,43 @@ build-and-test:
           docker exec retroshare
           curl --verbose http://127.0.0.1:9092/rsMsgs/getChatLobbyList | jq
         - docker container stop retroshare
+          
+build-android-arm-apk:
+    stage: build
+    script:
+        - mkdir Dockercontext
+        - >
+          docker build -t retroshare:android_arm_latest
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
+          --build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
+          --file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
+          Dockercontext
+# see https://stackoverflow.com/a/59055906
+        - >
+          docker cp
+          $(docker create --rm retroshare:android_arm_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
+          $CI_PROJECT_DIR/RetroShare_Android_Service-arm.apk
+    artifacts:
+      paths:
+        - RetroShare_Android_Service-arm.apk
+
+# Use separate runner to avoid no space left on device
+build-android-arm64-apk:
+    stage: build
+    script:
+        - mkdir Dockercontext
+        - >
+          docker build -t retroshare:android_arm64_latest
+          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
+          --build-arg ANDROID_PLATFORM_VER=21
+          --build-arg ANDROID_NDK_ARCH=arm64
+          --build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
+          --file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
+          Dockercontext
+        - >
+          docker cp
+          $(docker create --rm retroshare:android_arm64_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
+          $CI_PROJECT_DIR/RetroShare_Android_Service-arm64.apk
+    artifacts:
+      paths:
+        - RetroShare_Android_Service-arm64.apk

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "supportlibs/rapidjson"]
 	path = supportlibs/rapidjson
 	url = https://github.com/Tencent/rapidjson.git
+[submodule "supportlibs/libsam3"]
+	path = supportlibs/libsam3
+	url = https://github.com/i2p/libsam3.git

.travis.yml

@@ -8,10 +8,10 @@ language: cpp
 
 matrix:
   include:
-    - os: linux
+#    - os: linux
-      dist: bionic
+#      dist: bionic
-      sudo: required
+#      sudo: required
-      compiler: gcc
+#      compiler: gcc
     - os: osx
       osx_image: xcode10.2
       compiler: clang
@@ -68,11 +68,11 @@ before_script:
     fi
 
 script:
-  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j4; fi
+  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then travis_wait 30 make -j$(nproc); fi
 
 after_success:
   - if [ $TRAVIS_OS_NAME == osx ]; then build_scripts/OSX/travis_makeOSXPackage.sh ; fi
-  - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
+#  - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
 
 
 # branches:

README.asciidoc

@@ -9,9 +9,9 @@ RetroShare provides file sharing, chat, messages, forums, channels and more.
 
 .Build Status
 |===============================================================================
-|GNU/Linux (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
+|GNU/Linux, Android (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
-|GNU/Linux, macOS, (via Travis CI) | image:https://travis-ci.org/RetroShare/RetroShare.svg?branch=master[link="https://travis-ci.org/RetroShare/RetroShare"]
 |Windows (via AppVeyor) | image:https://ci.appveyor.com/api/projects/status/github/RetroShare/RetroShare?svg=true[link="https://ci.appveyor.com/project/RetroShare58622/retroshare"]
+|macOS, (via Travis CI) | image:https://app.travis-ci.com/RetroShare/RetroShare.svg?branch=master[link="https://app.travis-ci.com/github/RetroShare/RetroShare"]
 |===============================================================================
 
 

build_scripts/Android/Dockerfile

@@ -0,0 +1,115 @@
+## To prepare an image suitable as base for Gitlab CI use
+## image name must match gitlab repository name, you can play just with the tag
+## the part after :
+# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm_base"
+# docker build --squash -t "${CI_REGISTRY_IMAGE}" \
+# --build-arg QT_INSTALLER_JWT_TOKEN="your qt JWT token goes here"  .
+#
+# To build Android ARMv8 (64 bit) package pass also
+# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm64_base"
+# --build-arg ANDROID_NDK_ARCH=arm64 --build-arg ANDROID_PLATFORM_VER=21
+
+## --squash is very important in case of GitlabCI shared runners as they are
+## limited to 25GB disk size
+
+## To push it to gitlab CI registry you need first to login and the to push
+# docker login registry.gitlab.com
+# docker push ${CI_IMAGE_NAME}
+
+## To extract the generated APK easily you can run after the build complete
+# docker cp \
+#	$(docker create --rm ${CI_IMAGE_NAME}):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk \
+#	/tmp/RetroShare_Android_Service.apk
+
+
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get clean
+RUN apt-get install -y -qq \
+	bash build-essential bzip2 cmake curl chrpath doxygen \
+	git p7zip python qt5-default qttools5-dev tclsh unzip wget zip
+
+# Dependencies to create Android pkg
+RUN apt-get install -y -qq \
+	openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle
+
+ARG FRESHCLONE=0
+ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
+ARG REPO_BRANCH=master
+ARG REPO_DEPTH="--depth 2000"
+
+RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
+	git fetch --tags
+
+ENV PREPARE_TOOLCHAIN="/RetroShare/build_scripts/Android/prepare-toolchain-clang.sh"
+ENV NATIVE_LIBS_TOOLCHAIN_PATH="/android-toolchain/"
+
+ARG ANDROID_PLATFORM_VER=16
+ARG ANDROID_NDK_ARCH=arm
+
+ENV ANDROID_SDK_PATH="/opt/android-sdk"
+ENV ANDROID_HOME="$ANDROID_SDK_PATH"
+ENV ANDROID_SDK_ROOT="$ANDROID_SDK_PATH"
+
+ENV ANDROID_NDK_PATH="/opt/android-ndk"
+ENV ANDROID_NDK_ROOT="$ANDROID_NDK_PATH"
+
+ENV PATH="$PATH:$ANDROID_HOME/tools"
+ENV PATH="$PATH:$ANDROID_HOME/platform-tools"
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
+RUN mkdir /bCache
+WORKDIR /bCache
+
+## Quick HACK to ease development
+#COPY prepare-toolchain-clang.sh $PREPARE_TOOLCHAIN
+
+RUN $PREPARE_TOOLCHAIN install_android_sdk
+RUN $PREPARE_TOOLCHAIN install_android_ndk
+RUN $PREPARE_TOOLCHAIN
+
+ARG QT_INSTALLER_JWT_TOKEN
+RUN $PREPARE_TOOLCHAIN install_qt_android
+# Avoid Qt account details leak into the image
+RUN rm -f /root/.local/share/Qt/qtaccount.ini
+# Shrink image by removing unneded Qt components
+RUN rm -r \
+	$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Docs/ \
+	$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Examples/ \
+	$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Tools/ 
+
+RUN mkdir /jsonapi-generator-build
+WORKDIR /jsonapi-generator-build/
+RUN qmake ../RetroShare/jsonapi-generator/src/ \
+		CONFIG+=no_retroshare_plugins \
+		CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
+		CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
+	make -j$(nproc)
+
+RUN mkdir /retroshare-service-android-build
+WORKDIR /retroshare-service-android-build
+ARG RS_SERVICE_QMAKE_EXTRA_OPTS
+RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
+	-spec android-clang \
+	CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
+	RS_UPNP_LIB=miniupnpc \
+	JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
+	NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
+	CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
+	CONFIG+=no_rs_service_terminal_login \
+	CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
+	$RS_SERVICE_QMAKE_EXTRA_OPTS
+RUN	make -j$(nproc)
+RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
+RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
+	--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
+	--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
+	--jdk $JAVA_HOME --gradle
+
+
+RUN rm -rf /bCache
+
+# Clean apt cache
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*

build_scripts/Android/README.asciidoc

@@ -81,6 +81,7 @@ JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
 NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
 CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
 CONFIG+=no_rs_service_terminal_login
+CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3
 -------------------------------------------------------------------------------
 
 TIP: Some versions of QtCreator try to find the Android SDK in

build_scripts/Android/prepare-toolchain-clang.sh

@@ -2,7 +2,8 @@
 
 # Script to prepare RetroShare Android package building toolchain
 #
-# Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the
@@ -38,6 +39,15 @@ define_default_value ANDROID_PLATFORM_VER "16"
 define_default_value NATIVE_LIBS_TOOLCHAIN_PATH "${HOME}/Builds/android-toolchains/retroshare-android-${ANDROID_PLATFORM_VER}-${ANDROID_NDK_ARCH}/"
 define_default_value HOST_NUM_CPU $(nproc)
 
+define_default_value ANDROID_SDK_INSTALL "false"
+define_default_value ANDROID_SDK_TOOLS_VERSION "3859397"
+define_default_value ANDROID_SDK_TOOLS_SHA256 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
+define_default_value ANDROID_SDK_VERSION "29.0.3"
+
+define_default_value ANDROID_NDK_INSTALL "false"
+define_default_value ANDROID_NDK_VERSION "r21"
+define_default_value ANDROID_NDK_SHA256 b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b
+
 define_default_value BZIP2_SOURCE_VERSION "1.0.6"
 define_default_value BZIP2_SOURCE_SHA256 a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
 
@@ -48,16 +58,22 @@ define_default_value SQLITE_SOURCE_YEAR "2018"
 define_default_value SQLITE_SOURCE_VERSION "3250200"
 define_default_value SQLITE_SOURCE_SHA256 da9a1484423d524d3ac793af518cdf870c8255d209e369bd6a193e9f9d0e3181
 
-define_default_value SQLCIPHER_SOURCE_VERSION "4.2.0"
+define_default_value SQLCIPHER_SOURCE_VERSION "4.4.3"
-define_default_value SQLCIPHER_SOURCE_SHA256 105c1b813f848da038c03647a8bfc9d42fb46865e6aaf4edfd46ff3b18cdccfc
+define_default_value SQLCIPHER_SOURCE_SHA256 b8df69b998c042ce7f8a99f07cf11f45dfebe51110ef92de95f1728358853133
 
 define_default_value LIBUPNP_SOURCE_VERSION "1.8.4"
 define_default_value LIBUPNP_SOURCE_SHA256 976c3e4555604cdd8391ed2f359c08c9dead3b6bf131c24ce78e64d6669af2ed
 
-define_default_value INSTALL_QT_ANDROID "false"
+define_default_value QT_ANDROID_VIA_INSTALLER "false"
-define_default_value QT_VERSION "5.12.0"
+define_default_value QT_VERSION "5.12.11"
+define_default_value QT_INSTALLER_VERSION "4.1.1"
+define_default_value QT_INSTALLER_SHA256 1266ffd0d1b0e466244e3bc8422975c1aa9d96745b6bb28d422f7f92df11f34c
+define_default_value QT_INSTALLER_JWT_TOKEN ""
+define_default_value QT_INSTALL_PATH "${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt/"
+
 define_default_value QT_ANDROID_INSTALLER_SHA256 a214084e2295c9a9f8727e8a0131c37255bf724bfc69e80f7012ba3abeb1f763
 
+define_default_value RESTBED_SOURCE_REPO "https://github.com/Corvusoft/restbed.git"
 define_default_value RESTBED_SOURCE_VERSION f74f9329dac82e662c1d570b7cd72c192b729eb4
 
 define_default_value UDP_DISCOVERY_CPP_SOURCE "https://github.com/truvorskameikin/udp-discovery-cpp.git"
@@ -72,6 +88,64 @@ define_default_value RAPIDJSON_SOURCE_SHA256 bf7ced29704a1e696fbccf2a2b4ea068e77
 define_default_value MINIUPNPC_SOURCE_VERSION "2.1.20190625"
 define_default_value MINIUPNPC_SOURCE_SHA256 8723f5d7fd7970de23635547700878cd29a5c2bb708b5e5475b2d1d2510317fb
 
+# zlib and libpng versions walks toghether
+define_default_value ZLIB_SOURCE_VERSION "1.2.11"
+define_default_value ZLIB_SOURCE_SHA256 4ff941449631ace0d4d203e3483be9dbc9da454084111f97ea0a2114e19bf066
+
+define_default_value LIBPNG_SOURCE_VERSION "1.6.37"
+define_default_value LIBPNG_SOURCE_SHA256 505e70834d35383537b6491e7ae8641f1a4bed1876dbfe361201fc80868d88ca
+
+define_default_value LIBJPEG_SOURCE_VERSION "9d"
+define_default_value LIBJPEG_SOURCE_SHA256 6c434a3be59f8f62425b2e3c077e785c9ce30ee5874ea1c270e843f273ba71ee
+
+define_default_value TIFF_SOURCE_VERSION "4.2.0"
+define_default_value TIFF_SOURCE_SHA256 eb0484e568ead8fa23b513e9b0041df7e327f4ee2d22db5a533929dfc19633cb
+
+define_default_value CIMG_SOURCE_VERSION "2.9.7"
+define_default_value CIMG_SOURCE_SHA256 595dda9718431a123b418fa0db88e248c44590d47d9b1646970fa0503e27fa5c
+
+define_default_value PHASH_SOURCE_REPO "https://gitlab.com/g10h4ck/pHash.git"
+define_default_value PHASH_SOURCE_VERSION origin/android-ndk
+
+define_default_value MVPTREE_SOURCE_REPO "https://github.com/starkdg/mvptree.git"
+define_default_value MVPTREE_SOURCE_VERSION origin/master
+
+define_default_value REPORT_DIR "$(pwd)/$(basename ${NATIVE_LIBS_TOOLCHAIN_PATH})_build_report/"
+
+cArch=""
+eABI=""
+cmakeABI=""
+
+case "${ANDROID_NDK_ARCH}" in
+"arm")
+	cArch="${ANDROID_NDK_ARCH}"
+	eABI="eabi"
+	;;
+"arm64")
+	cArch="aarch64"
+	eABI=""
+	;;
+"x86")
+	cArch="i686"
+	eABI=""
+	;;
+"x86_64")
+	echo "ANDROID_NDK_ARCH=${ANDROID_NDK_ARCH} not supported yet"
+	exit -1
+	cArch="??"
+	eABI=""
+esac
+
+export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
+export PREFIX="${SYSROOT}/usr/"
+export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
+export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
+export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
+export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
+
+# Used to instruct cmake to explicitely ignore host libraries
+export HOST_IGNORE_PREFIX="/usr/"
+
 
 ## $1 filename, $2 sha256 hash
 function check_sha256()
@@ -104,159 +178,269 @@ function verified_download()
 	}
 }
 
-cArch=""
+# This function is the result of reading and testing many many stuff be very
-eABI=""
+# careful editing it
+function andro_cmake()
+{
+# Using android.toolchain.cmake as documented here
+# https://developer.android.com/ndk/guides/cmake seens to break more things then
+# it fixes :-\
 
+	cmakeProc=""
 	case "${ANDROID_NDK_ARCH}" in
 	"arm")
-	cArch="${ANDROID_NDK_ARCH}"
+		cmakeProc="armv7-a"
-	eABI="eabi"
 	;;
 	"arm64")
-	cArch="aarch64"
+		cmakeProc="aarch64"
-	eABI=""
 	;;
 	"x86")
-	cArch="i686"
+		cmakeProc="i686"
-	eABI=""
+	;;
+	"x86_64")
+		cmakeProc="x86_64"
+	;;
+	*)
+		echo "Unhandled NDK architecture ${ANDROID_NDK_ARCH}"
+		exit -1
 	;;
 	esac
 
-export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
+	_hi="$HOST_IGNORE_PREFIX"
-export PREFIX="${SYSROOT}/usr/"
-export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
-export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
-export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
-export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
 
+	cmake \
+		-DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \
+		-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+		-DCMAKE_PREFIX_PATH="${PREFIX}" \
+		-DCMAKE_SYSTEM_PREFIX_PATH="${PREFIX}" \
+		-DCMAKE_INCLUDE_PATH="${PREFIX}/include" \
+		-DCMAKE_SYSTEM_INCLUDE_PATH="${PREFIX}/include" \
+		-DCMAKE_LIBRARY_PATH="${PREFIX}/lib" \
+		-DCMAKE_SYSTEM_LIBRARY_PATH="${PREFIX}/lib" \
+		-DCMAKE_INSTALL_PREFIX="${PREFIX}" \
+		-DCMAKE_IGNORE_PATH="$_hi/include;$_hi/lib;$_hi/lib64" \
+		$@
+
+	# It is probably ok to do not touch CMAKE_PROGRAM_PATH and
+	# CMAKE_SYSTEM_PROGRAM_PATH
+}
+
+function git_source_get()
+{
+	sourceDir="$1" ; shift #$1
+	sourceRepo="$1"  ; shift  #$2
+	sourceVersion="$1"  ; shift  #$3
+	# extra paramethers are treated as submodules
+
+	[ -d "$sourceDir" ] &&
+	{
+		pushd "$sourceDir"
+		actUrl="$(git remote get-url origin)"
+		[ "$actUrl" != "$sourceRepo" ] && rm -rf "${sourceDir}"
+		popd
+	} || true
+
+	[ -d $sourceDir ] || git clone "$sourceRepo" "$sourceDir"
+	pushd $sourceDir
+	
+	git fetch --all
+	git reset --hard ${sourceVersion}
+
+	while [ "$1" != "" ] ; do
+		git submodule update --init "$1"
+		pushd "$1"
+		git reset --hard
+		shift
+		popd
+	done
+
+	popd
+}
+
+declare -A TASK_REGISTER
+
+function task_register()
+{
+	TASK_REGISTER[$1]=true
+}
+
+function task_unregister()
+{
+	# we may simply wipe them but we could benefit from keeping track of
+	# unregistered tasks too
+	TASK_REGISTER[$1]=false
+}
+
+function task_logfile()
+{
+	echo "$REPORT_DIR/$1.log"
+}
+
+function task_run()
+{
+	mTask="$1" ; shift
+
+	[ "${TASK_REGISTER[$mTask]}" != "true" ] &&
+	{
+		echo "Attempt to run not registered task $mTask $@"
+		return -1
+	}
+
+	logFile="$(task_logfile $mTask)"
+	if [ -f "$logFile" ] ; then
+		echo "Task $mTask already run more details at $logFile"
+	else
+		date | tee > "$logFile"
+		$mTask $@ |& tee --append "$logFile"
+		mRetval="${PIPESTATUS[0]}"
+		echo "Task $mTask return ${mRetval} more details at $logFile"
+		date | tee --append "$logFile"
+		return ${mRetval}
+	fi
+}
+
+function task_zap()
+{
+	rm -f "$(task_logfile $1)"
+}
+
+DUPLICATED_INCLUDES_LIST_FILE="${REPORT_DIR}/duplicated_includes_list"
+DUPLICATED_INCLUDES_DIR="${REPORT_DIR}/duplicated_includes/"
+
+task_register install_android_sdk
+install_android_sdk()
+{
+	tFile="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
+
+	verified_download "${tFile}" "${ANDROID_SDK_TOOLS_SHA256}" \
+		"https://dl.google.com/android/repository/${tFile}"
+
+	unzip "${tFile}"
+	mkdir -p "$ANDROID_SDK_PATH"
+	rm -rf "$ANDROID_SDK_PATH/tools/"
+	mv --verbose tools/ "$ANDROID_SDK_PATH/tools/"
+
+	# Install Android SDK
+	yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses && \
+		$ANDROID_SDK_PATH/tools/bin/sdkmanager --update
+	$ANDROID_SDK_PATH/tools/bin/sdkmanager "platforms;android-$ANDROID_PLATFORM_VER"
+	$ANDROID_SDK_PATH/tools/bin/sdkmanager "build-tools;$ANDROID_SDK_VERSION"
+}
+
+task_register install_android_ndk
+install_android_ndk()
+{
+	tFile="android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
+
+	verified_download "${tFile}" "${ANDROID_NDK_SHA256}" \
+		"https://dl.google.com/android/repository/${tFile}"
+
+	unzip "${tFile}"
+	mkdir -p "$ANDROID_NDK_PATH"
+	rm -rf "$ANDROID_NDK_PATH"
+	mv --verbose  "android-ndk-${ANDROID_NDK_VERSION}/" "$ANDROID_NDK_PATH/"
+}
 
 ## More information available at https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html
-build_toolchain()
+task_register bootstrap_toolchain
+bootstrap_toolchain()
 {
-	echo "build_toolchain()
+	rm -rf "${NATIVE_LIBS_TOOLCHAIN_PATH}"
-################################################################################
-################################################################################
-################################################################################
-"
-
-	rm -rf ${NATIVE_LIBS_TOOLCHAIN_PATH}
 	${ANDROID_NDK_PATH}/build/tools/make_standalone_toolchain.py --verbose \
 		--arch ${ANDROID_NDK_ARCH} --install-dir ${NATIVE_LIBS_TOOLCHAIN_PATH} \
 		--api ${ANDROID_PLATFORM_VER}
-	find "${PREFIX}/include/" -not -type d > "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles"
+
+	# Avoid problems with arm64 some libraries installing on lib64
+	ln -s "${PREFIX}/lib/" "${PREFIX}/lib64"
+
+	pushd "${PREFIX}/include/"
+	find . -not -type d > "${DUPLICATED_INCLUDES_LIST_FILE}"
+	popd
 }
 
 ## This avoid <cmath> include errors due to -isystem and -I ordering issue
-delete_copied_includes()
+task_register deduplicate_includes
+deduplicate_includes()
 {
-	echo "delete_copied_includes()
+	while read delFile ; do
-################################################################################
+		mNewPath="${DUPLICATED_INCLUDES_DIR}/$delFile"
-################################################################################
+		mkdir --verbose --parents "$(dirname "$mNewPath")"
-################################################################################
+		mv --verbose "${PREFIX}/include/$delFile" "$mNewPath"
-"
+	done < "${DUPLICATED_INCLUDES_LIST_FILE}"
-	cat "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles" | while read delFile ; do
-		rm "$delFile"
-	done
 }
 
-## More information available at https://gitlab.com/relan/provisioners/merge_requests/1 and http://stackoverflow.com/a/34032216
+task_register reduplicate_includes
+reduplicate_includes()
+{
+	pushd "${DUPLICATED_INCLUDES_DIR}"
+	find . -not -type d | while read delFile ; do
+		mv --verbose "${delFile}" "${PREFIX}/include/$delFile"
+	done
+	popd
+}
+
+# $1 optional prefix prepended only if return value is not empty
+# $2 optional suffix appended only if return value is not empty
+task_register get_qt_arch
+get_qt_arch()
+{
+	local QT_VERSION_COMP="$(echo $QT_VERSION | awk -F. '{print $1*1000000+$2*1000+$3}')" 
+	local QT_ARCH=""
+
+	# Qt >= 5.15.0 ships all Android architectures toghether
+	[ "$QT_VERSION_COMP" -lt "5015000" ] &&
+	{
+		case "${ANDROID_NDK_ARCH}" in
+		"arm")
+			QT_ARCH="armv7"
+			;;
+		"arm64")
+			QT_ARCH="arm64_v8a"
+			;;
+		"x86")
+			QT_ARCH="x86"
+			;;
+		esac
+
+		echo "$1$QT_ARCH$2"
+	}
+}
+
+task_register get_qt_dir
+get_qt_dir()
+{
+	echo "${QT_INSTALL_PATH}/${QT_VERSION}/android$(get_qt_arch _)/"
+}
+
+## More information available at https://wiki.qt.io/Online_Installer_4.x
+task_register install_qt_android
 install_qt_android()
 {
-	echo "install_qt_android()
+	[ "$QT_INSTALLER_JWT_TOKEN" == "" ] &&
-################################################################################
+	{
-################################################################################
+		echo "To run Qt installer QT_INSTALLER_JWT_TOKEN environement variable \
-################################################################################
+need to be set to a valid JWT token see https://wiki.qt.io/Online_Installer_4.x"
-"
+		return -1
+	}
 
-	QT_VERSION_CODE=$(echo $QT_VERSION | tr -d .)
+	QT_VERSION_CODE="$(echo $QT_VERSION | tr -d .)"
-	QT_INSTALL_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt
+	QT_INSTALLER="qt-unified-linux-x86_64-${QT_INSTALLER_VERSION}-online.run"
-	QT_INSTALLER="qt-unified-linux-x64-3.0.2-online.run"
+	tMajDotMinVer="$(echo $QT_INSTALLER_VERSION | awk -F. '{print $1"."$2}')"
-
+	verified_download $QT_INSTALLER $QT_INSTALLER_SHA256 \
-	verified_download $QT_INSTALLER $QT_ANDROID_INSTALLER_SHA256 \
+		"https://master.qt.io/archive/online_installers/${tMajDotMinVer}/${QT_INSTALLER}"
-		http://master.qt.io/archive/online_installers/3.0/${QT_INSTALLER}
 
 	chmod a+x ${QT_INSTALLER}
-
+	QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} \
-	QT_INSTALLER_SCRIPT="qt_installer_script.js"
+		install qt.qt5.${QT_VERSION_CODE}.android$(get_qt_arch _) \
-	cat << EOF > "${QT_INSTALLER_SCRIPT}"
+		--accept-licenses --accept-obligations --confirm-command \
-function Controller() {
+		--default-answer --no-default-installations \
-    installer.autoRejectMessageBoxes();
+		--root "${QT_INSTALL_PATH}"
-    installer.installationFinished.connect(function() {
-        gui.clickButton(buttons.NextButton);
-    });
-
-    var welcomePage = gui.pageWidgetByObjectName("WelcomePage");
-    welcomePage.completeChanged.connect(function() {
-        if (gui.currentPageWidget().objectName == welcomePage.objectName)
-            gui.clickButton(buttons.NextButton);
-    });
-}
-
-Controller.prototype.WelcomePageCallback = function() {
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.CredentialsPageCallback = function() {
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.IntroductionPageCallback = function() {
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.TargetDirectoryPageCallback = function() {
-    gui.currentPageWidget().TargetDirectoryLineEdit.setText("$QT_INSTALL_PATH");
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.ComponentSelectionPageCallback = function() {
-    var widget = gui.currentPageWidget();
-
-    // You can get these component names by running the installer with the
-    // --verbose flag. It will then print out a resource tree.
-
-    widget.deselectComponent("qt.tools.qtcreator");
-    widget.deselectComponent("qt.tools.doc");
-    widget.deselectComponent("qt.tools.examples");
-
-    widget.selectComponent("qt.$QT_VERSION_CODE.android_armv7");
-
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.LicenseAgreementPageCallback = function() {
-    gui.currentPageWidget().AcceptLicenseRadioButton.setChecked(true);
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.StartMenuDirectoryPageCallback = function() {
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.ReadyForInstallationPageCallback = function() {
-    gui.clickButton(buttons.NextButton);
-}
-
-Controller.prototype.FinishedPageCallback = function() {
-    var checkBoxForm = gui.currentPageWidget().LaunchQtCreatorCheckBoxForm;
-    if (checkBoxForm && checkBoxForm.launchQtCreatorCheckBox)
-        checkBoxForm.launchQtCreatorCheckBox.checked = false;
-    gui.clickButton(buttons.FinishButton);
-}
-EOF
-
-QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} --script ${QT_INSTALLER_SCRIPT}
 }
 
 ## More information available at retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136
+task_register build_bzlib
 build_bzlib()
 {
-	echo "build_bzlib()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="bzip2-${BZIP2_SOURCE_VERSION}"
 	rm -rf $B_dir
 
@@ -279,14 +463,9 @@ build_bzlib()
 }
 
 ## More information available at http://doc.qt.io/qt-5/opensslsupport.html
+task_register build_openssl
 build_openssl()
 {
-	echo "build_openssl()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="openssl-${OPENSSL_SOURCE_VERSION}"
 	rm -rf $B_dir
 
@@ -308,6 +487,10 @@ build_openssl()
 		--openssldir="${SYSROOT}/etc/ssl"
 #	sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile
 #	sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile
+
+	# Avoid documentation build which is unneded and time consuming
+	echo "exit 0; " > util/process_docs.pl
+	
 	make -j${HOST_NUM_CPU}
 	make install
 	rm -f ${PREFIX}/lib/libssl.so*
@@ -315,14 +498,9 @@ build_openssl()
 	cd ..
 }
 
+task_register build_sqlite
 build_sqlite()
 {
-	echo "build_sqlite()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="sqlite-autoconf-${SQLITE_SOURCE_VERSION}"
 	rm -rf $B_dir
 
@@ -335,18 +513,13 @@ build_sqlite()
 	make -j${HOST_NUM_CPU}
 	make install
 	rm -f ${PREFIX}/lib/libsqlite3.so*
-#	${CC} -shared -o libsqlite3.so -fPIC sqlite3.o -ldl
-#	cp libsqlite3.so "${SYSROOT}/usr/lib"
 	cd ..
 }
 
+task_register build_sqlcipher
 build_sqlcipher()
 {
-	echo "build_sqlcipher()
+	task_run build_sqlite
-################################################################################
-################################################################################
-################################################################################
-"
 
 	B_dir="sqlcipher-${SQLCIPHER_SOURCE_VERSION}"
 	rm -rf $B_dir
@@ -358,14 +531,14 @@ build_sqlcipher()
 
 	tar -xf $T_file
 	cd $B_dir
-	case "${ANDROID_NDK_ARCH}" in
+#	case "${ANDROID_NDK_ARCH}" in
-	"arm64")
+#	"arm64")
-	# SQLCipher config.sub is outdated and doesn't recognize newer architectures
+#	# SQLCipher config.sub is outdated and doesn't recognize newer architectures
-		rm config.sub
+#		rm config.sub
-		autoreconf --verbose --install --force
+#		autoreconf --verbose --install --force
-		automake --add-missing --copy --force-missing
+#		automake --add-missing --copy --force-missing
-	;;
+#	;;
-	esac
+#	esac
 	./configure --with-pic --build=$(sh ./config.guess) \
 		--host=${cArch}-linux \
 		--prefix="${PREFIX}" --with-sysroot="${SYSROOT}" \
@@ -377,14 +550,9 @@ build_sqlcipher()
 	cd ..
 }
 
+task_register build_libupnp
 build_libupnp()
 {
-	echo "build_libupnp()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="pupnp-release-${LIBUPNP_SOURCE_VERSION}"
 	B_ext=".tar.gz"
 	B_file="${B_dir}${B_ext}"
@@ -408,14 +576,9 @@ build_libupnp()
 	cd ..
 }
 
+task_register build_rapidjson
 build_rapidjson()
 {
-	echo "build_rapidjson()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="rapidjson-${RAPIDJSON_SOURCE_VERSION}"
 	D_file="${B_dir}.tar.gz"
 	verified_download $D_file $RAPIDJSON_SOURCE_SHA256 \
@@ -424,42 +587,25 @@ build_rapidjson()
 	cp -r "${B_dir}/include/rapidjson/" "${PREFIX}/include/rapidjson"
 }
 
+task_register build_restbed
 build_restbed()
 {
-	echo "build_restbed()
+	S_dir="restbed"
-################################################################################
+	B_dir="${S_dir}-build"
-################################################################################
+	git_source_get "$S_dir" "$RESTBED_SOURCE_REPO" "${RESTBED_SOURCE_VERSION}" \
-################################################################################
+		"dependency/asio" "dependency/catch"
-"
 
-	[ -d restbed ] || git clone --depth=2000 https://github.com/Corvusoft/restbed.git
+	rm -rf "$B_dir"; mkdir "$B_dir"
-	cd restbed
+	pushd "$B_dir"
-#	git fetch --tags
+	andro_cmake -DBUILD_TESTS=OFF -DBUILD_SSL=OFF -B. -H../${S_dir}
-#	git checkout tags/${RESTBED_SOURCE_VERSION}
-	git checkout ${RESTBED_SOURCE_VERSION}
-	git submodule update --init dependency/asio
-	git submodule update --init dependency/catch
-	git submodule update --init dependency/kashmir
-	cd ..
-
-	rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
-	cmake \
-		-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
-		-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
 	make -j${HOST_NUM_CPU}
 	make install
-	cp "${PREFIX}/library/librestbed.a" "${PREFIX}/lib/"
+	popd
-	cd ..
 }
 
+task_register build_udp-discovery-cpp
 build_udp-discovery-cpp()
 {
-	echo "build_udp-discovery-cpp()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	S_dir="udp-discovery-cpp"
 	[ -d $S_dir ] || git clone $UDP_DISCOVERY_CPP_SOURCE $S_dir
 	cd $S_dir
@@ -468,23 +614,16 @@ build_udp-discovery-cpp()
 
 	B_dir="udp-discovery-cpp-build"
 	rm -rf ${B_dir}; mkdir ${B_dir}; cd ${B_dir}
-	cmake \
+	andro_cmake -B. -H../$S_dir
-		-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-		-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../udp-discovery-cpp
 	make -j${HOST_NUM_CPU}
 	cp libudp-discovery.a "${PREFIX}/lib/"
 	cp ../$S_dir/*.hpp "${PREFIX}/include/"
 	cd ..
 }
 
+task_register build_xapian
 build_xapian()
 {
-	echo "build_xapian()
-################################################################################
-################################################################################
-################################################################################
-"
-
 	B_dir="xapian-core-${XAPIAN_SOURCE_VERSION}"
 	D_file="$B_dir.tar.xz"
 	verified_download $D_file $XAPIAN_SOURCE_SHA256 \
@@ -503,15 +642,12 @@ build_xapian()
 		--prefix="${PREFIX}" --with-sysroot="${SYSROOT}"
 	make -j${HOST_NUM_CPU}
 	make install
+	cd ..
 }
 
+task_register build_miniupnpc
 build_miniupnpc()
 {
-	echo "build_miniupnpc()
-################################################################################
-################################################################################
-################################################################################
-"
 	S_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}"
 	B_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}-build"
 	D_file="$S_dir.tar.gz"
@@ -521,29 +657,228 @@ build_miniupnpc()
 	tar -xf $D_file
 	mkdir $B_dir
 	cd $B_dir
-	cmake \
+	andro_cmake \
 		-DUPNPC_BUILD_STATIC=TRUE \
 		-DUPNPC_BUILD_SHARED=FALSE \
 		-DUPNPC_BUILD_TESTS=FALSE \
 		-DUPNPC_BUILD_SAMPLE=FALSE \
-		-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+		-B. -S../$S_dir
-		-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -S../$S_dir
 	make -j${HOST_NUM_CPU}
 	make install
+	cd ..
 }
 
-build_toolchain
+task_register build_zlib
-[ "${INSTALL_QT_ANDROID}X" != "trueX" ] || install_qt_android
+build_zlib()
-build_bzlib
+{
-build_openssl
+	S_dir="zlib-${ZLIB_SOURCE_VERSION}"
-build_sqlite
+	B_dir="zlib-${ZLIB_SOURCE_VERSION}-build"
-build_sqlcipher
+	D_file="$S_dir.tar.xz"
-build_libupnp
+	verified_download $D_file $ZLIB_SOURCE_SHA256 \
-build_rapidjson
+		http://www.zlib.net/${D_file}
-build_restbed
+	rm -rf $S_dir $B_dir
-build_udp-discovery-cpp
+	tar -xf $D_file
-build_xapian
+	mkdir $B_dir
-build_miniupnpc
+	cd $B_dir
-delete_copied_includes
+	andro_cmake -B. -S../$S_dir
+	make -j${HOST_NUM_CPU}
+	make install
+	rm -f ${PREFIX}/lib/libz.so*
+	cd ..
+}
 
-echo NATIVE_LIBS_TOOLCHAIN_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}
+task_register build_libpng
+build_libpng()
+{
+	task_run build_zlib
+
+	S_dir="libpng-${LIBPNG_SOURCE_VERSION}"
+	B_dir="libpng-${LIBPNG_SOURCE_VERSION}-build"
+	D_file="$S_dir.tar.xz"
+	verified_download $D_file $LIBPNG_SOURCE_SHA256 \
+		https://download.sourceforge.net/libpng/${D_file}
+	rm -rf $S_dir $B_dir
+	tar -xf $D_file
+
+	# libm is part of bionic An android
+	sed -i -e 's/find_library(M_LIBRARY m)/set(M_LIBRARY "")/' $S_dir/CMakeLists.txt
+	
+	# Disable hardware acceleration as they are problematic for Android
+	# compilation and are not supported by all phones, it is necessary to fiddle
+	# with CMakeLists.txt as libpng 1.6.37 passing it as cmake options seems not
+	# working properly
+	# https://github.com/imagemin/optipng-bin/issues/97
+	# https://github.com/opencv/opencv/issues/7600
+	echo "add_definitions(-DPNG_ARM_NEON_OPT=0)" >> $S_dir/CMakeLists.txt
+
+	mkdir $B_dir
+	pushd $B_dir
+
+	HW_OPT="OFF"
+#	[ "$ANDROID_PLATFORM_VER" -ge "22" ] && HW_OPT="ON"
+
+	andro_cmake \
+		-DPNG_SHARED=OFF \
+		-DPNG_STATIC=ON \
+		-DPNG_TESTS=OFF \
+		-DPNG_HARDWARE_OPTIMIZATIONS=$HW_OPT \
+		-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+		-B. -S../$S_dir
+	make -j${HOST_NUM_CPU}
+	make install
+	popd
+}
+
+task_register build_libjpeg
+build_libjpeg()
+{
+	S_dir="jpeg-${LIBJPEG_SOURCE_VERSION}"
+	D_file="jpegsrc.v${LIBJPEG_SOURCE_VERSION}.tar.gz"
+	verified_download $D_file $LIBJPEG_SOURCE_SHA256 \
+		https://www.ijg.org/files/$D_file
+	rm -rf $S_dir
+	tar -xf $D_file
+	cd $S_dir
+	./configure --with-pic --prefix="${PREFIX}" --host=${cArch}-linux
+	make -j${HOST_NUM_CPU}
+	make install
+	rm -f ${PREFIX}/lib/libjpeg.so*
+	cd ..
+}
+
+task_register build_tiff
+build_tiff()
+{
+	S_dir="tiff-${TIFF_SOURCE_VERSION}"
+	B_dir="${S_dir}-build"
+	D_file="tiff-${TIFF_SOURCE_VERSION}.tar.gz"
+
+	verified_download $D_file $TIFF_SOURCE_SHA256 \
+		https://download.osgeo.org/libtiff/${D_file}
+
+	rm -rf $S_dir $B_dir
+	tar -xf $D_file
+	mkdir $B_dir
+
+	# Disable tools building, not needed for retroshare, and depending on some
+	# OpenGL headers not available on Android
+	echo "" > $S_dir/tools/CMakeLists.txt
+
+	# Disable tests building, not needed for retroshare, and causing linker
+	# errors
+	echo "" > $S_dir/test/CMakeLists.txt
+	
+	# Disable extra tools building, not needed for retroshare, and causing
+	# linker errors
+	echo "" > $S_dir/contrib/CMakeLists.txt
+	
+	# Disable more unneded stuff 
+	echo "" > $S_dir/build/CMakeLists.txt
+	echo "" > $S_dir/html/CMakeLists.txt
+	echo "" > $S_dir/man/CMakeLists.txt
+	echo "" > $S_dir/port/CMakeLists.txt
+
+	# Change to static library build
+	sed -i 's\add_library(tiff\add_library(tiff STATIC\' \
+		$S_dir/libtiff/CMakeLists.txt
+
+	cd $B_dir
+	#TODO: build dependecies to support more formats
+	andro_cmake \
+		-Dlibdeflate=OFF -Djbig=OFF -Dlzma=OFF -Dzstd=OFF -Dwebp=OFF \
+		-Djpeg12=OFF \
+		-Dcxx=OFF \
+		-B. -S../$S_dir
+	make -j${HOST_NUM_CPU}
+	make install
+	cd ..
+}
+
+task_register build_cimg
+build_cimg()
+{
+	task_run build_libpng
+	task_run build_libjpeg
+	task_run build_tiff
+
+	S_dir="CImg-${CIMG_SOURCE_VERSION}"
+	D_file="CImg_${CIMG_SOURCE_VERSION}.zip"
+
+	verified_download $D_file $CIMG_SOURCE_SHA256 \
+		https://cimg.eu/files/${D_file}
+
+	unzip -o $D_file
+
+	cp --archive --verbose "$S_dir/CImg.h" "$PREFIX/include/"
+}
+
+task_register build_phash
+build_phash()
+{
+	task_run build_cimg
+
+	S_dir="pHash"
+	B_dir="${S_dir}-build"
+
+	git_source_get "$S_dir" "$PHASH_SOURCE_REPO" "${PHASH_SOURCE_VERSION}"
+
+	rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
+	andro_cmake -DPHASH_DYNAMIC=OFF -DPHASH_STATIC=ON  -B. -H../pHash
+	make -j${HOST_NUM_CPU}
+	make install
+	popd
+}
+
+task_register build_mvptree
+build_mvptree()
+{
+	S_dir="mvptree"
+	B_dir="${S_dir}-build"
+
+	git_source_get "$S_dir" "$MVPTREE_SOURCE_REPO" "${MVPTREE_SOURCE_VERSION}"
+	rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
+	andro_cmake -B. -H../${S_dir}
+	make -j${HOST_NUM_CPU}
+	make install
+	popd
+}
+
+task_register get_native_libs_toolchain_path
+get_native_libs_toolchain_path()
+{
+	echo ${NATIVE_LIBS_TOOLCHAIN_PATH}
+}
+
+task_register build_default_toolchain
+build_default_toolchain()
+{
+	task_run bootstrap_toolchain || return $?
+	task_run build_bzlib || return $?
+	task_run build_openssl || return $?
+	task_run build_sqlcipher || return $?
+	task_run build_rapidjson || return $?
+	task_run build_restbed || return $?
+	task_run build_udp-discovery-cpp || return $?
+	task_run build_xapian || return $?
+	task_run build_miniupnpc || return $?
+	task_run build_phash || return $?
+	task_run deduplicate_includes || return $?
+	task_run get_native_libs_toolchain_path || return $?
+}
+
+if [ "$1" == "" ]; then
+	rm -rf "$REPORT_DIR"
+	mkdir -p "$REPORT_DIR"
+	cat "$0" > "$REPORT_DIR/build_script"
+	env > "$REPORT_DIR/build_env"
+	build_default_toolchain
+else
+	# do not delete report directory in this case so we can reuse material
+	# produced by previous run, like deduplicated includes
+	mkdir -p "$REPORT_DIR"
+	while [ "$1" != "" ] ; do
+		task_zap $1
+		task_run $1 || exit $?
+		shift
+	done
+fi

build_scripts/GitlabCI/Android.Dockerfile

@@ -0,0 +1,46 @@
+ARG ANDROID_NDK_ARCH=arm
+FROM registry.gitlab.com/retroshare/retroshare:android_${ANDROID_NDK_ARCH}_base
+
+RUN apt-get update -y && apt-get upgrade -y
+
+ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
+ARG REPO_BRANCH=master
+
+WORKDIR /RetroShare
+RUN git remote add testing $REPO_URL && \
+	git fetch --tags testing $REPO_BRANCH && \
+	git reset --hard testing/$REPO_BRANCH && \
+	git --no-pager log --max-count 1
+
+RUN rm -rf /jsonapi-generator-build ; mkdir /jsonapi-generator-build
+WORKDIR /jsonapi-generator-build/
+RUN qmake ../RetroShare/jsonapi-generator/src/ \
+		CONFIG+=no_retroshare_plugins \
+		CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
+		CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
+	make -j$(nproc)
+
+RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
+WORKDIR /retroshare-service-android-build
+
+# ARG declared before FROM get wiped after it, so we need declaring it again
+ARG ANDROID_NDK_ARCH=arm
+ARG RS_SERVICE_QMAKE_EXTRA_OPTS
+RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
+	-spec android-clang \
+	CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
+	RS_UPNP_LIB=miniupnpc \
+	JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
+	NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
+	CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
+	CONFIG+=no_rs_service_terminal_login \
+	CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
+	$RS_SERVICE_QMAKE_EXTRA_OPTS
+RUN	make -j$(nproc)
+RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
+
+ARG ANDROID_PLATFORM_VER=16
+RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
+	--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
+	--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
+	--jdk $JAVA_HOME --gradle

build_scripts/GitlabCI/base.Dockerfile

@@ -2,42 +2,43 @@
 ## force cloning a new
 
 ## To prepare an image suitable as base for Gitlab CI use
-# docker build -t "${CI_REGISTRY_IMAGE}:base" --build-arg KEEP_SOURCE=true --build-arg REPO_DEPTH="" -f base.Dockerfile .
+# export CI_REGISTRY_IMAGE="registry.gitlab.com/retroshare/retroshare:base"
-
+# docker build -t "${CI_REGISTRY_IMAGE}" -f base.Dockerfile .
-## Now you need to tag it so you can later push it
-# docker tag ${ID_OF_THE_CREATED_IMAGE} registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
 
 ## To push it to gitlab CI registry you need first to login and the to push
 # docker login registry.gitlab.com
-# docker push registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
+# docker push "${CI_REGISTRY_IMAGE}"
-
 
 ## To run the container
-# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}:base" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
+# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
 
 FROM ubuntu
 
-ARG CACHEBUST=0
+ENV DEBIAN_FRONTEND=noninteractive
-RUN \
+
-	apt-get update -y && apt-get upgrade -y && \
+RUN apt-get update -y && apt-get upgrade -y -qq && \
-	apt-get install -y build-essential libssl-dev libbz2-dev libsqlite3-dev \
+	apt-get install -y -qq build-essential cimg-dev libssl-dev libbz2-dev \
+		libsqlite3-dev \
 		libsqlcipher-dev libupnp-dev pkg-config libz-dev \
 		qt5-default libxapian-dev qttools5-dev doxygen rapidjson-dev \
 		git cmake curl
 
+RUN git clone --depth 1 https://github.com/aetilius/pHash.git && \
+	rm -rf pHash-build && mkdir pHash-build && cd pHash-build && \
+	cmake -B. -H../pHash -DCMAKE_INSTALL_PREFIX=/usr && \
+	make -j$(nproc) && make install && cd .. && \
+	rm -rf pHash-build pHash
+
 ARG FRESHCLONE=0
 ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
 ARG REPO_BRANCH=master
 ARG REPO_DEPTH="--depth 2000"
-ARG KEEP_SOURCE=false
-RUN apt-get update -y && apt-get upgrade -y
 RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
 	git fetch --tags && cd ..
-RUN \
+RUN mkdir RetroShare-build && cd RetroShare-build && \
-	mkdir RetroShare-build && cd RetroShare-build && \
 	qmake ../RetroShare \
-		CONFIG+=no_retroshare_plugins CONFIG+=ipv6 \
+		CONFIG+=no_retroshare_plugins \
 		CONFIG+=retroshare_service CONFIG+=no_retroshare_gui \
 		CONFIG+=rs_jsonapi CONFIG+=rs_deep_search && \
 	(make -j$(nproc) || make -j$(nproc) || make) && make install && \
-	cd .. && rm -rf RetroShare-build && ($KEEP_SOURCE || rm -rf RetroShare)
+	cd .. && rm -rf RetroShare-build

build_scripts/GitlabCI/getRepoArgs.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+[ -n "$CI_MERGE_REQUEST_ID" ] &&
+	echo \
+		--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
+		--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
+	echo \
+		--build-arg REPO_URL="$CI_REPOSITORY_URL" \
+		--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

build_scripts/OBS

@@ -1 +1 @@
-Subproject commit 6d6ee4aab171c23c603f7e813ba66d22192393ad
+Subproject commit b260e58346b1eec782bdf88a7e8f3c9d36fd3ecb

build_scripts/Windows-msys2/build/pack.bat

@@ -147,8 +147,10 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
 	call :copy_dependencies "%%D" "%RsDeployPath%"
 )
 
+if exist "%SourcePath%\retroshare-gui\src\qss" (
 	echo copy qss
 	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+)
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

build_scripts/Windows-msys2/env/tools/prepare-msys2.bat

@@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	)
 )
 
-set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
+if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
-set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
+if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
+
+set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
+set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
 
 %cecho% info "Remove previous MSYS2 version"
 call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
@@ -41,12 +44,10 @@ if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download M
 set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
 
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
-call "%EnvRootPath%\update-msys2.bat"
-call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows-msys2/env/tools/root/update-msys2.bat

@@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
 goto :EOF
 
 :update
-set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
+set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
 
 echo Update MSYS2 %~1
-"%MSYSSH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYSSH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 :exit
 endlocal

build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi

@@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /r "${DEPLOYDIR}\qss\*.*"
+  File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link

build_scripts/Windows/build/pack.bat

@@ -132,8 +132,10 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
 copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
 del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
 
+if exist "%SourcePath%\retroshare-gui\src\qss" (
 	echo copy qss
 	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
+)
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

build_scripts/Windows/env/tools/prepare-msys2.bat

@@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	)
 )
 
-set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
+if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
-set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
+if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
+
+set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
+set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
 set CMakeInstall=cmake-3.19.0-win32-x86.zip
 set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.19.0/%CMakeInstall%
 set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
@@ -64,17 +67,11 @@ if "%FoundProfile%"=="0" (
 
 set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
 
-%cecho% info "Update keyring"
-"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
-"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
-
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
-call "%EnvRootPath%\update-msys2.bat"
-call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows/env/tools/root/update-msys2.bat

@@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
 goto :EOF
 
 :update
-set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
+set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
 
 echo Update MSYS2 %~1
-"%MSYSSH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYSSH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman --noconfirm -Su"
 
 :exit
 endlocal

build_scripts/Windows/installer/retroshare-Qt5.nsi

@@ -334,7 +334,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
+  File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link

libretroshare/src/chat/distributedchat.cc

@@ -1513,7 +1513,7 @@ ChatLobbyVirtualPeerId DistributedChatService::makeVirtualPeerId(ChatLobbyId lob
 }
 
 
-void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id) 
+bool DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
 {
 	RsStackMutex stack(mDistributedChatMtx); /********** STACK LOCKED MTX ******/
 
@@ -1525,10 +1525,12 @@ void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
 	if(it == _lobby_invites_queue.end())
 	{
 		std::cerr << " (EE) lobby invite not in cache!!" << std::endl;
-		return ;
+        return false;
 	}
 
 	_lobby_invites_queue.erase(it) ;
+
+    return true;
 }
 
 bool DistributedChatService::joinVisibleChatLobby(const ChatLobbyId& lobby_id,const RsGxsId& gxs_id)

libretroshare/src/chat/distributedchat.h

@@ -61,7 +61,7 @@ class DistributedChatService
 		void getChatLobbyList(std::list<ChatLobbyId>& clids) ;
 		bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& clinfo) ;
 		bool acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& identity) ;
-		void denyLobbyInvite(const ChatLobbyId& id) ;
+        bool denyLobbyInvite(const ChatLobbyId& id) ;
 		void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
 		void invitePeerToLobby(const ChatLobbyId&, const RsPeerId& peer_id,bool connexion_challenge = false) ;
 		void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;

libretroshare/src/deep_search/commonutils.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,54 +18,49 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <algorithm>
+#include <thread>
+
 #include "deep_search/commonutils.hpp"
 #include "util/stacktrace.h"
-#include "util/rsdebug.h"
+#include "util/rsthreads.h"
+#include "util/rsdebuglevel0.h"
+
+#ifndef XAPIAN_AT_LEAST
+/// Added in Xapian 1.4.2.
+#define XAPIAN_AT_LEAST(A,B,C) \
+	(XAPIAN_MAJOR_VERSION > (A) || \
+	(XAPIAN_MAJOR_VERSION == (A) && \
+	(XAPIAN_MINOR_VERSION > (B) || \
+	(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
+#endif
 
 namespace DeepSearch
 {
-
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags, int blockSize )
-{
-	try
-	{
-		std::unique_ptr<Xapian::WritableDatabase> dbPtr(
-		        new Xapian::WritableDatabase(path, flags, blockSize) );
-		return dbPtr;
-	}
-	catch(Xapian::DatabaseLockError)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-	catch(...)
-	{
-		RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
-		        << "deleting it might help without causing any harm: "
-		        << path << std::endl;
-		print_stacktrace();
-	}
-
-	return nullptr;
-}
-
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags )
 {
 	try
 	{
+#if XAPIAN_AT_LEAST(1,3,2)
 		std::unique_ptr<Xapian::Database> dbPtr(
 		        new Xapian::Database(path, flags) );
+#else
+		std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
+		if(flags)
+		{
+			RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
+			         "library version: ", XAPIAN_VERSION, " < 1.3.2" );
+		}
+#endif
 		return dbPtr;
 	}
-	catch(Xapian::DatabaseOpeningError e)
+	catch(Xapian::DatabaseOpeningError& e)
 	{
 		RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
 		          << ", probably nothing has been indexed yet." << std::endl;
 	}
-	catch(Xapian::DatabaseLockError)
+	catch(Xapian::DatabaseLockError&)
 	{
 		RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
 		        << path << std::endl;
@@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
 	return date;
 }
 
+StubbornWriteOpQueue::~StubbornWriteOpQueue()
+{
+	auto fErr = flush(0);
+	if(fErr)
+	{
+		RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
+		          " operations irreparably lost ", fErr );
+		print_stacktrace();
+	}
+}
+
+void StubbornWriteOpQueue::push(write_op op)
+{
+	RS_DBG4("");
+
+	{
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		mOpStore.push(op);
+	}
+
+	flush();
+}
+
+std::error_condition StubbornWriteOpQueue::flush(
+        rstime_t acceptDelay, rstime_t callTS )
+{
+	RS_DBG4("");
+
+	{
+		// Return without attempt to open the database if the queue is empty
+		std::unique_lock<std::mutex> lock(mQueueMutex);
+		if(mOpStore.empty()) return std::error_condition();
+	}
+
+	std::unique_ptr<Xapian::WritableDatabase> dbPtr;
+	try
+	{
+		dbPtr = std::make_unique<Xapian::WritableDatabase>(
+		            mDbPath, Xapian::DB_CREATE_OR_OPEN );
+	}
+	catch(Xapian::DatabaseLockError)
+	{
+		if(acceptDelay)
+		{
+			rstime_t tNow = time(nullptr);
+			rstime_t maxRemaining = tNow - (callTS + acceptDelay);
+			if(maxRemaining > 0)
+			{
+				std::chrono::milliseconds interval(
+				            std::max(rstime_t(50), maxRemaining*1000/5) );
+				RS_DBG3( "Cannot acquire database write lock, retrying in:",
+				         interval.count(), "ms" );
+				RsThread::async([this, acceptDelay, callTS, interval]()
+				{
+					std::this_thread::sleep_for(interval);
+					flush(acceptDelay, callTS);
+				});
+				return std::error_condition();
+			}
+			else
+			{
+				RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
+				return std::errc::timed_out;
+			}
+		}
+		else return std::errc::resource_unavailable_try_again;
+	}
+	catch(...)
+	{
+		RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
+		print_stacktrace();
+		return std::errc::io_error;
+	}
+
+	std::unique_lock<std::mutex> lock(mQueueMutex);
+	while(!mOpStore.empty())
+	{
+		auto op = mOpStore.front(); mOpStore.pop();
+		op(*dbPtr);
+	}
+	return std::error_condition();
+}
+
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
+{
+	if(rsHtmlDoc.empty()) return rsHtmlDoc;
+
+	const bool isPlainMsg =
+	        rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
+	if(isPlainMsg) return rsHtmlDoc;
+
+	auto oSize = rsHtmlDoc.size();
+	auto bodyTagBegin(rsHtmlDoc.find("<body"));
+	if(bodyTagBegin >= oSize) return rsHtmlDoc;
+
+	auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
+	if(bodyTagEnd >= oSize) return rsHtmlDoc;
+
+	std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
+
+	// strip also CSS inside <style></style>
+	oSize = retVal.size();
+	auto styleTagBegin(retVal.find("<style"));
+	if(styleTagBegin < oSize)
+	{
+		auto styleEnd(retVal.find("</style>", styleTagBegin));
+		if(styleEnd < oSize)
+			retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
+	}
+
+	std::string::size_type oPos;
+	std::string::size_type cPos;
+	int itCount = 0;
+	while((oPos = retVal.find("<")) < retVal.size())
+	{
+		if((cPos = retVal.find(">")) <= retVal.size())
+			retVal.erase(oPos, 1+cPos-oPos);
+		else break;
+
+		// Avoid infinite loop with crafty input
+		if(itCount > 1000)
+		{
+			RS_WARN( "Breaking stripping loop due to max allowed iterations ",
+			         "rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
+			break;
+		}
+		++itCount;
+	}
+
+	return retVal;
+}
+
 }

libretroshare/src/deep_search/commonutils.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -21,6 +21,9 @@
 
 #include <xapian.h>
 #include <memory>
+#include <functional>
+#include <queue>
+#include <mutex>
 
 #include "util/rstime.h"
 
@@ -33,13 +36,34 @@
 
 namespace DeepSearch
 {
-
+typedef std::function<void(Xapian::WritableDatabase&)> write_op;
-std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
-        const std::string& path, int flags = 0, int blockSize = 0 );
 
 std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
         const std::string& path, int flags = 0 );
 
 std::string timetToXapianDate(const rstime_t& time);
 
+std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
+
+struct StubbornWriteOpQueue
+{
+	explicit StubbornWriteOpQueue(const std::string& dbPath):
+	    mDbPath(dbPath) {}
+
+	~StubbornWriteOpQueue();
+
+	void push(write_op op);
+
+	std::error_condition flush(
+	        rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
+
+private:
+	std::queue<write_op> mOpStore;
+	rstime_t mLastFlush;
+
+	std::mutex mQueueMutex;
+
+	const std::string mDbPath;
+};
+
 }

libretroshare/src/deep_search/filesindex.cpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -18,48 +18,48 @@
  *                                                                             *
  *******************************************************************************/
 
-#include "deep_search/filesindex.hpp"
-#include "deep_search/commonutils.hpp"
-#include "util/rsdebug.h"
-#include "retroshare/rsinit.h"
-#include "retroshare/rsversion.h"
 
 #include <utility>
 
+#include "deep_search/filesindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "util/rsdebuglevel1.h"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsversion.h"
+
 /*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
 DeepFilesIndex::indexersRegister = {};
 
-bool DeepFilesIndex::indexFile(
+std::error_condition DeepFilesIndex::indexFile(
         const std::string& path, const std::string& name,
         const RsFileHash& hash )
 {
-	auto dbPtr = DeepSearch::openWritableDatabase(
-	            mDbPath, Xapian::DB_CREATE_OR_OPEN );
-	if(!dbPtr) return false;
-	Xapian::WritableDatabase& db(*dbPtr);
-
 	const std::string hashString = hash.toStdString();
 	const std::string idTerm("Q" + hashString);
 
-	Xapian::Document oldDoc;
+	auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
-	Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
+	if(db)
-	if( pIt != db.postlist_end(idTerm) )
 	{
-		oldDoc = db.get_document(*pIt);
+		Xapian::Document oldDoc;
+		Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
+		if( pIt != db->postlist_end(idTerm) )
+		{
+			oldDoc = db->get_document(*pIt);
 			if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
 			        RS_HUMAN_READABLE_VERSION &&
 			    std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
 			        indexersRegister.size() )
 			{
-			/* Looks like this file has already been indexed by this RetroShare
+				/* Looks like this file has already been indexed by this
-			 * exact version, so we can skip it. If the version was different it
+				 * RetroShare exact version, so we can skip it. If the version
-			 * made sense to reindex it as better indexers might be available
+				 * was different it made sense to reindex it as better indexers
-			 * since last time it was indexed */
+				 * might be available since last time it was indexed */
-			Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
+				RS_DBG3("skipping laready indexed file: ", hash, " ", name);
-			       << hash << " " << name << std::endl;
+				return std::error_condition();
-			return true;
 			}
 		}
+		db.reset(); // Release DB read lock ASAP
+	}
 
 	Xapian::Document doc;
 
@@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
 	doc.add_value(
 	            INDEXERS_COUNT_VALUENO,
 	            std::to_string(indexersRegister.size()) );
-	db.replace_document(idTerm, doc);
 
-	return true;
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); });
+
+	return std::error_condition();
 }
 
-bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
+std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 {
-	Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
+	RS_DBG3(hash);
-	       << hash << std::endl;
 
-	std::unique_ptr<Xapian::WritableDatabase> db =
+	mWriteQueue.push([hash](Xapian::WritableDatabase& db)
-	        DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
+	{ db.delete_document("Q" + hash.toStdString()); });
-	if(!db) return false;
 
-	db->delete_document("Q" + hash.toStdString());
+	return std::error_condition();
-	return true;
 }
 
 /*static*/ std::string DeepFilesIndex::dbDefaultPath()
@@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
 /*static*/ bool DeepFilesIndex::registerIndexer(
         int order, const DeepFilesIndex::IndexerFunType& indexerFun )
 {
-	Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
+	RS_DBG1(order);
 
 	indexersRegister.insert(std::make_pair(order, indexerFun));
 	return true;
 }
 
-uint32_t DeepFilesIndex::search(
+std::error_condition DeepFilesIndex::search(
         const std::string& queryStr,
         std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
 {
 	results.clear();
 
 	auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
-	if(!dbPtr) return 0;
+	if(!dbPtr) return std::errc::bad_file_descriptor;
 	Xapian::Database& db(*dbPtr);
 
 	// Set up a QueryParser with a stemmer and suitable prefixes.
@@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
 		results.push_back(s);
 	}
 
-	return static_cast<uint32_t>(results.size());
+	return std::error_condition();
 }
 
 

libretroshare/src/deep_search/filesindex.hpp

@@ -1,8 +1,8 @@
 /*******************************************************************************
  * RetroShare full text indexing and search implementation based on Xapian     *
  *                                                                             *
- * Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org>              *
- * Copyright (C) 2019  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License version 3 as    *
@@ -19,9 +19,6 @@
  *******************************************************************************/
 #pragma once
 
-#include "retroshare/rstypes.h"
-#include "util/rsdebug.h"
-
 #include <string>
 #include <cstdint>
 #include <vector>
@@ -29,6 +26,9 @@
 #include <map>
 #include <functional>
 
+#include "retroshare/rstypes.h"
+#include "deep_search/commonutils.hpp"
+
 struct DeepFilesSearchResult
 {
 	DeepFilesSearchResult() : mWeight(0) {}
@@ -41,7 +41,8 @@ struct DeepFilesSearchResult
 class DeepFilesIndex
 {
 public:
-	DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
+	explicit DeepFilesIndex(const std::string& dbPath):
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
 
 	/**
 	 * @brief Search indexed files
@@ -49,7 +50,7 @@ public:
 	 * no limits
 	 * @return search results count
 	 */
-	uint32_t search( const std::string& queryStr,
+	std::error_condition search( const std::string& queryStr,
 	                 std::vector<DeepFilesSearchResult>& results,
 	                 uint32_t maxResults = 100 );
 
@@ -57,7 +58,7 @@ public:
 	 * @return false if file could not be indexed because of error or
 	 *	unsupported type, true otherwise.
 	 */
-	bool indexFile(
+	std::error_condition indexFile(
 	        const std::string& path, const std::string& name,
 	        const RsFileHash& hash );
 
@@ -65,7 +66,7 @@ public:
 	 * @brief Remove file entry from database
 	 * @return false on error, true otherwise.
 	 */
-	bool removeFileFromIndex(const RsFileHash& hash);
+	std::error_condition removeFileFromIndex(const RsFileHash& hash);
 
 	static std::string dbDefaultPath();
 
@@ -96,8 +97,8 @@ private:
 
 	const std::string mDbPath;
 
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+
 	/** Storage for indexers function by order */
 	static std::multimap<int, IndexerFunType> indexersRegister;
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(1)
 };

libretroshare/src/deep_search/forumsindex.cpp

@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "deep_search/forumsindex.hpp"
+#include "deep_search/commonutils.hpp"
+#include "retroshare/rsinit.h"
+#include "retroshare/rsgxsforums.h"
+#include "util/rsdebuglevel4.h"
+
+std::error_condition DeepForumsIndex::search(
+        const std::string& queryStr,
+        std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
+{
+	results.clear();
+
+	std::unique_ptr<Xapian::Database> dbPtr(
+	            DeepSearch::openReadOnlyDatabase(mDbPath) );
+	if(!dbPtr) return std::errc::bad_file_descriptor;
+
+	Xapian::Database& db(*dbPtr);
+
+	// Set up a QueryParser with a stemmer and suitable prefixes.
+	Xapian::QueryParser queryparser;
+	//queryparser.set_stemmer(Xapian::Stem("en"));
+	queryparser.set_stemming_strategy(queryparser.STEM_SOME);
+	// Start of prefix configuration.
+	//queryparser.add_prefix("title", "S");
+	//queryparser.add_prefix("description", "XD");
+	// End of prefix configuration.
+
+	// And parse the query.
+	Xapian::Query query = queryparser.parse_query(queryStr);
+
+	// Use an Enquire object on the database to run the query.
+	Xapian::Enquire enquire(db);
+	enquire.set_query(query);
+
+	Xapian::MSet mset = enquire.get_mset(
+	            0, maxResults ? maxResults : db.get_doccount() );
+
+	for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
+	{
+		const Xapian::Document& doc = m.get_document();
+		DeepForumsSearchResult s;
+		s.mUrl = doc.get_value(URL_VALUENO);
+#if XAPIAN_AT_LEAST(1,3,5)
+		s.mSnippet = mset.snippet(doc.get_data());
+#endif // XAPIAN_AT_LEAST(1,3,5)
+		results.push_back(s);
+	}
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
+{
+	RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	forumIndexId.setQueryKV(
+	            RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
+	return forumIndexId.toString();
+}
+
+/*static*/ std::string DeepForumsIndex::postIndexId(
+        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
+{
+	RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
+	postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
+	return postIndexId.toString();
+}
+
+std::error_condition DeepForumsIndex::indexForumGroup(
+        const RsGxsForumGroup& forum )
+{
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
+	termgenerator.index_text(forum.mDescription, 1, "XD");
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(forum.mMeta.mGroupName);
+	termgenerator.increase_termpos();
+	termgenerator.index_text(forum.mDescription);
+
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	/* Store some fields for display purposes. Retrieved later to provide the
+	 * matching snippet on search */
+	doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
+
+	/* We use the identifier to ensure each object ends up in the database only
+	 * once no matter how many times we run the indexer.
+	 * "Q" prefix is a Xapian convention for unique id term. */
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumFromIndex(
+        const RsGxsGroupId& grpId )
+{
+	mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
+	{ db.delete_document("Q" + forumIndexId(grpId)); });
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
+{
+	RS_DBG4(post);
+
+	const auto& groupId = post.mMeta.mGroupId;
+	const auto& msgId = post.mMeta.mMsgId;
+
+	if(groupId.isNull() || msgId.isNull())
+	{
+		RS_ERR("Got post with invalid id ", post);
+		print_stacktrace();
+		return std::errc::invalid_argument;
+	}
+
+	// Set up a TermGenerator that we'll use in indexing.
+	Xapian::TermGenerator termgenerator;
+	//termgenerator.set_stemmer(Xapian::Stem("en"));
+
+	// We make a document and tell the term generator to use this.
+	Xapian::Document doc;
+	termgenerator.set_document(doc);
+
+	// Index each field with a suitable prefix.
+	termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
+	termgenerator.index_text(
+	            DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
+
+	// Avoid indexing RetroShare-gui HTML tags
+	const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
+	termgenerator.index_text(cleanMsg, 1, "XD" );
+
+	// Index fields without prefixes for general search.
+	termgenerator.index_text(post.mMeta.mMsgName);
+
+	termgenerator.increase_termpos();
+	termgenerator.index_text(cleanMsg);
+	// store the RS link so we are able to retrive it on matching search
+	const std::string rsLink(postIndexId(groupId, msgId));
+	doc.add_value(URL_VALUENO, rsLink);
+
+	// Store some fields for display purposes.
+	doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
+
+	// We use the identifier to ensure each object ends up in the
+	// database only once no matter how many times we run the
+	// indexer.
+	const std::string idTerm("Q" + rsLink);
+	doc.add_boolean_term(idTerm);
+
+	mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
+	{ db.replace_document(idTerm, doc); } );
+
+
+	return std::error_condition();
+}
+
+std::error_condition DeepForumsIndex::removeForumPostFromIndex(
+        RsGxsGroupId grpId, RsGxsMessageId msgId )
+{
+	// "Q" prefix is a Xapian convention for unique id term.
+	std::string idTerm("Q" + postIndexId(grpId, msgId));
+	mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
+	{ db.delete_document(idTerm); } );
+
+	return std::error_condition();
+}
+
+/*static*/ std::string DeepForumsIndex::dbDefaultPath()
+{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

libretroshare/src/deep_search/forumsindex.hpp

@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * RetroShare full text indexing and search implementation based on Xapian     *
+ *                                                                             *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#pragma once
+
+#include <system_error>
+#include <vector>
+#include <xapian.h>
+
+#include "util/rstime.h"
+#include "retroshare/rsgxsforums.h"
+#include "retroshare/rsevents.h"
+#include "deep_search/commonutils.hpp"
+
+struct DeepForumsSearchResult
+{
+	std::string mUrl;
+	double mWeight;
+	std::string mSnippet;
+};
+
+struct DeepForumsIndex
+{
+	explicit DeepForumsIndex(const std::string& dbPath) :
+	    mDbPath(dbPath), mWriteQueue(dbPath) {}
+
+	/**
+	 * @brief Search indexed GXS groups and messages
+	 * @param[in] maxResults maximum number of acceptable search results, 0 for
+	 * no limits
+	 * @return search results count
+	 */
+	std::error_condition search( const std::string& queryStr,
+	                             std::vector<DeepForumsSearchResult>& results,
+	                             uint32_t maxResults = 100 );
+
+	std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
+
+	std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
+
+	std::error_condition indexForumPost(const RsGxsForumMsg& post);
+
+	std::error_condition removeForumPostFromIndex(
+	        RsGxsGroupId grpId, RsGxsMessageId msgId );
+
+	static std::string dbDefaultPath();
+
+private:
+	static std::string forumIndexId(const RsGxsGroupId& grpId);
+	static std::string postIndexId(
+	        const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
+
+	enum : Xapian::valueno
+	{
+		/// Used to store retroshare url of indexed documents
+		URL_VALUENO,
+
+		/// @see Xapian::BAD_VALUENO
+		BAD_VALUENO = Xapian::BAD_VALUENO
+	};
+
+	const std::string mDbPath;
+
+	DeepSearch::StubbornWriteOpQueue mWriteQueue;
+};

libretroshare/src/file_sharing/dir_hierarchy.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -29,6 +31,7 @@
 #include "dir_hierarchy.h"
 #include "filelist_io.h"
 #include "file_sharing_defaults.h"
+#include "util/cxx17retrocompat.h"
 
 #ifdef RS_DEEP_FILES_INDEX
 #	include "deep_search/filesindex.hpp"
@@ -70,7 +73,8 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
     mTotalFiles = 0 ;
 }
 
-bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
+bool InternalFileHierarchyStorage::getDirHashFromIndex(
+        const DirectoryStorage::EntryIndex& index, RsFileHash& hash ) const
 {
     if(!checkIndex(index,FileStorageNode::TYPE_DIR))
         return false ;
@@ -79,6 +83,7 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E
 
     return true;
 }
+
 bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
 {
     std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mDirHashes.find(hash) ;
@@ -88,30 +93,34 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di
 
     index = it->second;
 
-    // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them.
+	/* make sure the hash actually points to some existing directory. If not,
-    if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast<DirEntry*>(mNodes[index])->dir_hash != hash)
+	 * remove it. This is an opportunistic update of dir hashes: when we need
+	 * them, we check them. */
+	if( !checkIndex(index, FileStorageNode::TYPE_DIR) ||
+	        static_cast<DirEntry*>(mNodes[index])->dir_hash != hash )
 	{
-        std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl;
+		RS_INFO("removing non existing dir hash: ", hash, " from dir hash list");
-
 		mDirHashes.erase(it);
 		return false;
 	}
     return true;
 }
-bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
-{
-    std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mFileHashes.find(hash) ;
 
-    if(it == mFileHashes.end())
+bool InternalFileHierarchyStorage::getIndexFromFileHash(
-        return false;
+        const RsFileHash& hash, DirectoryStorage::EntryIndex& index )
+{
+	auto it = std::as_const(mFileHashes).find(hash);
+	if(it == mFileHashes.end()) return false;
 
 	index = it->second;
 
-    // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them.
+	/* make sure the hash actually points to some existing file. If not, remove
-    if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast<FileEntry*>(mNodes[index])->file_hash != hash)
+	 * it. This is an opportunistic update of file hashes: when we need them,
+	 * we check them. */
+	if( !checkIndex(it->second, FileStorageNode::TYPE_FILE) ||
+	        static_cast<FileEntry*>(mNodes[index])->file_hash != hash )
 	{
-        std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl;
+		RS_INFO("removing non existing file hash: ", hash, " from file hash list");
-
 		mFileHashes.erase(it);
 		return false;
 	}
@@ -155,7 +164,10 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
     return e < mNodes.size() && mNodes[e] != NULL ;
 }
 
-bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::set<std::string>& subdirs, const RsFileHash& random_hash_seed)
+bool InternalFileHierarchyStorage::updateSubDirectoryList(
+        const DirectoryStorage::EntryIndex& indx,
+        const std::set<std::string>& subdirs,
+        const RsFileHash& random_hash_seed )
 {
 	if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
 		return false;
@@ -284,10 +296,17 @@ bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,
     return true;
 }
 
-bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
+bool InternalFileHierarchyStorage::updateSubFilesList(
+        const DirectoryStorage::EntryIndex& indx,
+        const std::map<std::string,DirectoryStorage::FileTS>& subfiles,
+        std::map<std::string,DirectoryStorage::FileTS>& new_files )
 {
 	if(!checkIndex(indx, FileStorageNode::TYPE_DIR))
+	{
+		RS_ERR("indx: ", indx, std::errc::not_a_directory);
+		print_stacktrace();
 		return false;
+	}
 
     DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
     new_files = subfiles ;
@@ -312,9 +331,11 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
             continue;
         }
 
-        if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)	// file is newer and/or has different size
+		// file is newer and/or has different size
+		if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)
         {
-            f.file_hash.clear();																// hash needs recomputing
+			// hash needs recomputing
+			f.file_hash.clear();
             f.file_modtime = it->second.modtime;
             f.file_size = it->second.size;
 
@@ -342,11 +363,14 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
     }
     return true;
 }
-bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
+bool InternalFileHierarchyStorage::updateHash(
+        const DirectoryStorage::EntryIndex& file_index, const RsFileHash& hash )
 {
 	if(!checkIndex(file_index, FileStorageNode::TYPE_FILE))
 	{
-        std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
+		RS_ERR( "Cannot update file at index ", file_index,
+		        ". Not a valid index, or not a file." );
+		print_stacktrace();
 		return false;
 	}
 #ifdef DEBUG_DIRECTORY_STORAGE
@@ -436,13 +460,19 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
 	return mNodes.size()-1 ;
 }
 
-bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,rstime_t most_recent_time,rstime_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
+bool InternalFileHierarchyStorage::updateDirEntry(
+        const DirectoryStorage::EntryIndex& indx, const std::string& dir_name,
+        rstime_t most_recent_time, rstime_t dir_modtime,
+        const std::vector<RsFileHash>& subdirs_hash,
+        const std::vector<FileEntry>& subfiles_array )
 {
 	if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
 	{
-        std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
+		RS_ERR( "cannot update dir at index ", indx, ". Not a valid index, or "
+		        "not an existing dir." );
 		return false;
 	}
+
 	DirEntry& d(*static_cast<DirEntry*>(mNodes[indx]));
 
 #ifdef DEBUG_DIRECTORY_STORAGE
@@ -703,14 +733,14 @@ const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorag
         return NULL ;
 }
 
-const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
+const InternalFileHierarchyStorage::DirEntry*
+InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
 {
-    if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
+	if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return nullptr;
-        return NULL ;
-
 	return static_cast<DirEntry*>(mNodes[indx]);
 }
-const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
+const InternalFileHierarchyStorage::FileEntry*
+InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
 {
     if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
         return NULL ;
@@ -754,7 +784,10 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,DirectorySt
 class DirectoryStorageExprFileEntry: public RsRegularExpression::ExpFileEntry
 {
 public:
-    DirectoryStorageExprFileEntry(const InternalFileHierarchyStorage::FileEntry& fe,const InternalFileHierarchyStorage::DirEntry& parent) : mFe(fe),mDe(parent) {}
+	DirectoryStorageExprFileEntry(
+	        const InternalFileHierarchyStorage::FileEntry& fe,
+	        const InternalFileHierarchyStorage::DirEntry& parent ) :
+	    mFe(fe), mDe(parent) {}
 
     inline virtual const std::string& file_name()       const { return mFe.file_name ; }
     inline virtual uint64_t           file_size()       const { return mFe.file_size ; }
@@ -768,14 +801,18 @@ private:
     const InternalFileHierarchyStorage::DirEntry& mDe ;
 };
 
-int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const
+int InternalFileHierarchyStorage::searchBoolExp(
+        RsRegularExpression::Expression* exp,
+        std::list<DirectoryStorage::EntryIndex>& results ) const
 {
-    for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
+	for(auto& it: std::as_const(mFileHashes))
-        if(mNodes[it->second] != NULL && exp->eval(
+		if(mNodes[it.second])
-                    DirectoryStorageExprFileEntry(*static_cast<const FileEntry*>(mNodes[it->second]),
+			if(exp->eval(
-                                                  *static_cast<const DirEntry*>(mNodes[mNodes[it->second]->parent_index])
+			            DirectoryStorageExprFileEntry(
+			                *static_cast<const FileEntry*>(mNodes[it.second]),
+			                *static_cast<const DirEntry*>(mNodes[mNodes[it.second]->parent_index])
 			                                      ) ))
-            results.push_back(it->second);
+			results.push_back(it.second);
 
     return 0;
 }
@@ -784,26 +821,42 @@ int InternalFileHierarchyStorage::searchTerms(
         const std::list<std::string>& terms,
         std::list<DirectoryStorage::EntryIndex>& results ) const
 {
-    // most entries are likely to be files, so we could do a linear search over the entries tab.
+	/* most entries are likely to be files, so we could do a linear search over
-    // instead we go through the table of hashes.
+	 * the entries tab. Instead we go through the table of hashes.*/
 
-    for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
+	for(auto& it : std::as_const(mFileHashes))
-        if(mNodes[it->second] != NULL)
 	{
-            const std::string &str1 = static_cast<FileEntry*>(mNodes[it->second])->file_name;
+		// node may be null for some hash waiting to be deleted
+		if(mNodes[it.second])
+		{
+			rs_view_ptr<FileEntry> tFileEntry =
+			        static_cast<FileEntry*>(mNodes[it.second]);
 
-            for(std::list<std::string>::const_iterator iter(terms.begin()); iter != terms.end(); ++iter)
+			/* Most file will just have file name stored, but single file shared
+			 * without a shared dir will contain full path instead of just the
+			 * name, so purify it to perform the search */
+			std::string tFilename = tFileEntry->file_name;
+			if(tFileEntry->file_name.find("/") != std::string::npos)
+			{
+				std::string _tParentDir;
+				RsDirUtil::splitDirFromFile(
+				            tFileEntry->file_name, _tParentDir, tFilename );
+			}
+
+			for(auto& termIt : std::as_const(terms))
 			{
 				/* always ignore case */
-                const std::string &str2 = (*iter);
+				if(tFilename.end() != std::search(
-
+				            tFilename.begin(), tFilename.end(),
-                if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() ))
+				            termIt.begin(), termIt.end(),
+				            RsRegularExpression::CompareCharIC() ))
 				{
-                    results.push_back(it->second);
+					results.push_back(it.second);
 					break;
 				}
 			}
 		}
+	}
 	return 0;
 }
 
@@ -958,7 +1011,8 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
 
 bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
 {
-    std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
+	RS_ERR(s);
+	print_stacktrace();
 	return false;
 }
 

libretroshare/src/file_sharing/directory_storage.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -30,6 +32,7 @@
 #include "directory_storage.h"
 #include "dir_hierarchy.h"
 #include "filelist_io.h"
+#include "util/cxx17retrocompat.h"
 
 #ifdef RS_DEEP_FILES_INDEX
 #	include "deep_search/filesindex.hpp"
@@ -67,8 +70,11 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
 
     return *this;
 }
-DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
+DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
-DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
+{ return mStorage->getSubFileIndex(mParentIndex, mFileTabIndex); }
+
+DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
+{ return mStorage->getSubDirIndex(mParentIndex, mDirTabIndex); }
 
 DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
 DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
@@ -139,7 +145,9 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx, const std:
     mChanged = true ;
     return res ;
 }
-bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
+bool DirectoryStorage::updateSubFilesList(
+        const EntryIndex& indx, const std::map<std::string,FileTS>& subfiles,
+        std::map<std::string,FileTS>& new_files )
 {
     RS_STACK_MUTEX(mDirStorageMtx) ;
     bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
@@ -348,7 +356,8 @@ int LocalDirectoryStorage::searchHash(const RsFileHash& hash, RsFileHash& real_h
     return false ;
 }
 
-void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
+void LocalDirectoryStorage::setSharedDirectoryList(
+        const std::list<SharedDirInfo>& lst )
 {
 	std::set<std::string> dirs_with_new_virtualname ;
     bool dirs_with_changed_flags = false ;
@@ -379,7 +388,9 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo
 			virtual_names.insert(candidate_virtual_name) ;
 		}
 
-		// now for each member of the processed list, check if it is an existing shared directory that has been changed. If so, we need to update the dir TS of that directory
+		/* now for each member of the processed list, check if it is an existing
+		 * shared directory that has been changed. If so, we need to update the
+		 * dir TS of that directory */
 
 		std::map<std::string,SharedDirInfo> new_dirs ;
 
@@ -538,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
 	        fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
 	{
 		DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-		ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
+		ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
 	}
 #endif // def RS_DEEP_FILES_INDEX
 
@@ -617,46 +628,76 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil
     return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
 }
 
-bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
+bool LocalDirectoryStorage::locked_getFileSharingPermissions(
+        const EntryIndex& indx, FileStorageFlags& flags,
+        std::list<RsNodeGroupId>& parent_groups )
 {
 	flags.clear();
 	parent_groups.clear();
 
-    std::string base_dir;
+	/* We got a request for root directory no need to do anything more after
+	 * clearing outputs */
+	if(!indx) return true;
 
-    const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
+	using FileStorageNode = InternalFileHierarchyStorage::FileStorageNode;
+	using EntryIndex = DirectoryStorage::EntryIndex;
 
-    if(n == NULL)
+	rs_view_ptr<const FileStorageNode> n = mFileHierarchy->getNode(indx);
+	if(!n)
+	{
+		RS_ERR("Node for index: ", indx, "not found");
+		print_stacktrace();
 		return false;
-
-    for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
-    {
-        const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
-
-        if(e == NULL)
-            break ;
-
-        if(e->parent_index == 0)
-        {
-            base_dir = e->dir_name ;
-            break ;
-        }
-        i = e->parent_index ;
 	}
 
-    if(!base_dir.empty())
+	// Climb down node tree up to root + 1
+	EntryIndex curIndex = indx;
+	while (n->parent_index)
 	{
-        std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
+		curIndex = n->parent_index;
+		n = mFileHierarchy->getNode(curIndex);
+	}
+
+	// Retrieve base name
+	std::string tBaseName;
+	switch (n->type())
+	{
+	// Handle single file shared case
+	case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE:
+		tBaseName = mFileHierarchy->getFileEntry(curIndex)->file_name;
+		break;
+	// Handle shared directory case
+	case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR:
+		tBaseName = mFileHierarchy->getDirEntry(curIndex)->dir_name;
+		break;
+	default:
+		RS_ERR("Got unhandled node type: ", n->type());
+		print_stacktrace();
+		return false;
+	}
+
+	// Use base name to retrieve sharing permissions
+	if(!tBaseName.empty())
+	{
+		auto it = std::as_const(mLocalDirs).find(tBaseName);
 
 		if(it == mLocalDirs.end())
 		{
-            std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
+			RS_ERR( "base name \"", tBaseName,
+			        "\" for index: ", indx, " not found in shared dir list." );
+			print_stacktrace();
 			return false;
 		}
 
 		flags = it->second.shareflags;
 		parent_groups = it->second.parent_groups;
 	}
+	else
+	{
+		RS_ERR("base name for indx: ", indx, " is empty");
+		print_stacktrace();
+		return false;
+	}
 
 	return true;
 }
@@ -705,18 +746,22 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
    return it->second.virtualname + "/" + res;
 }
 
-bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
+bool LocalDirectoryStorage::serialiseDirEntry(
+        const EntryIndex& indx, RsTlvBinaryData& bindata,
+        const RsPeerId& client_id )
 {
 	RS_STACK_MUTEX(mDirStorageMtx);
 
-    const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
+	const InternalFileHierarchyStorage::DirEntry* dir =
+	        mFileHierarchy->getDirEntry(indx);
 
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
     std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl;
 #endif
-    if(dir == NULL)
+
+	if(!dir)
 	{
-        std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
+		RS_ERR("Cannot find entry ", indx);
 		return false;
 	}
 
@@ -725,18 +770,28 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
 	FileStorageFlags node_flags;
 	std::list<RsNodeGroupId> node_groups;
 
-    // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
+	/* for each subdir, compute the node flags and groups, then ask rsPeers to
+	 * compute the mask that result from these flags for the particular peer
+	 * supplied in parameter */
 
 	for(uint32_t i=0;i<dir->subdirs.size();++i)
-        if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
+		if(indx != 0 || (
+		            locked_getFileSharingPermissions(
+		                dir->subdirs[i], node_flags, node_groups ) &&
+		            ( rsPeers->computePeerPermissionFlags(
+		                  client_id, node_flags, node_groups ) &
+		              RS_FILE_HINTS_BROWSABLE ) ))
 		{
 			RsFileHash hash;
 			if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
 			{
-              std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
+				RS_ERR( "Cannot get hash from subdir index: ",
+				        dir->subdirs[i], ". Weird bug." );
+				print_stacktrace();
 				return false;
 			}
 			allowed_subdirs.push_back(hash);
+
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
             std::cerr << "  pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl;
 #endif
@@ -746,50 +801,77 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
             std::cerr << "  not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl;
 #endif
 
-    // now count the files that do not have a null hash (meaning the hash has indeed been computed)
+	/* now count the files that do not have a null hash (meaning the hash has
-
+	 * indeed been computed), also in case files are shared singularly (without
+	 * a shared directory) so they are child of root check browsability
+	 * permission */
 	uint32_t allowed_subfiles = 0;
-
 	for(uint32_t i=0; i<dir->subfiles.size(); ++i)
 	{
-        const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
+		const InternalFileHierarchyStorage::FileEntry* file =
-        if(file != NULL && !file->file_hash.isNull())
+		        mFileHierarchy->getFileEntry(dir->subfiles[i]);
+		if(file != nullptr && !file->file_hash.isNull()
+		        && ( indx !=0 || (
+		                 locked_getFileSharingPermissions(
+		                     dir->subfiles[i], node_flags, node_groups ) &&
+		                 rsPeers->computePeerPermissionFlags(
+		                     client_id, node_flags, node_groups ) &
+		                 RS_FILE_HINTS_BROWSABLE ) ))
 			allowed_subfiles++;
 	}
 
-    unsigned char *section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
+	unsigned char* section_data = (unsigned char *)
-
+	        rs_malloc(FL_BASE_TMP_SECTION_SIZE);
-    if(!section_data)
+	if(!section_data) return false;
-        return false ;
 
 	uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
 	uint32_t section_offset = 0;
 
-    // we need to send:
+	/* we need to send:
-    //	- the name of the directory, its TS
+	 *  - the name of the directory, its TS
-    //	- the index entry for each subdir (the updte TS are exchanged at a higher level)
+	 *  - the index entry for each subdir (the updte TS are exchanged at a
-    //	- the file info for each subfile
+	 *    higher level)
-    //
+	 *  - the file info for each subfile */
+
 	std::string virtual_dir_name = locked_getVirtualDirName(indx);
 
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME       ,virtual_dir_name                   )) { free(section_data); return false ;}
+	/* Manual serialization AGAIN! This is terrible and should be ported to the
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) { free(section_data); return false ;}
+	 * new serialization system ASAP! */
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS       ,(uint32_t)dir->dir_modtime         )) { free(section_data); return false ;}
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_DIR_NAME, virtual_dir_name ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_RECURS_MODIF_TS,
+	            (uint32_t)dir->dir_most_recent_time ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)dir->dir_modtime ))
+	{ free(section_data); return false;}
 
 	// serialise number of subdirs and number of subfiles
-
+	if(!FileListIO::writeField(
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size()  )) { free(section_data); return false ;}
+	            section_data, section_size, section_offset,
-    if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles        )) { free(section_data); return false ;}
+	            FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subdirs.size() ))
+	{ free(section_data); return false; }
+	if(!FileListIO::writeField(
+	            section_data, section_size, section_offset,
+	            FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subfiles ))
+	{ free(section_data); return false; }
 
 	// serialise subdirs entry indexes
-
 	for(uint32_t i=0; i<allowed_subdirs.size(); ++i)
-        if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i]  )) { free(section_data); return false ;}
+		if(!FileListIO::writeField(
+		            section_data, section_size, section_offset,
+		            FILE_LIST_IO_TAG_ENTRY_INDEX, allowed_subdirs[i] ))
+		{ free(section_data); return false; }
 
 	// serialise directory subfiles, with info for each of them
 
-    unsigned char *file_section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
+	unsigned char* file_section_data =
-
+	        (unsigned char *) rs_malloc(FL_BASE_TMP_SECTION_SIZE);
 	if(!file_section_data)
 	{
 		free(section_data);
@@ -802,22 +884,60 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
 	{
 		uint32_t file_section_offset = 0;
 
-        const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
+		const InternalFileHierarchyStorage::FileEntry* file =
+		        mFileHierarchy->getFileEntry(dir->subfiles[i]);
 
-        if(file == NULL || file->file_hash.isNull())
+		if(file == nullptr || file->file_hash.isNull())
 		{
-            std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
+			RS_INFO( "skipping unhashed or Null file entry ",
+			         dir->subfiles[i], " to get/send file info." );
 			continue;
 		}
 
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME     ,file->file_name   )) { free(section_data);free(file_section_data);return false ;}
+		if(indx == 0)
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE     ,file->file_size   )) { free(section_data);free(file_section_data);return false ;}
+		{
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash   )) { free(section_data);free(file_section_data);return false ;}
+			if(!locked_getFileSharingPermissions(
-        if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS      ,(uint32_t)file->file_modtime)) { free(section_data);free(file_section_data);return false ;}
+			            dir->subfiles[i], node_flags, node_groups ))
+			{
+				RS_ERR( "Failure getting sharing permission for single file: ",
+				        dir->subfiles[i] );
+				print_stacktrace();
+				continue;
+			}
+
+			if(!( rsPeers->computePeerPermissionFlags(
+			          client_id, node_flags, node_groups ) &
+			      RS_FILE_HINTS_BROWSABLE ))
+			{
+				RS_INFO( "Skipping single file shared without browse "
+				         "permission" );
+				continue;
+			}
+		}
+
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_NAME, file->file_name ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_SIZE, file->file_size ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_FILE_SHA1_HASH, file->file_hash ))
+		{ free(section_data); free(file_section_data); return false; }
+		if(!FileListIO::writeField(
+		            file_section_data, file_section_size, file_section_offset,
+		            FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)file->file_modtime ))
+		{ free(section_data); free(file_section_data); return false; }
 
 		// now write the whole string into a single section in the file
-
+		if(!FileListIO::writeField(
-        if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) { free(section_data); free(file_section_data);return false ;}
+		            section_data, section_size, section_offset,
+		            FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,
+		            file_section_data, file_section_offset ))
+		{ free(section_data); free(file_section_data); return false; }
 
 #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
         std::cerr << "  pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl;
@@ -829,7 +949,8 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
     std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
 #endif
 
-    bindata.bin_data = realloc(section_data,section_offset) ;	// This discards the possibly unused trailing bytes in the end of section_data
+	// Discards the possibly unused trailing bytes in the end of section_data
+	bindata.bin_data = realloc(section_data,section_offset);
 	bindata.bin_len = section_offset;
 
 	return true;

libretroshare/src/file_sharing/directory_updater.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2016  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,13 +21,15 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  ******************************************************************************/
+
+#include "util/cxx17retrocompat.h"
 #include "util/folderiterator.h"
 #include "util/rstime.h"
 #include "rsserver/p3face.h"
-
 #include "directory_storage.h"
 #include "directory_updater.h"
 #include "file_sharing_defaults.h"
+#include "util/rsdebuglevel3.h"
 
 //#define DEBUG_LOCAL_DIR_UPDATER 1
 
@@ -123,51 +127,99 @@ bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
 {
 	if(mHashSalt.isNull())
 	{
-        std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
+		RS_ERR("no salt value in LocalDirectoryUpdater");
+		print_stacktrace();
 		return false;
 	}
 
 	mIsChecking = true;
 
 	RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-    std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
-#endif
 
-    // recursive update algorithm works that way:
+	/* recursive update algorithm works that way:
-    // 	- the external loop starts on the shared directory list and goes through sub-directories
+	 * - the external loop starts on the shared directory list and goes through
-    // 	- at the same time, it updates the local list of shared directories.  A single sweep is performed over the whole directory structure.
+	 *   sub-directories
-    // 	- the information that is costly to compute (the hash) is store externally into a separate structure.
+	 * - at the same time, it updates the local list of shared directories.
-    // 	- doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
+	 *   A single sweep is performed over the whole directory structure.
-    //
+	 * - the information that is costly to compute (the hash) is stored
+	 *   externally into a separate structure.
+	 * - doing so, changing directory names or moving files between directories
+	 * does not cause a re-hash of the content. */
 	std::list<SharedDirInfo> shared_directory_list;
 	mSharedDirectories->getSharedDirectoryList(shared_directory_list);
-
 	std::set<std::string> sub_dir_list;
 
-    // We re-check that each dir actually exists. It might have been removed from the disk.
+	/* Support also single files sharing as it make much more sense on some
+	 * platforms like Android */
+	std::map<std::string, DirectoryStorage::FileTS> singleFilesMap;
 
-    for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
+	/* We re-check that each dir actually exists. It might have been removed
-        if(RsDirUtil::checkDirectory( (*real_dir_it).filename ) )
+	 * from the disk. Accept also single files not just directories. */
-		        sub_dir_list.insert( (*real_dir_it).filename ) ;
+	for(auto& realDir: std::as_const(shared_directory_list))
+	{
+		const auto& fPath = realDir.filename;
+		if(RsDirUtil::checkDirectory(fPath))
+			sub_dir_list.insert(fPath);
+		else if (RsDirUtil::fileExists(fPath))
+		{
+			rstime_t lastWrite= RsDirUtil::lastWriteTime(fPath);
+			if(time(nullptr) >= lastWrite + MIN_TIME_AFTER_LAST_MODIFICATION)
+			{
+				uint64_t fSize = 0;
+				RsDirUtil::checkFile(fPath,fSize);
 
-    // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
+				singleFilesMap[fPath].modtime = lastWrite;
+				singleFilesMap[fPath].size = fSize;
+			}
+			else
+			{
+				some_files_not_ready = true;
+				RS_INFO( "file: \"", fPath, "\" is "
+				         "probably being written to. Keep it for later");
+			}
+		}
+		else RS_WARN( "Got non existent file \"", fPath,
+		              "\" in shared directories list. Ignored." );
+	}
 
-    mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
+	{
+		const auto tRoot = mSharedDirectories->root();
+		std::map<std::string, DirectoryStorage::FileTS> needsUpdate;
+		mSharedDirectories->updateSubFilesList(
+		            tRoot, singleFilesMap, needsUpdate);
+
+		for( DirectoryStorage::FileIterator storedSingleFilesIt(
+		         mSharedDirectories, mSharedDirectories->root() );
+		     storedSingleFilesIt; ++storedSingleFilesIt )
+		{
+			const auto& it = storedSingleFilesIt;
+			RsFileHash hash;
+			if( mHashCache->requestHash(
+			            it.name(), it.size(), it.modtime(), hash,
+			            this, *it ) )
+				mSharedDirectories->updateHash(*it, hash, it.hash() != hash);
+		}
+	}
+
+	/* make sure that entries in stored_dir_it are the same than paths in
+	 * real_dir_it, and in the same order. */
+	mSharedDirectories->updateSubDirectoryList(
+	            mSharedDirectories->root(), sub_dir_list, mHashSalt );
 
 	// now for each of them, go recursively and match both files and dirs
-
 	std::set<std::string> existing_dirs;
-
+	for( DirectoryStorage::DirIterator stored_dir_it(
-    for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
+	         mSharedDirectories, mSharedDirectories->root() );
+	     stored_dir_it; ++stored_dir_it )
 	{
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+		RS_DBG4("recursing into \"", stored_dir_it.name());
-        std::cerr << "[directory storage]   recursing into " << stored_dir_it.name() << std::endl;
-#endif
-		existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
 
-        recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ;		// here we need to use the list that was stored, instead of the shared dir list, because the two
+		existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
-                                                                            									// are not necessarily in the same order.
+		recursUpdateSharedDir(
+		            stored_dir_it.name(), *stored_dir_it,
+		            existing_dirs, 1, some_files_not_ready );
+		/* here we need to use the list that was stored, instead of the shared
+		 * dir list, because the two are not necessarily in the same order. */
 	}
 
 	RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
@@ -176,130 +228,135 @@ bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
 	return true;
 }
 
-void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
+void LocalDirectoryUpdater::recursUpdateSharedDir(
+        const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,
+        std::set<std::string>& existing_directories, uint32_t current_depth,
+        bool& some_files_not_ready )
 {
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+	RS_DBG4("parsing directory \"", cumulated_path, "\" index: ", indx);
-    std::cerr << "[directory storage]   parsing directory " << cumulated_path << ", index=" << indx << std::endl;
-#endif
 
-    // make sure list of subdirs is the same
+	/* make sure list of subdirs is the same
-    // make sure list of subfiles is the same
+	 * make sure list of subfiles is the same
-    // request all hashes to the hashcache
+	 * request all hashes to the hashcache */
 
-    librs::util::FolderIterator dirIt(cumulated_path,mFollowSymLinks,false);	// disallow symbolic links and files from the future.
+	// disallow symbolic links and files from the future.
+	librs::util::FolderIterator dirIt(cumulated_path, mFollowSymLinks, false);
 
 	rstime_t dir_local_mod_time;
 	if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
 	{
-        std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
+		RS_ERR("Cannot get local mod time for dir index: ", indx);
+		print_stacktrace();
 		return;
 	}
 
-    rstime_t now = time(NULL) ;
+	rstime_t now = time(nullptr);
-
+	/* the > is because we may have changed the virtual name, and therefore the
-    if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)	// the > is because we may have changed the virtual name, and therefore the TS wont match.
+	 * TS wont match. We only want to detect when the directory has changed on
-																		// we only want to detect when the directory has changed on the disk
+	 * the disk */
+	if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)
 	{
 		// collect subdirs and subfiles
-
 		std::map<std::string, DirectoryStorage::FileTS> subfiles;
 		std::set<std::string> subdirs;
 
 		for( ; dirIt.isValid(); dirIt.next() )
 			if(filterFile(dirIt.file_name()))
 			{
-			   switch(dirIt.file_type())
+				const auto fType = dirIt.file_type();
+				switch(fType)
 				{
 				case librs::util::FolderIterator::TYPE_FILE:
-
+					if(now >= dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION)
-                   if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
 					{
 						subfiles[dirIt.file_name()].modtime = dirIt.file_modtime();
 						subfiles[dirIt.file_name()].size = dirIt.file_size();
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+						RS_DBG4("adding sub-file \"", dirIt.file_name(), "\"");
-					   std::cerr << "  adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
-#endif
 					}
 					else
 					{
 						some_files_not_ready = true;
-
+						RS_INFO( "file: \"", dirIt.file_fullpath(), "\" is "
-                       std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being written to. Keeping it for later." << std::endl;
+						         "probably being written to. Keep it for later");
 					}
-
 					break;
-
 				case librs::util::FolderIterator::TYPE_DIR:
 				{
 					bool dir_is_accepted = true;
-
+					/* 64 is here as a safe limit, to make infinite loops
-				   if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64))	// 64 is here as a safe limit, to make loops impossible.
+					 * impossible.
+					 * TODO: Make it a visible constexpr in the header */
+					if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth)
+					        || (mMaxShareDepth == 0 && current_depth >= 64) )
 						dir_is_accepted = false;
 
 					if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
 					{
-					   std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
+						std::string real_path = RsDirUtil::removeSymLinks(
+						            cumulated_path + "/" + dirIt.file_name() );
 
-					   if(existing_directories.end() != existing_directories.find(real_path))
+						if( existing_directories.end() !=
+						        existing_directories.find(real_path) )
 						{
-						   std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
+							RS_WARN( "Directory: \"", cumulated_path,
+							         "\" has real path: \"", real_path,
+							         "\" which already belongs to another "
+							         "shared directory. Ignoring" );
 							dir_is_accepted = false;
 						}
-					   else
+						else existing_directories.insert(real_path);
-						   existing_directories.insert(real_path) ;
 					}
 
-				   if(dir_is_accepted)
+					if(dir_is_accepted) subdirs.insert(dirIt.file_name());
-					   subdirs.insert(dirIt.file_name());
+
+					RS_DBG4("adding sub-dir \"", dirIt.file_name(), "\"");
 
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-				   std::cerr << "  adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
-#endif
-			   }
 					break;
+				}
 				default:
-				   std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
+					RS_ERR( "Got Dir entry of unknown type:", fType,
+					        "with path \"", cumulated_path, "/",
+					        dirIt.file_name(), "\"" );
+					print_stacktrace();
+					break;
 				}
 			}
-       // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
 
+		/* update folder modificatoin time, which is the only way to detect
+		 * e.g. removed or renamed files. */
 		mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime());
 
 		// update file and dir lists for current directory.
-
 		mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt);
 
 		std::map<std::string, DirectoryStorage::FileTS> new_files;
 		mSharedDirectories->updateSubFilesList(indx, subfiles, new_files);
 
 		// now go through list of subfiles and request the hash to hashcache
-
+		for( DirectoryStorage::FileIterator dit(mSharedDirectories,indx);
-       for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
+		     dit; ++dit )
 		{
-		   // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
+			/* ask about the hash. If not present, ask HashCache.
-
+			 * If not present, or different, the callback will update it. */
 			RsFileHash hash;
 
-		   // mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed.
+			/* mSharedDirectories does two things: store H(F), and
+			 * compute H(H(F)), which is used in FT.
+			 * The later is always needed. */
 
-		   if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit))
+			if( mHashCache->requestHash(
+			            cumulated_path + "/" + dit.name(),
+			            dit.size(), dit.modtime(), hash, this, *dit ) )
 				mSharedDirectories->updateHash(*dit, hash, hash != dit.hash());
 		}
 	}
-#ifdef DEBUG_LOCAL_DIR_UPDATER
-    else
-        std::cerr << "  directory is unchanged. Keeping existing files and subdirs list." << std::endl;
-#endif
 
 	// go through the list of sub-dirs and recursively update
-
+	for( DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories, indx);
-		for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
+	     stored_dir_it; ++stored_dir_it )
-		{
+		recursUpdateSharedDir( cumulated_path + "/" + stored_dir_it.name(),
-#ifdef DEBUG_LOCAL_DIR_UPDATER
+		                       *stored_dir_it, existing_directories,
-			std::cerr << "  recursing into " << stored_dir_it.name() << std::endl;
+		                       current_depth+1, some_files_not_ready );
-#endif
-			recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
-		}
 }
 
 bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@@ -347,7 +404,8 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
 
 bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
 {
-    return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
+	return mSharedDirectories->getEntryType(
+	            DirectoryStorage::EntryIndex(client_param) ) == DIR_TYPE_FILE;
 }
 
 void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)

libretroshare/src/file_sharing/hash_cache.cc

@@ -187,9 +187,10 @@ void HashStorage::threadTick()
 			else
 				rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
 
-            //RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
-            if(rsEvents)
 			{
+				/* Emit deprecated event only for retrocompatibility
+				 * TODO: create a proper event with structured data instead of a
+				 * formatted string */
 				auto ev = std::make_shared<RsSharedDirectoriesEvent>();
 				ev->mEventCode = RsSharedDirectoriesEventCode::HASHING_FILE;
 				ev->mMessage = tmpout;
@@ -218,8 +219,7 @@ void HashStorage::threadTick()
 				mChanged = true ;
 				mTotalHashedSize += size ;
 			}
-			else
+			else RS_ERR("Failure hashing file: ", job.full_path);
-				std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
 
 			mHashingTime += rstime::RsScopeTimer::currentTime() - seconds_origin ;
 			mHashedBytes += size ;
@@ -234,10 +234,17 @@ void HashStorage::threadTick()
             ++mHashCounter ;
         }
 	}
-    // call the client
 
+	// call the client
 	if(!hash.isNull())
 		job.client->hash_callback(job.client_param, job.full_path, hash, size);
+
+	/* Notify we completed hashing a file */
+	auto ev = std::make_shared<RsFileHashingCompletedEvent>();
+	ev->mFilePath = job.full_path;
+	ev->mHashingSpeed = mCurrentHashingSpeed;
+	ev->mFileHash = hash;
+	rsEvents->postEvent(ev);
 }
 
 bool HashStorage::requestHash(const std::string& full_path,uint64_t size,rstime_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)

libretroshare/src/file_sharing/p3filelists.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Mr.Alice <mralice@users.sourceforge.net>                  *
+ * Copyright (C) 2018  Mr.Alice <mralice@users.sourceforge.net>                *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -30,7 +32,7 @@
 #include "retroshare/rsids.h"
 #include "retroshare/rspeers.h"
 #include "retroshare/rsinit.h"
-
+#include "util/cxx17retrocompat.h"
 #include "rsserver/p3face.h"
 
 #define P3FILELISTS_DEBUG() std::cerr << time(NULL)    << " : FILE_LISTS : " << __FUNCTION__ << " : "
@@ -1044,31 +1046,36 @@ void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIn
 }
 
 // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
-int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
+int p3FileDatabase::RequestDirDetails(
+        void* ref, DirDetails& d, FileSearchFlags flags ) const
 {
 	RS_STACK_MUTEX(mFLSMtx);
 
 	d.children.clear();
 
-    // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
+	/* Case where the pointer is NULL, which means we're at the top of the list
-    // or at the top of our own list of shared directories, depending on the flags.
+	 * of shared directories for all friends (including us) or at the top of our
+	 * own list of shared directories, depending on the flags.
+	 *
+	 * Friend index is used as follows:
+	 *  0		: own id
+	 *  1...n	: other friends
+	 *
+	 *  entry_index: starts at 0.
+	 *
+	 * The point is: we cannot use (0,0) because it encodes to NULL. No existing
+	 * combination should encode to NULL.
+	 * So we need to properly convert the friend index into 0 or into a friend
+	 * tab index in mRemoteDirectories.
+	 *
+	 * We should also check the consistency between flags and the content of ref.
+	 */
 
-    // Friend index is used as follows:
+	if (ref == nullptr)
-    //	0		: own id
-    //  1...n	: other friends
-    //
-    // entry_index: starts at 0.
-    //
-    // The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
-    // So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
-    //
-    // We should also check the consistency between flags and the content of ref.
-
-    if (ref == NULL)
 	{
-        d.ref = NULL ;
+		d.ref = nullptr;
 		d.type = DIR_TYPE_ROOT;
-        d.parent = NULL;
+		d.parent = nullptr;
         d.prow = -1;
         d.name = "root";
         d.hash.clear() ;
@@ -1079,10 +1086,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
         if(flags & RS_FILE_HINTS_LOCAL)
 		{
-            void *p;
+			void *p = nullptr;
 
 			{
-			convertEntryIndexToPointer<sizeof(void*)>(0,0,p);	// root of own directories
+				// root of own directories
+				convertEntryIndexToPointer<sizeof(void*)>(0, 0, p);
 				DirStub stub;
             stub.type = DIR_TYPE_PERSON;
             stub.name = mServCtrl->getOwnId().toStdString();
@@ -1092,9 +1100,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
 
             if(mExtraFiles->size() > 0)
             {
-            convertEntryIndexToPointer<sizeof(void*)>(0,1,p);	// local shared files from extra list
+				// local shared files from extra list
+				convertEntryIndexToPointer<sizeof(void*)>(0, 1, p);
 				DirStub stub;
-            stub.type = DIR_TYPE_PERSON;						// not totally exact, but used as a trick.
+				// not totally exact, but used as a trick.
+				stub.type = DIR_TYPE_PERSON;
             stub.name = "[Extra List]";
             stub.ref  = p;
 
@@ -1132,9 +1142,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
 
 	// check consistency
-    if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||  (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
+	if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||
+	        (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
 	{
-        P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
+		RS_ERR("Remote request on local index or local request on remote index");
 		return false;
 	}
 
@@ -1150,25 +1161,28 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
     }
 
 
-    DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
+	DirectoryStorage* storage =
+	        (flags & RS_FILE_HINTS_LOCAL) ?
+	            ((DirectoryStorage*) mLocalSharedDirs) :
+	            ((DirectoryStorage*) mRemoteDirectories[fi-1]);
 
-    // Case where the index is the top of a single person. Can be us, or a friend.
+	/* Case where the index is the top of a single person.
-
+	 * Can be us, or a friend. */
-    if(storage==NULL || !storage->extractData(e,d))
+	if(!storage || !storage->extractData(e,d))
 	{
-#ifdef DEBUG_FILE_HIERARCHY
+		RS_WARN( "request on index; ", e, ", for directory ID:",
-        P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
+		         ( (!storage)? ("[NULL]") : (storage->peerId().toStdString()) ),
-#endif
+		         " failed" );
 		return false;
 	}
 
-    // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
+	/* update indexes. This is a bit hacky, but does the job. The cast to
-    // a pointer into an int.
+	 * intptr_t is the proper way to convert a pointer into an int. */
-
 	convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref);
 
 	for(uint32_t i=0; i<d.children.size(); ++i)
-        convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
+		convertEntryIndexToPointer<sizeof(void*)>(
+		            (intptr_t) d.children[i].ref, fi, d.children[i].ref );
 
     if(e == 0)	// root
     {
@@ -1307,7 +1321,9 @@ uint32_t p3FileDatabase::watchPeriod()
     return mLocalDirWatcher->fileWatchPeriod();
 }
 
-int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
+int p3FileDatabase::SearchKeywords(
+        const std::list<std::string>& keywords, std::list<DirDetails>& results,
+        FileSearchFlags flags, const RsPeerId& client_peer_id )
 {
     if(flags & RS_FILE_HINTS_LOCAL)
     {
@@ -1319,10 +1335,10 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
 
             mLocalSharedDirs->searchTerms(keywords,firesults) ;
 
-            for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
+			for(auto& it: std::as_const(firesults))
 			{
-                void *p=NULL;
+				void *p = nullptr;
-                convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
+				convertEntryIndexToPointer<sizeof(void*)>(it, 0, p);
 				pointers.push_back(p);
 			}
         }
@@ -1476,7 +1492,9 @@ bool p3FileDatabase::search(
     return false;
 }
 
-int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
+int p3FileDatabase::filterResults(
+        const std::list<void*>& firesults, std::list<DirDetails>& results,
+        FileSearchFlags flags, const RsPeerId& peer_id ) const
 {
     results.clear();
 
@@ -1484,22 +1502,26 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
 
     /* translate/filter results */
 
-    for(std::list<void*>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
+	for(void* rit: std::as_const(firesults))
 	{
 		DirDetails cdetails;
 
-        if(!RequestDirDetails (*rit,cdetails,RS_FILE_HINTS_LOCAL))
+		if(!RequestDirDetails(rit, cdetails, RS_FILE_HINTS_LOCAL))
 		{
-            P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << *rit << std::endl;
+			RS_ERR("Cannot retrieve dir details for entry: ", rit);
+			print_stacktrace();
 			continue ;
 		}
-#ifdef DEBUG_P3FILELISTS
+
-        P3FILELISTS_DEBUG() << "Filtering candidate " << *rit << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
+		RS_DBG( "Filtering candidate: ", rit,
-#endif
+		        ", name: ", cdetails.name, ", path: ", cdetails.path,
+		        ", flags: ", cdetails.flags, ", peer: ", peer_id );
 
 		if(!peer_id.isNull())
 		{
-            FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
+			FileSearchFlags permission_flags =
+			        rsPeers->computePeerPermissionFlags(
+			            peer_id, cdetails.flags, cdetails.parent_groups );
 
 			if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
 			{
@@ -1518,6 +1540,18 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
             results.push_back(cdetails);
     }
 
+	for(auto& res: results)
+	{
+		/* Most file will just have file name stored, but single file shared
+		 * without a shared dir will contain full path instead of just the
+		 * name, so purify it to perform the search */
+		if(res.name.find("/") != std::string::npos )
+		{
+			std::string _tDirPath;
+			RsDirUtil::splitDirFromFile(res.name, _tDirPath, res.name);
+		}
+	}
+
 	return !results.empty();
 }
 
@@ -1598,17 +1632,16 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
 	{
 		RS_STACK_MUTEX(mFLSMtx);
 
-#ifdef DEBUG_P3FILELISTS
+		RS_DBG( "Received directory sync request from peer ", item->PeerId(),
-        P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
+		        ". hash=", item->entry_hash, ", flags=", item->flags,
-#endif
+		        ", request id: ", item->request_id, ", last known TS: ",
+		        item->last_known_recurs_modf_TS );
 
 		EntryIndex entry_index = DirectoryStorage::NO_INDEX;
-
 		if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
 		{
-#ifdef DEBUG_P3FILELISTS
+			RS_DBG("Cannot find entry index for hash ", item->entry_hash,
-            P3FILELISTS_DEBUG() << "  (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
+			       " cannot respond to sync request." );
-#endif
 			return;
 		}
 
@@ -1622,47 +1655,60 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
 
 		if(entry_type != DIR_TYPE_DIR)
 		{
-#ifdef DEBUG_P3FILELISTS
+			RS_DBG( "Directory does not exist anymore, or is not a directory, "
-            P3FILELISTS_DEBUG() << "  Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
+			        "or permission denied. Answering with proper flags." );
-#endif
+			ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
-            ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
+			        RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
 		}
-        else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
+		else if( entry_index != 0 &&
+		         (!mLocalSharedDirs->getFileSharingPermissions(
+		              entry_index, node_flags,node_groups ) ||
+		          !(rsPeers->computePeerPermissionFlags(
+		                item->PeerId(), node_flags, node_groups ) &
+		            RS_FILE_HINTS_BROWSABLE)) )
 		{
-            P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
+			RS_ERR("cannot get file permissions for entry index: ", entry_index,
-            ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
+			       ", or permission denied." );
+			ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
+			        RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
 		}
 		else
 		{
 			rstime_t local_recurs_max_time;
-            mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
+			mLocalSharedDirs->getDirectoryRecursModTime(
+			            entry_index, local_recurs_max_time );
 
-            if(item->last_known_recurs_modf_TS != local_recurs_max_time)	// normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
+			/* normally, should be "<", but since we provided the TS it should
+			 * be equal, so != is more robust. */
+			if(item->last_known_recurs_modf_TS != local_recurs_max_time)
 			{
-#ifdef DEBUG_P3FILELISTS
+				RS_DBG( "Directory is more recent than what the friend knows. "
-                P3FILELISTS_DEBUG() << "  Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
+				        "Sending full dir content as response.");
-#endif
 
-                ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
+				ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
+				        RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
 				ritem->last_known_recurs_modf_TS = local_recurs_max_time;
 
-                // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
+				/* We supply the peer id, in order to possibly remove some
-                mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
+				 * subdirs, if entries are not allowed to be seen by this peer.
+				 */
+				mLocalSharedDirs->serialiseDirEntry(
+				            entry_index, ritem->directory_content_data,
+				            item->PeerId() );
 			}
 			else
 			{
-#ifdef DEBUG_P3FILELISTS
+				RS_DBG3( "Directory is up to date w.r.t. what the friend knows."
-                P3FILELISTS_DEBUG() << "  Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
+				         " Sending ACK." );
-#endif
 
-                ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
+				ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
+				        RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE;
 				ritem->last_known_recurs_modf_TS = local_recurs_max_time;
 			}
 		}
 	}
 
 	// sends the response.
-
 	splitAndSendItem(ritem);
 }
 

libretroshare/src/ft/ftcontroller.cc

@@ -735,8 +735,7 @@ bool ftController::completeFile(const RsFileHash& hash)
 		RsDirUtil::splitDirFromFile(fc->mDestination,dst_dir,dst_file) ;
 
 		// We use this intermediate file in case the destination directory is not available, so as to not keep the partial file name.
-
+		std::string intermediate_file_name = RsDirUtil::makePath(src_dir, dst_file);
-		std::string intermediate_file_name = src_dir+'/'+dst_file ;
 
 		// I don't know how the size can be zero, but believe me, this happens,
 		// and it causes an error on linux because then the file may not even exist.
@@ -1542,6 +1541,11 @@ std::string ftController::getPartialsDirectory()
 	return mPartialsPath;
 }
 
+bool  ftController::FileServerCancel(const RsFileHash& hash)
+{
+    RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
+    return mDataplex->deleteServer(hash);
+}
 bool ftController::setDestinationDirectory(const RsFileHash& hash,const std::string& dest_dir)
 {
 	RsStackMutex stack(ctrlMutex); /******* LOCKED ********/

libretroshare/src/ft/ftcontroller.h

@@ -157,6 +157,7 @@ public:
         bool  getFileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info);
         bool  setDestinationName(const RsFileHash& hash,const std::string& dest_name) ;
         bool  setDestinationDirectory(const RsFileHash& hash,const std::string& dest_name) ;
+        bool  FileServerCancel(const RsFileHash& hash);
 
 		// Download speed
         bool getPriority(const RsFileHash& hash,DwlSpeed& p);

libretroshare/src/ft/ftdatamultiplex.cc

@@ -1039,6 +1039,23 @@ void ftDataMultiplex::handlePendingCrcRequests()
 			}
 }
 
+bool ftDataMultiplex::deleteServer(const RsFileHash& hash)
+{
+    RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
+
+    auto sit = mServers.find(hash);
+
+    if(sit == mServers.end())
+        return false;
+
+    // We don't delete servers that are clients at the same time !
+    if(dynamic_cast<ftFileCreator*>(sit->second) == NULL)
+        delete sit->second;
+
+    mServers.erase(sit);
+    return true;
+}
+
 void ftDataMultiplex::deleteUnusedServers()
 {
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

libretroshare/src/ft/ftdatamultiplex.h

@@ -113,9 +113,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
 		bool    FileDetails(const RsFileHash &hash, FileSearchFlags hintsflag, FileInfo &info);
 
         void deleteUnusedServers() ;
+        bool deleteServer(const RsFileHash& hash);	// deletes FtServers for the given hash. Used when removing an extra file from shares.
         void handlePendingCrcRequests() ;
 
-
 		/*************** SEND INTERFACE (calls ftDataSend) *******************/
 
 		/* Client Send */

libretroshare/src/ft/ftserver.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2008 by Robert Fernie <retroshare@lunamutt.com>                   *
+ * Copyright (C) 2008  Robert Fernie <retroshare@lunamutt.com>                 *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -817,7 +819,10 @@ bool  ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t
 }
 
 bool ftServer::ExtraFileRemove(const RsFileHash& hash)
-{ return mFileDatabase->removeExtraFile(hash); }
+{
+    mFtController->FileServerCancel(hash);
+    return mFileDatabase->removeExtraFile(hash);
+}
 
 bool ftServer::ExtraFileHash( std::string localpath, rstime_t period, TransferRequestFlags flags )
 {
@@ -873,7 +878,9 @@ int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetai
 {
 	return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
 }
-int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
+int ftServer::SearchKeywords(
+        std::list<std::string> keywords, std::list<DirDetails> &results,
+        FileSearchFlags flags, const RsPeerId& peer_id )
 {
 	return mFileDatabase->SearchKeywords(keywords, results, flags, peer_id);
 }
@@ -2030,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
 
 	std::vector<DeepFilesSearchResult> dRes;
 	DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
-	if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
+	if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
 	{
 		RsFileSearchResultItem resIt;
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
   VALIDATE_MAX_WAITING_TIME(60)
 {
     mDataAccess = new RsGxsDataAccess(gds);
-
-    // Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-    // This code is only called because it of deep indexing in channels. But loading
-    // the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
-    // eats memory, as many tests have shown. Not because of leaks, but because new threads are
-    // apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
-    // that RS will use them again.
-    //
-    //    * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
-    //      service (e.g. channels here) should update the index when a new message is received. The question to how old messages
-    //	    are processed is open. I believe that they shouldn't. New users will progressively process them.
-    //
-    //    * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
-    //      unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
-    //      It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
-    //      doesn't counterbalance such a low risk.
-    //
-    if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
-    {
-        std::vector<RsGxsGroupId> grpsToDel;
-        GxsMsgReq msgsToDel;
-
-        RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
-                                             this, *mSerialiser,
-                                             grpsToDel,msgsToDel);
-
-        for(auto& grpId: grpsToDel)
-        {
-            uint32_t token2=0;
-            deleteGroup(token2,grpId);
-        }
-
-        if(!msgsToDel.empty())
-        {
-            uint32_t token1=0;
-            deleteMsgs(token1,msgsToDel);
-        }
-    }
-#endif
 }
 
 void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@@ -362,12 +322,12 @@ void RsGenExchange::tick()
 			{
 				mIntegrityCheck = new RsGxsIntegrityCheck( mDataStore, this,
 				                                           *mSerialiser, mGixs);
-				mIntegrityCheck->start("gxs integrity");
+				std::string thisName = typeid(*this).name();
-				mChecking = true;
+				mChecking = mIntegrityCheck->start("gxs IC4 "+thisName);
 			}
 		}
 
-		if(mIntegrityCheck->isDone())
+		if(mIntegrityCheck->isDone() || !mChecking)
 		{
             std::vector<RsGxsGroupId> grpIds;
             GxsMsgReq msgIds;
@@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
             msgDeleted.push_back(note.msgIds);
     }
 
-    for(const auto& msgreq:msgDeleted)
+	/* Three nested for looks like a performance bomb, but as Cyril says here
-        for(const auto& msgit:msgreq)
+	 * https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
-            for(const auto& msg:msgit.second)
+	 * this should actually not explode at all because it is just one message at
-                mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
+	 * time that get notified */
+	for(const auto& msd : mMsgDeletePublish)
+		for(auto& msgMap : msd.mMsgs)
+			for(auto& msgId : msgMap.second)
+				mNotifications.push_back(
+				            new RsGxsMsgDeletedChange(msgMap.first, msgId) );
 
 	mMsgDeletePublish.clear();
 }

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
     return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
 }
 
-#ifndef RS_DEEP_CHANNEL_INDEX
 static bool termSearch(const std::string& src, const std::string& substring)
 {
 	/* always ignore case */
-	return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
+	return src.end() != std::search(
+	            src.begin(), src.end(), substring.begin(), substring.end(),
+	            RsRegularExpression::CompareCharIC() );
 }
-#endif // ndef RS_DEEP_CHANNEL_INDEX
+
 
 bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
 {
@@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
     return true ;
 }
 
-void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
 {
 	std::set<RsGxsGroupId> groupsToNotifyResults;
 
@@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 
 		for (const RsGxsGroupSummary& gps : group_infos)
 		{
-#ifndef RS_DEEP_CHANNEL_INDEX
+#ifdef TO_REMOVE
+			/* Because of deep search is enabled search results may bring more
+			 * info then we already have also about post that are indexed by
+			 * xapian, so we don't apply this filter anymore. */
+
 			/* Only keep groups that are not locally known, and groups that are
 			 * not already in the mDistantSearchResults structure.
-			 * mDataStore may in some situations allocate an empty group meta data, so it's important
+			 * mDataStore may in some situations allocate an empty group meta
-			 * to test that the group meta is both non null and actually corresponds to the group id we seek. */
+			 * data, so it's important to test that the group meta is both non
-
+			 * null and actually corresponds to the group id we seek. */
 			auto& meta(grpMeta[gps.mGroupId]);
+			if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
+#endif // def TO_REMOVE
 
-			if(meta != nullptr && meta->mGroupId == gps.mGroupId)
-                continue;
-
-#ifdef NXS_NET_DEBUG_9
-            std::cerr << "  group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
-#endif
-
-#else // ndef RS_DEEP_CHANNEL_INDEX
-			/* When deep search is enabled search results may bring more info
-			 * then we already have also about post that are indexed by xapian,
-			 * so we don't apply this filter in this case. */
-#endif
 			const RsGxsGroupId& grpId(gps.mGroupId);
 
 			groupsToNotifyResults.insert(grpId);
@@ -5332,16 +5330,17 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
 		mObserver->receiveDistantSearchResults(req, grpId);
 }
 
-void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
+void RsGxsNetService::receiveTurtleSearchResults(
+        TurtleRequestId req,
+        const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
 {
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
 #endif
 	auto it = mSearchRequests.find(req);
-
 	if(mSearchRequests.end() == it)
 	{
-        std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
+		RS_WARN("Received search results for unknown request: ", req);
 		return;
 	}
     RsGxsGroupId grpId = it->second;
@@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
 	mObserver->receiveDistantSearchResults(req, grpId);
 }
 
+std::error_condition RsGxsNetService::distantSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
+{
+	if(!mGxsNetTunnel)
+	{
+		free(searchData);
+		return std::errc::function_not_supported;
+	}
+
+	return mGxsNetTunnel->turtleSearchRequest(
+	            searchData, dataSize, serviceType, requestId );
+}
+
+std::error_condition RsGxsNetService::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG("");
+	return mObserver->handleDistantSearchRequest(
+	            requestData, requestSize, resultData, resultSize );
+}
+
+std::error_condition RsGxsNetService::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	return mObserver->receiveDistantSearchResult(
+	            requestId, resultData, resultSize );
+}
+
 bool RsGxsNetService::search( const std::string& substring,
                               std::list<RsGxsGroupSummary>& group_infos )
 {
 	group_infos.clear();
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-
-#warning TODO: filter deep index search result to non circle-restricted groups.
-//    /!\
-//    /!\   These results should be filtered to only return results coming from a non restricted group!
-//    /!\
-
-	std::vector<DeepChannelsSearchResult> results;
-	DeepChannelsIndex::search(substring, results);
-
-	for(auto dsr : results)
-	{
-		RsUrl rUrl(dsr.mUrl);
-		const auto& uQ(rUrl.query());
-		auto rit = uQ.find("id");
-		if(rit != rUrl.query().end())
-		{
-			RsGroupNetworkStats stats;
-			RsGxsGroupId grpId(rit->second);
-			if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
-			{
-				RsGxsGroupSummary s;
-
-				s.mGroupId = grpId;
-
-				if((rit = uQ.find("name")) != uQ.end())
-					s.mGroupName = rit->second;
-				if((rit = uQ.find("signFlags")) != uQ.end())
-					s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
-				if((rit = uQ.find("publishTs")) != uQ.end())
-					s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
-				if((rit = uQ.find("authorId")) != uQ.end())
-					s.mAuthorId  = RsGxsId(rit->second);
-
-				s.mSearchContext = dsr.mSnippet;
-
-				s.mNumberOfMessages = stats.mMaxVisibleCount;
-				s.mLastMessageTs    = stats.mLastGroupModificationTS;
-				s.mPopularity       = stats.mSuppliers;
-
-				group_infos.push_back(s);
-			}
-		}
-	}
-#else // RS_DEEP_CHANNEL_INDEX
 	RsGxsGrpMetaTemporaryMap grpMetaMap;
 	{
 		RS_STACK_MUTEX(mNxsMutex) ;
@@ -5492,7 +5477,6 @@ bool RsGxsNetService::search( const std::string& substring,
 
 			group_infos.push_back(s);
 		}
-#endif // RS_DEEP_CHANNEL_INDEX
 
 #ifdef NXS_NET_DEBUG_8
 	GXSNETDEBUG___ << "  performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;

libretroshare/src/gxs/rsgxsnetservice.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Christopher Evi-Parker                               *
+ * Copyright (C) 2012  Christopher Evi-Parker                                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSGXSNETSERVICE_H
+#pragma once
-#define RSGXSNETSERVICE_H
 
 #include <list>
 #include <queue>
@@ -130,18 +131,53 @@ public:
 
     virtual bool msgAutoSync() const override { return mAllowMsgSync; }
     virtual bool grpAutoSync() const override { return mGrpAutoSync; }
-    /*!
-     * \brief Search methods.
-     * 			These four methods are used to request distant search and receive the results.
-     * \param group_id
-     */
-    virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
-    virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
 
-    virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
+	/// @see RsNetworkExchangeService
+	std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNetworkExchangeService
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/** Request group data via turtle search
+	 * @param group_id */
+	TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
+
+	/**
+	 * @brief Search for matching groups names over turtle search.
+	 * @deprecated this method is kept mostly for retrocompatibility with older
+	 * peers, newly implemented search functions should instead be based on the
+	 * service generic search.
+	 * @see RsNetworkExchangeService
+	 */
+	RS_DEPRECATED_FOR(distantSearchRequest)
+	TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
+
+	/** @see RsNetworkExchangeService
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(receiveDistantSearchResult)
+	void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        const uint8_t* encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) override;
+
+	/**
+	 * @deprecated kept for retrocompatibility with older peers, new code should
+	 * instead be based on the service generic search */
+	RS_DEPRECATED_FOR(handleRemoteSearchRequest)
+	virtual bool search( const std::string& substring,
+	                     std::list<RsGxsGroupSummary>& group_infos) override;
     virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
     virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
-    virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
 
     virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@@ -629,5 +665,3 @@ private:
 
     bool mUseMetaCache;
 };
-
-#endif // RSGXSNETSERVICE_H

libretroshare/src/gxs/rsgxsnettunnel.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -27,6 +29,8 @@
 #include "gxs/rsnxs.h"
 #include "rsgxsnettunnel.h"
 
+/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
+
 //#define DEBUG_RSGXSNETTUNNEL 1
 
 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@@ -36,42 +40,93 @@
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA    =   1;
 static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH  = 100;
 
-RsGxsDistSync *rsGxsDistSync = NULL;
 
 RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
 {
 	mRandomBias.clear();
 
-	mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20);	// adds some variance in order to avoid doing all this tasks at once across services
+	/* adds some variance in order to avoid doing all this tasks at once across
-	mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
+	 * services */
-	mLastDump = time(NULL) + (RSRandom::random_u32()%20);
+	auto now = time(nullptr);
+	mLastKeepAlive = now + (RsRandom::random_u32()%20);
+	mLastAutoWash = now + (RsRandom::random_u32()%20);
+	mLastDump = now + (RsRandom::random_u32()%20);
 }
 
-//===========================================================================================================================================//
+//============================================================================//
 //                         Transport Items                                    //
-//===========================================================================================================================================//
+//============================================================================//
 
-const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
+enum class RsGxsNetTunnelItemSubtypes : uint8_t
+{
+	VIRTUAL_PEER             = 0x01,
 
+	KEEP_ALIVE               = 0x02,
+
+	RANDOM_BIAS              = 0x03,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
+	SEARCH_SUBSTRING         = 0x04,
+
+	SEARCH_GROUP_REQUEST     = 0x05,
+
+	// SEARCH_GROUP_SUMMARY  = 0x06, removed
+
+	SEARCH_GROUP_DATA        = 0x07,
+
+	/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
+	SEARCH_GROUP_SUMMARY     = 0x08,
+
+	/** Generic search request generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REQUEST   = 0x09,
+
+	/** Generic search reply generated and handled by specific service
+	 * (channels, forums...) */
+	SERVICE_SEARCH_REPLY     = 0x0a
+};
+
+RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
+constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
+        static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
+
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                = 0x01 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  = 0x02 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 = 0x03 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     = 0x04 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
-// const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    = 0x07 ;
+RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
 const uint8_t  RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
+// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
 
-class RsGxsNetTunnelItem: public RsItem
+struct RsGxsNetTunnelItem: RsItem
 {
 public:
-	explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
+	explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            static_cast<uint8_t>(subtype) )
+	{
+		/* no priority. All items are encapsulated into generic Turtle items
+		 * anyway. */
+	}
+
+	virtual ~RsGxsNetTunnelItem() = default;
+	virtual void clear() {}
+
+	RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
+	explicit RsGxsNetTunnelItem(uint8_t item_subtype):
+	    RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
+	            item_subtype )
 	{
 		// no priority. All items are encapsulated into generic Turtle items anyway.
 	}
-
-	virtual ~RsGxsNetTunnelItem() {}
-	virtual void clear() {}
 };
 
 class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@@ -113,7 +168,86 @@ public:
 	Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
 };
 
-class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
+struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReqItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(RsServiceType::NONE), mSearchData(nullptr),
+	    mSearchDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
+	    mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mSearchData; /// Service search request data
+	uint32_t mSearchDataSize; /// Search data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mSearchData);
+		mSearchData = nullptr;
+		mSearchDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReqItem() override { clear(); }
+};
+
+struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
+{
+	RsGxsServiceTurtleSearchReplyItem():
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(RsServiceType::NONE), mReplyData(nullptr),
+	    mReplyDataSize(0) {}
+
+	explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
+	    RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
+	    mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
+
+	/// Type of the service which originated the search request
+	RsServiceType mServiceType;
+
+	uint8_t* mReplyData; /// Service search reply data
+	uint32_t mReplyDataSize; /// Search reply data size
+
+	/// @see RsSerializable
+	void serial_process(
+	        RsGenericSerializer::SerializeJob j,
+	        RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mServiceType);
+
+		RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
+		RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
+	}
+
+	/// @see RsItem
+	void clear() override
+	{
+		free(mReplyData);
+		mReplyData = nullptr;
+		mReplyDataSize = 0;
+	}
+
+	~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
+};
+
+class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
+RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
 {
 public:
     explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@@ -164,6 +298,7 @@ public:
 		RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
 	}
 };
+
 class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
 {
 public:
@@ -193,28 +328,41 @@ public:
 class RsGxsNetTunnelSerializer: public RsServiceSerializer
 {
 public:
-	RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
+	RsGxsNetTunnelSerializer():
+	    RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
 
 	virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
 	{
 		if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
 		{
-			GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
+			RS_ERR( "received item with wrong service ID ", service);
-			return NULL ;
+			print_stacktrace();
+			return nullptr;
 		}
 
-		switch(item_subtype)
+		switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
 		{
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER                : return new RsGxsNetTunnelVirtualPeerItem ;
+		case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE                  : return new RsGxsNetTunnelKeepAliveItem ;
+			return new RsGxsNetTunnelVirtualPeerItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS                 : return new RsGxsNetTunnelRandomBiasItem ;
+		case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING     : return new RsGxsNetTunnelTurtleSearchSubstringItem;
+			return new RsGxsNetTunnelKeepAliveItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+			return new RsGxsNetTunnelRandomBiasItem;
-		case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA    : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
+			return new RsGxsNetTunnelTurtleSearchSubstringItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+			return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
+			return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
+		case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+			return new RsGxsNetTunnelTurtleSearchGroupDataItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+			return new RsGxsServiceTurtleSearchReqItem;
+		case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+			return new RsGxsServiceTurtleSearchReplyItem;
 		default:
-			GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
+			RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
-			return NULL ;
+			return nullptr;
 		}
 	}
 };
@@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
+TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
+        const std::string& match_string,
+        RsNetworkExchangeService* client_service )
 {
     GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
 
@@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
 	search_item.service = client_service->serviceType() ;
 
     uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
-    unsigned char *mem = (unsigned char*)rs_malloc(size) ;
+	uint8_t* mem = rs_malloc<uint8_t>(size);
 
     if(mem == NULL)
         return 0 ;
@@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
     return mTurtle->turtleSearch(mem,size,this) ;
 }
 
-bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
+std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
+        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+        RsServiceType serviceType, TurtleRequestId& requestId )
 {
-	GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
+	if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
+		return std::errc::invalid_argument;
 
-	RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
+	RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
+	searchItem.mSearchDataSize = dataSize;
+	searchItem.mSearchData = searchData;
 
-    RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
+	RsGxsNetTunnelSerializer tSerializer;
 
-    if(substring_sr != NULL)
+	uint32_t size = tSerializer.size(&searchItem);
-    {
+	uint8_t* buf = rs_malloc<uint8_t>(size);
-		GXS_NET_TUNNEL_DEBUG() << "  : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
 
-        max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
+	tSerializer.serialise(&searchItem, buf, &size);
 
-        std::list<RsGxsGroupSummary> results ;
+	requestId = mTurtle->turtleSearch(buf, size, this);
-        RsNetworkExchangeService *service = nullptr;
+	if(!requestId) return std::errc::result_out_of_range;
 
-        {
+	return std::error_condition();
-            RS_STACK_MUTEX(mGxsNetTunnelMtx);
-
-            auto it = mSearchableServices.find(substring_sr->service) ;
-
-            if(it != mSearchableServices.end())
-                service = it->second;
 }
 
-        if(service != nullptr && service->search(substring_sr->substring_match,results))
+rs_view_ptr<RsNetworkExchangeService>
+RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
+{
+	RS_STACK_MUTEX(mGxsNetTunnelMtx);
+	auto it = mSearchableServices.find(serviceType);
+	if( it != mSearchableServices.end()) return it->second;
+	return nullptr;
+}
+
+bool RsGxsNetTunnelService::receiveSearchRequest(
+        uint8_t* search_request_data, uint32_t search_request_data_len,
+        uint8_t*& search_result_data, uint32_t& search_result_data_size,
+        uint32_t& max_allowed_hits )
+{
+	/* Must return true only if there are matching results available, false in
+	 * all other cases. @see RsTurleClientService */
+
+	RS_DBG3("");
+
+	RsGxsNetTunnelSerializer tSerializer;
+
+	std::unique_ptr<RsItem> item;
+	item.reset(tSerializer.deserialise(
+	               search_request_data, &search_request_data_len ));
+
+	if(!item)
+	{
+		RS_ERR( "Deserialization failed: ",
+		        search_request_data, search_request_data_len, item.get() );
+		print_stacktrace();
+		return false;
+	}
+
+	switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
+	{
+	case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
+	{
+		if(!search_result_data)
+		{
+			RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
+			        "results!" );
+			print_stacktrace();
+			break;
+		}
+
+		auto substring_sr =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
+		if(!substring_sr)
+		{
+			RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
+			         item->PacketSubType(), " but casting failed!");
+			break;
+		}
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
+		std::list<RsGxsGroupSummary> results;
+		auto tService = retrievieSearchableServiceLocking(substring_sr->service);
+		if(tService && tService->search(substring_sr->substring_match, results))
 		{
 			RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
-
-			GXS_NET_TUNNEL_DEBUG() << "  : " << results.size() << " result found. Sending back." << std::endl;
-
 			search_result_item.service = substring_sr->service;
 			search_result_item.group_infos = results;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			tSerializer.serialise(
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
+			            &search_result_item, search_result_data,
-
+			            &search_result_data_size );
-            delete item;
-
-			if(search_result_data == NULL)
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
 
 			return true;
 		}
+
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
+	{
+		auto *substring_gr =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
+
+		if(!substring_gr)
+		{
+			RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
+			         item->PacketSubType(), " but casting failed!" );
+			break;
 		}
 
-    RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
-
-    if(substring_gr != NULL)
-    {
-		RS_STACK_MUTEX(mGxsNetTunnelMtx);
-		auto it = mSearchableServices.find(substring_gr->service) ;
-
 		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
-
+		uint8_t* encrypted_group_data = nullptr;
-        unsigned char *encrypted_group_data = NULL ;
 		uint32_t encrypted_group_data_len = 0;
 
-        if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
+		auto tService = retrievieSearchableServiceLocking(substring_gr->service);
+		if(tService && tService->search(
+		            substring_gr->hashed_group_id,
+		            encrypted_group_data, encrypted_group_data_len ))
 		{
 			RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
-
 			search_result_item.service = substring_gr->service;
 			search_result_item.encrypted_group_data = encrypted_group_data;
 			search_result_item.encrypted_group_data_len = encrypted_group_data_len;
+			search_result_data_size = tSerializer.size(&search_result_item);
+			search_result_data = rs_malloc<uint8_t>(search_result_data_size);
 
-            search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
+			tSerializer.serialise(
-			search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
+			            &search_result_item,
-
+			            search_result_data, &search_result_data_size );
-			if(search_result_data == NULL)
-				return false ;
-
-			RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
-
-            delete item;
 			return true;
 		}
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
+	{
+		RS_DBG3("SERVICE_SEARCH_REQUEST");
+
+		auto searchItem =
+		        static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
+
+		max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
+
+		uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
+		auto sService = retrievieSearchableServiceLocking(sType);
+		if(!sService)
+		{
+			RS_WARN("Got search request for non searchable service: ", sType);
+			break;
+		}
+
+		RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
+
+		auto errc = sService->handleDistantSearchRequest(
+		            searchItem->mSearchData, searchItem->mSearchDataSize,
+		            replyItem.mReplyData, replyItem.mReplyDataSize );
+		if(errc)
+		{
+			// Some error has been reported by the searchable service
+			RS_WARN("searchable service: ", sType , " reported: ", errc);
+			break;
+		}
+
+		if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
+		    (replyItem.mReplyData && !replyItem.mReplyDataSize) )
+		{
+			// Inconsistent behaviour from searcheable service
+			RS_ERR( "searchable service: ", sType , " silently failed handling "
+			        "inconsistent result mReplyData: ", replyItem.mReplyData,
+			        " mReplyDataSize: ", replyItem.mReplyDataSize );
+			break;
+		}
+
+		/* Our node have 0 matching results */
+		if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
+			break;
+
+		search_result_data_size = tSerializer.size(&replyItem);
+		search_result_data = rs_malloc<uint8_t>(search_result_data_size);
+
+		tSerializer.serialise(
+		            &replyItem, search_result_data, &search_result_data_size );
+
+		return true;
+	}
+	default:
+		RS_WARN("Got unknown item type: ", item->PacketSubType());
+		break;
 	}
 
-    delete item;
 	return false;
 }
 
-void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
+void RsGxsNetTunnelService::receiveSearchResult(
+        TurtleSearchRequestId request_id,
+        uint8_t* search_result_data, uint32_t search_result_data_len )
 {
-    RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
+	RS_DBG3(request_id);
 
-	GXS_NET_TUNNEL_DEBUG() << "  : received search result for search request " << std::hex << request_id << "" << std::endl;
+	std::unique_ptr<RsItem> item;
+	item.reset(RsGxsNetTunnelSerializer().deserialise(
+	               search_result_data,&search_result_data_len ));
 
-    RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
+	auto castFailedWarn = [](const uint8_t subtype)
-
-    if(result_gs != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
+		RS_WARN( "Got item with subtype: ", subtype,
-
+		         " but cast failed!" );
-#ifdef DEBUG_RSGXSNETTUNNEL
+	};
-		for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
-			std::cerr << "   group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
-#endif
-
-		auto it = mSearchableServices.find(result_gs->service) ;
 
+	auto searchableServiceGet = [this](const auto pservice)
+	{
+		auto service = static_cast<uint16_t>(pservice);
+		auto it = mSearchableServices.find(service);
 		if(it == mSearchableServices.end())
 		{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
+			RS_WARN( "got item for service ", service,
-            delete item;
+			         " which is not in the searchable services list." );
-			return ;
+			return static_cast<RsNetworkExchangeService*>(nullptr);
 		}
 
-		it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
+		return it->second;
+	};
 
-		delete item;
+	const auto tSubtype = item->PacketSubType();
-		return ;
+	switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
-	}
-
-    RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
-
-    if(result_gd != NULL)
 	{
-		GXS_NET_TUNNEL_DEBUG() << "  : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
-
-		auto it = mSearchableServices.find(result_gd->service) ;
-
-		if(it == mSearchableServices.end())
 	{
-			GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
+		auto result_gs =
-            delete item;
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
+		            item.get() );
+
+		if(!result_gs)
+		{
+			castFailedWarn(tSubtype);
+			break;
+		}
+
+		RS_DBG2( " got result is of type group summary result for service ",
+		         result_gs->service );
+
+		auto service = searchableServiceGet(result_gs->service);
+		if(service)
+			service->receiveTurtleSearchResults(
+			            request_id, result_gs->group_infos );
 		return;
 	}
+	case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
+	{
+		auto result_gd =
+		        dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
 
-		it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
+		if(!result_gd)
-
+		{
-        result_gd->encrypted_group_data = NULL ; // prevents deletion
+			castFailedWarn(tSubtype);
-		delete item;
+			break;
-
-		return ;
 		}
 
-	GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
+		RS_DBG2("got group data result for service: ", result_gd->service);
+
+		auto service = searchableServiceGet(result_gd->service);
+		if(service)
+			service->receiveTurtleSearchResults(
+			        request_id,
+			        result_gd->encrypted_group_data,
+			        result_gd->encrypted_group_data_len );
+
+		/* Ensure ownershipt is passed down preventing deletion */
+		result_gd->encrypted_group_data = nullptr;
+		break;
+	}
+	case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
+	{
+		auto searchReply =
+		        static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
+
+		auto service = searchableServiceGet(searchReply->mServiceType);
+		if(service)
+			service->receiveDistantSearchResult(
+			        request_id,
+			        searchReply->mReplyData,
+			        searchReply->mReplyDataSize );
+
+		/* Ensure memory ownership is passed down preventing deletion */
+		searchReply->mReplyData = nullptr;
+		break;
+	}
+	default:
+		RS_WARN("got item of unknown type: ", item->PacketSubType());
+		break;
+	}
 }
 
-void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias      ) const
+void RsGxsNetTunnelService::getStatistics(
+        std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
+        std::map<RsGxsNetTunnelVirtualPeerId,
+        RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
+        std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
+        turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
 {
     groups = mGroups ;
     virtual_peers = mVirtualPeers ;

libretroshare/src/gxs/rsgxsnettunnel.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2018 by Cyril Soler <retroshare.project@gmail.com>                *
+ * Copyright (C) 2018  Cyril Soler <retroshare.project@gmail.com>              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -23,6 +25,7 @@
 #pragma once
 
 #include <map>
+#include <system_error>
 
 #include "turtle/p3turtle.h"
 #include "retroshare/rsgxsdistsync.h"
@@ -100,7 +103,7 @@
 //        and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
 //
 
-class RsGxsNetTunnelItem ;
+struct RsGxsNetTunnelItem;
 class RsNetworkExchangeService ;
 
 class RsGxsNetTunnelService:
@@ -109,7 +112,7 @@ class RsGxsNetTunnelService:
 {
 public:
 	RsGxsNetTunnelService();
-	  virtual ~RsGxsNetTunnelService() ;
+	~RsGxsNetTunnelService() override;
 
       /*!
        * \brief registerSearchableService
@@ -182,23 +185,37 @@ public:
 	  void dump() const;
 
 	/*!
-       * \brief connectToTurtleRouter
 	* Should be called after allocating a RsGxsNetTunnelService
 	* \param tr turtle router object
 	*/
-	  virtual void connectToTurtleRouter(p3turtle *tr) ;
+	void connectToTurtleRouter(p3turtle *tr) override;
 
-      TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
+	/** Gxs services (channels, forums...) are supposed to use this to request
-      TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
+	 * searches on distant peers */
+	std::error_condition turtleSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId );
 
-      /*!
+	///@see RsTurtleClientService
-       * \brief receiveSearchRequest
+	bool receiveSearchRequest(
-       * 			See RsTurtleClientService::@
+	        unsigned char* search_request_data,
-       */
+	        uint32_t search_request_data_len,
-	  virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
+	        unsigned char*& search_result_data,
-	  virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
+	        uint32_t& search_result_data_len,
+	        uint32_t& max_allowed_hits ) override;
 
-	void threadTick() override; /// @see RsTickingThread
+	///@see RsTurtleClientService
+	virtual void receiveSearchResult(
+	        TurtleSearchRequestId request_id,
+	        unsigned char* search_result_data,
+	        uint32_t search_result_data_len ) override;
+
+	TurtleRequestId turtleGroupRequest(
+	        const RsGxsGroupId& group_id,
+	        RsNetworkExchangeService* client_service );
+
+	/// @see RsTickingThread
+	void threadTick() override;
 
 	  // Overloads p3Config
 
@@ -213,6 +230,11 @@ public:
             std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
 		    Bias20Bytes& bias) const;
 
+	RS_DEPRECATED
+	TurtleRequestId turtleSearchRequest(
+	        const std::string& match_string,
+	        RsNetworkExchangeService* client_service );
+
 protected:
 	  // interaction with turtle router
 
@@ -233,6 +255,8 @@ private:
 	  void sendKeepAlivePackets() ;
 	  void handleIncoming(RsGxsNetTunnelItem *item) ;
 	  void flush_pending_items();
+	rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
+	        uint16_t serviceType );
 
 	  std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ;	// groups on the client and server side
 

libretroshare/src/gxs/rsgxsnotify.h

@@ -97,3 +97,13 @@ private:
     bool mMetaChange;
 };
 
+struct RsGxsMsgDeletedChange : RsGxsNotify
+{
+	RsGxsMsgDeletedChange(
+	        const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
+	    RsGxsNotify(gid), messageId(msgId) {}
+
+	NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
+
+	const RsGxsMessageId messageId;
+};

libretroshare/src/gxs/rsgxsutil.cc

@@ -3,8 +3,8 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013  Christopher Evi-Parker                                  *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -29,12 +29,6 @@
 #include "pqi/pqihash.h"
 #include "gxs/rsgixs.h"
 
-#ifdef RS_DEEP_CHANNEL_INDEX
-#	include "deep_search/channelsindex.hpp"
-#	include "services/p3gxschannels.h"
-#	include "rsitems/rsgxschannelitems.h"
-#endif
-
 // The goals of this set of methods is to check GXS messages and groups for consistency, mostly
 // re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
 // happen anyway, but we still conduct these test as an extra safety measure.
@@ -198,8 +192,7 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
 
 RsGxsIntegrityCheck::RsGxsIntegrityCheck(
         RsGeneralDataService* const dataService, RsGenExchange* genex,
-    RsSerialType&
+        RsSerialType&, RsGixs* gixs )
-                            , RsGixs* gixs )
   : mDs(dataService), mGenExchangeClient(genex),
     mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
 
@@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
     return true;
 }
 
-bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
+bool RsGxsSinglePassIntegrityCheck::check(
-#ifdef RS_DEEP_CHANNEL_INDEX
+        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-                                , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
+        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
-#endif
-                                , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
 {
 #ifdef DEBUG_GXSUTIL
     GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
 #endif
-#ifdef RS_DEEP_CHANNEL_INDEX
-    bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
-    std::set<RsGxsGroupId> indexedGroups;
-#endif
 
     // first take out all the groups
     std::map<RsGxsGroupId, RsNxsGrp*> grp;
@@ -393,54 +380,13 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
             }
             else
                 msgIds.erase(msgIds.find(grp->grpId));	// could not get them, so group is removed from list.
-
-#ifdef RS_DEEP_CHANNEL_INDEX
-            // This should be moved to p3gxschannels. It is really not the place for this here!
-
-            if( isGxsChannels
-                    && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
-                    && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
-            {
-                RsGxsGrpMetaData meta;
-                meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
-
-                uint32_t blz = grp->grp.bin_len;
-                RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
-                                                      &blz);
-
-                if( RsGxsChannelGroupItem* cgIt =
-                        dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
-                {
-                    RsGxsChannelGroup cg;
-                    cgIt->toChannelGroup(cg, false);
-                    cg.mMeta = meta;
-
-                    indexedGroups.insert(grp->grpId);
-                    DeepChannelsIndex::indexChannelGroup(cg);
         }
 		else
 		{
-                    std::cerr << __PRETTY_FUNCTION__ << " Group: "
+			RS_WARN( "deleting group ", grp->grpId,
-                              << meta.mGroupId.toStdString() << " "
+			         " with wrong hash or null/corrupted meta data. meta=",
-                              << meta.mGroupName
+			         grp->metaData );
-                              << " doesn't seems a channel, please "
-                              << "report to developers"
-                              << std::endl;
-                    print_stacktrace();
-                }
-
-                delete rIt;
-            }
-#endif // def RS_DEEP_CHANNEL_INDEX
-        }
-        else
-        {
-            std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
 			grpsToDel.push_back(grp->grpId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-            if(isGxsChannels)
-                DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
-#endif // def RS_DEEP_CHANNEL_INDEX
 		}
 
         delete grp;
@@ -471,13 +417,7 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
 		for (auto& msgId:msgIdV)
 			if(nxsMsgS.find(msgId) == nxsMsgS.end())
-            {
 				msgsToDel[grpId].insert(msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
-#endif // def  RS_DEEP_CHANNEL_INDEX
-            }
     }
 
     for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
 
             if(msg->metaData == NULL || currHash != msg->metaData->mHash)
             {
-                std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
+				RS_WARN( "deleting message ", msg->msgId, " in group ",
+				         msg->grpId,
+				         " with wrong hash or null/corrupted meta data. meta=",
+				         static_cast<void*>(msg->metaData) );
                 msgsToDel[msg->grpId].insert(msg->msgId);
-#ifdef RS_DEEP_CHANNEL_INDEX
-                if(isGxsChannels)
-                    DeepChannelsIndex::removeChannelPostFromIndex(
-                                msg->grpId, msg->msgId );
-#endif // def RS_DEEP_CHANNEL_INDEX
-            }
-            else if (subscribed_groups.count(msg->metaData->mGroupId))
-            {
-#ifdef RS_DEEP_CHANNEL_INDEX
-                // This should be moved to p3gxschannels. It is really not the place for this here!
-
-                if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
-                {
-                    RsGxsMsgMetaData meta;
-                    meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
-
-                    uint32_t blz = msg->msg.bin_len;
-                    RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
-                                                          &blz);
-
-                    if( RsGxsChannelPostItem* cgIt =
-                            dynamic_cast<RsGxsChannelPostItem*>(rIt) )
-                    {
-                        RsGxsChannelPost cg;
-                        cgIt->toChannelPost(cg, false);
-                        cg.mMeta = meta;
-
-                        DeepChannelsIndex::indexChannelPost(cg);
-                    }
-                    else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
-                    else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
-                    else
-                    {
-                        std::cerr << __PRETTY_FUNCTION__ << " Message: "
-                                  << meta.mMsgId.toStdString()
-                                  << " in group: "
-                                  << meta.mGroupId.toStdString() << " "
-                                  << " doesn't seems a channel post, please "
-                                  << "report to developers"
-                                  << std::endl;
-                        print_stacktrace();
-                    }
-
-                    delete rIt;
-                }
-#endif // def RS_DEEP_CHANNEL_INDEX
             }
 
             delete msg;

libretroshare/src/gxs/rsgxsutil.h

@@ -3,8 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2013-2013 by Christopher Evi-Parker                               *
+ * Copyright (C) 2013 Christopher Evi-Parker                                   *
- * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -162,15 +163,6 @@ class RsGxsIntegrityCheck : public RsThread
     enum CheckState { CheckStart, CheckChecking };
 
 public:
-
-
-    /*!
-     *
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
 	RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
 	                     RsGenExchange* genex, RsSerialType&,
 	                     RsGixs* gixs );
@@ -201,19 +193,9 @@ private:
 class RsGxsSinglePassIntegrityCheck
 {
 public:
-
+	static bool check(
-    /*!
+	        uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
-     *
+	        std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
-     * @param dataService
-     * @param mGroupTS
-     * @param chunkSize
-     * @param sleepPeriod
-     */
-    static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
-#ifdef RS_DEEP_CHANNEL_INDEX
-                      , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
-#endif
-                      , std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
 };
 
 class GroupUpdate

libretroshare/src/gxs/rsnxs.h

@@ -3,8 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com>         *
+ * Copyright (C) 2011  Robert Fernie <retroshare.project@gmail.com>            *
- * Copyright 2011-2011 by Christopher Evi-Parker                               *
+ * Copyright (C) 2011  Christopher Evi-Parker                                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -20,17 +22,15 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
+#pragma once
-#ifndef RSGNP_H
-#define RSGNP_H
 
 #include <set>
 #include <string>
-#include "util/rstime.h"
 #include <stdlib.h>
 #include <list>
 #include <map>
 
+#include "util/rstime.h"
 #include "services/p3service.h"
 #include "retroshare/rsreputations.h"
 #include "retroshare/rsidentity.h"
@@ -61,9 +61,8 @@
 class RsNetworkExchangeService
 {
 public:
-
+	RsNetworkExchangeService() = default;
-	RsNetworkExchangeService(){ return;}
+	virtual ~RsNetworkExchangeService() = default;
-    virtual ~RsNetworkExchangeService() {}
 
     virtual uint16_t serviceType() const =0;
     /*!
@@ -85,9 +84,24 @@ public:
     virtual bool msgAutoSync() const =0;
     virtual bool grpAutoSync() const =0;
 
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
 	///                    DISTANT SEARCH FUNCTIONS                          ///
-    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
+
+	/// Trigger remote generic GXS service search
+	virtual std::error_condition distantSearchRequest(
+	        rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
+	        RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
+
+	/// Handle remote generic GXS services search requests to specific service
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
+
+	/// Receive remote generic GXS services search result
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
 
     /*!
      * \brief turtleGroupRequest
@@ -121,7 +135,11 @@ public:
 	 * \param req Turtle search request ID associated with this result
 	 * \param encrypted_group_data  Group data
 	 */
-	virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
+	RS_DEPRECATED_FOR("receiveDistantSearchResult")
+	virtual void receiveTurtleSearchResults(
+	        TurtleRequestId req,
+	        rs_owner_ptr<const uint8_t> encrypted_group_data,
+	        uint32_t encrypted_group_data_len ) = 0;
 
     /*!
      * \brief retrieveTurtleSearchResults
@@ -141,7 +159,9 @@ public:
     virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
     virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
 
+	RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
 	virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
+
 	virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
 
     /*!
@@ -306,5 +326,3 @@ public:
 		}
 	}
 };
-
-#endif // RSGNP_H

libretroshare/src/gxs/rsnxsobserver.cpp

@@ -1,9 +1,8 @@
 /*******************************************************************************
- * libretroshare/src/util: rserrno.h                                           *
+ * RetroShare General eXchange System                                          *
  *                                                                             *
- * libretroshare: retroshare core library                                      *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
- *                                                                             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,6 +18,19 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#pragma once
 
-const char* rsErrnoName(int err);
+#include "gxs/rsnxsobserver.h"
+
+const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
+
+std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
+const noexcept
+{
+	switch(static_cast<RsNxsObserverErrorNum>(ev))
+	{
+	case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
+		return std::errc::operation_not_supported;
+	default:
+		return std::error_condition(ev, *this);
+	}
+}

libretroshare/src/gxs/rsnxsobserver.h

@@ -3,7 +3,10 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher                *
+ * Copyright (C) 2011-2012 Robert Fernie                                       *
+ * Copyright (C) 2011-2012 Christopher Evi-Parker                              *
+ * Copyright (C) 2021  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,21 +22,61 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RSNXSOBSERVER_H
+#pragma once
-#define RSNXSOBSERVER_H
 
-#include <set>
+#include <system_error>
+#include <vector>
+
+#include "retroshare/rsgxsiface.h"
 #include "rsitems/rsnxsitems.h"
+#include "util/rsdebug.h"
 
 typedef uint32_t TurtleRequestId;
 
+enum class RsNxsObserverErrorNum : int32_t
+{
+	NOT_OVERRIDDEN_BY_OBSERVER = 2004,
+};
+
+struct RsNxsObserverErrorCategory: std::error_category
+{
+	const char* name() const noexcept override
+	{ return "RetroShare NXS Observer"; }
+
+	std::string message(int ev) const override
+	{
+		switch (static_cast<RsNxsObserverErrorNum>(ev))
+		{
+		case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
+			return "Method not overridden by observer";
+		default:
+			return rsErrorNotInCategory(ev, name());
+		}
+	}
+
+	std::error_condition default_error_condition(int ev) const noexcept override;
+
+	const static RsNxsObserverErrorCategory instance;
+};
+
+
+namespace std
+{
+/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
+ * namespace */
+template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
+}
+
+/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
+ * same namespace of RsJsonApiErrorNum */
+inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
+{
+	return std::error_condition(
+	            static_cast<int>(e), RsNxsObserverErrorCategory::instance );
+};
+
 class RsNxsObserver
 {
-public:
-
-    RsNxsObserver() {}
-
-
 public:
 
     /*!
@@ -56,6 +99,46 @@ public:
         std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
     }
 
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * requests there.
+	 * @param[in] requestData search query
+	 * @param[in] requestSize search query size
+	 * @param[out] resultData results data storage for a pointer to search
+	 * result reply data or nullptr if no mathing results where found
+	 * @param[out] resultSize storage for results data size or 0 if no matching
+	 * results where found
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		/* Avoid unused paramethers warning this way so doxygen can still parse
+		 * paramethers documentation */
+		(void) requestData; (void) requestSize;
+		(void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
+	/** If advanced search functionalities like deep indexing are supported at
+	 * observer/service level, this method should be overridden to handle search
+	 * results there.
+	 * @param[in] requestId search query id
+	 * @param[out] resultData results data
+	 * @param[out] resultSize results data size
+	 * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
+	 * returned to inform the caller that this method was not overridden by the
+	 * observer so do not use it for other meanings. */
+	virtual std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+	{
+		(void) requestId; (void) resultData; (void) resultSize;
+		return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
+	}
+
     /*!
      * @param grpId group id
      */
@@ -70,6 +153,7 @@ public:
      * @param grpId group id
      */
     virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
-};
 
-#endif // RSNXSOBSERVER_H
+	RsNxsObserver() = default;
+	virtual ~RsNxsObserver() = default;
+};

libretroshare/src/libretroshare.pro

@@ -398,7 +398,6 @@ HEADERS +=	pqi/authssl.h \
 			pqi/pqissl.h \
 			pqi/pqissllistener.h \
 			pqi/pqisslpersongrp.h \
-                        pqi/pqissli2pbob.h \
 			pqi/pqisslproxy.h \
 			pqi/pqistore.h \
 			pqi/pqistreamer.h \
@@ -459,7 +458,7 @@ HEADERS +=	rsitems/rsitem.h \
 			rsitems/rsgxsupdateitems.h \
 			rsitems/rsserviceinfoitems.h \
 
-HEADERS +=  services/autoproxy/p3i2pbob.h \
+HEADERS +=  \
             services/rseventsservice.h \
             services/autoproxy/rsautoproxymonitor.h \
             services/p3msgservice.h \
@@ -513,8 +512,7 @@ HEADERS +=	util/folderiterator.h \
     util/cxx11retrocompat.h \
     util/cxx14retrocompat.h \
     util/cxx17retrocompat.h \
-            util/rsurl.h \
+            util/rsurl.h
-    util/rserrno.h
 
 SOURCES +=	ft/ftchunkmap.cc \
 			ft/ftcontroller.cc \
@@ -563,7 +561,6 @@ SOURCES +=	pqi/authgpg.cc \
 			pqi/pqissl.cc \
 			pqi/pqissllistener.cc \
 			pqi/pqisslpersongrp.cc \
-                        pqi/pqissli2pbob.cpp \
 			pqi/pqisslproxy.cc \
 			pqi/pqistore.cc \
 			pqi/pqistreamer.cc \
@@ -621,7 +618,6 @@ SOURCES +=	serialiser/rsbaseserial.cc \
 
 SOURCES +=  services/autoproxy/rsautoproxymonitor.cc \
     services/rseventsservice.cc \
-            services/autoproxy/p3i2pbob.cc \
             services/p3msgservice.cc \
 			services/p3service.cc \
 			services/p3statusservice.cc \
@@ -641,10 +637,10 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsexpr.cc \
 			util/smallobject.cc \
 			util/rsdir.cc \
-			util/rsmemory.cc \
 			util/rsdiscspace.cc \
 			util/rsnet.cc \
 			util/rsnet_ss.cc \
+			util/rsdnsutils.cc \
 			util/extaddrfinder.cc \
 			util/dnsresolver.cc \
 			util/rsprint.cc \
@@ -655,8 +651,7 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsrecogn.cc \
             util/rstime.cc \
             util/rsurl.cc \
-            util/rsbase64.cc \
+            util/rsbase64.cc
-    util/rserrno.cc
 
 equals(RS_UPNP_LIB, miniupnpc) {
         HEADERS += rs_upnp/upnputil.h rs_upnp/upnphandler_miniupnp.h
@@ -720,7 +715,8 @@ SOURCES += rsitems/rsnxsitems.cc \
 	gxs/gxstokenqueue.cc \
 	gxs/rsgxsnetutils.cc \
 	gxs/rsgxsutil.cc \
-	gxs/rsgxsrequesttypes.cc
+        gxs/rsgxsrequesttypes.cc \
+        gxs/rsnxsobserver.cpp
 
 # Tor
 HEADERS += 	retroshare/rstor.h 
@@ -986,6 +982,14 @@ rs_jsonapi {
     SOURCES += jsonapi/jsonapi.cpp
 }
 
+rs_deep_forums_index {
+    HEADERS *= deep_search/commonutils.hpp
+    SOURCES *= deep_search/commonutils.cpp
+
+    HEADERS += deep_search/forumsindex.hpp
+    SOURCES += deep_search/forumsindex.cpp
+}
+
 rs_deep_channels_index {
     HEADERS *= deep_search/commonutils.hpp
     SOURCES *= deep_search/commonutils.cpp
@@ -1059,6 +1063,44 @@ rs_broadcast_discovery {
     }
 }
 
+rs_sam3 {
+    SOURCES += \
+        services/autoproxy/p3i2psam3.cpp \
+        pqi/pqissli2psam3.cpp \
+
+    HEADERS += \
+        services/autoproxy/p3i2psam3.h \
+        pqi/pqissli2psam3.h \
+}
+
+rs_sam3_libsam3 {
+    DUMMYQMAKECOMPILERINPUT = FORCE
+    libsam3.name = Generating libsam3.
+    libsam3.input = DUMMYQMAKECOMPILERINPUT
+    libsam3.output = $$clean_path($${LIBSAM3_BUILD_PATH}/libsam3.a)
+    libsam3.CONFIG += target_predeps combine
+    libsam3.variable_out = PRE_TARGETDEPS
+    win32-g++:isEmpty(QMAKE_SH) {
+        LIBSAM3_MAKE_PARAMS = CC=gcc
+        libsam3.commands = \
+            cd /D $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/libsam3 || cd . $$escape_expand(\\n\\t) \
+            $(CHK_DIR_EXISTS) $$shell_path($$LIBSAM3_BUILD_PATH) $(MKDIR) $$shell_path($${LIBSAM3_BUILD_PATH}) $$escape_expand(\\n\\t) \
+            $(COPY_DIR) $$shell_path($${LIBSAM3_SRC_PATH}) $$shell_path($${LIBSAM3_BUILD_PATH}) || cd . $$escape_expand(\\n\\t)
+    } else {
+        LIBSAM3_MAKE_PARAMS =
+        libsam3.commands = \
+            cd $${RS_SRC_PATH} && ( \
+            git submodule update --init supportlibs/libsam3 || \
+            true ) && \
+            mkdir -p $${LIBSAM3_BUILD_PATH} && \
+            (cp -r $${LIBSAM3_SRC_PATH}/* $${LIBSAM3_BUILD_PATH} || true) &&
+    }
+    libsam3.commands += \
+        cd $$shell_path($${LIBSAM3_BUILD_PATH}) && \
+        $(MAKE) build $${LIBSAM3_MAKE_PARAMS}
+    QMAKE_EXTRA_COMPILERS += libsam3
+}
+
 ###########################################################################################################
 # OLD CONFIG OPTIONS.
 # Not used much - but might be useful one day.

libretroshare/src/pqi/p3netmgr.cc

@@ -47,14 +47,15 @@ struct RsLog::logInfo p3netmgrzoneInfo = {RsLog::Default, "p3netmgr"};
 
 /* Network setup States */
 
-const uint32_t RS_NET_NEEDS_RESET = 	0x0000;
+constexpr uint32_t RS_NET_NEEDS_RESET = 0x0000;
-const uint32_t RS_NET_UNKNOWN = 	0x0001;
+constexpr uint32_t RS_NET_UNKNOWN =     0x0001;
-const uint32_t RS_NET_UPNP_INIT = 	0x0002;
+constexpr uint32_t RS_NET_UPNP_INIT =   0x0002;
-const uint32_t RS_NET_UPNP_SETUP =  	0x0003;
+constexpr uint32_t RS_NET_UPNP_SETUP =  0x0003;
-const uint32_t RS_NET_EXT_SETUP =  	0x0004;
+constexpr uint32_t RS_NET_EXT_SETUP =   0x0004;
-const uint32_t RS_NET_DONE =    	0x0005;
+constexpr uint32_t RS_NET_DONE =        0x0005;
-const uint32_t RS_NET_LOOPBACK =    	0x0006;
+constexpr uint32_t RS_NET_LOOPBACK =    0x0006;
-//const uint32_t RS_NET_DOWN =    	0x0007;
+//constexpr uint32_t RS_NET_DOWN =        0x0007;
+constexpr uint32_t RS_NET_SHUTDOWN =    0x00FF; //Highest value to not restart UPnP nor ExtAddrFinder
 
 /* Stun modes (TODO) */
 //const uint32_t RS_STUN_DHT =      	0x0001;
@@ -68,6 +69,8 @@ const uint32_t MAX_UPNP_COMPLETE = 	600; /* 10 min... seems to take a while */
 
 //const uint32_t MIN_TIME_BETWEEN_NET_RESET = 		5;
 
+const uint32_t MIN_TIME_EXT_FINDER_UPDATE = 300; /* 5min to check if external IP is changed */
+
 /****
  * #define NETMGR_DEBUG 1
  * #define NETMGR_DEBUG_RESET 1
@@ -107,19 +110,17 @@ void pqiNetStatus::print(std::ostream &out)
 }
 
 
-p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
+p3NetMgrIMPL::p3NetMgrIMPL()
-    mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
+    : mPeerMgr(nullptr), mLinkMgr(nullptr)
-    mDoNotNetCheckUntilTs(0)
+    , mNetMtx("p3NetMgr"), mNetMode(RS_NET_MODE_UDP), mVsDisc(RS_VS_DISC_FULL), mVsDht(RS_VS_DHT_FULL)// default to full.
+    , mNetInitTS(0), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
+    , mUseExtAddrFinder(true), mNetExtAddrFinderTs(0), mDoNotNetCheckUntilTs(0)
 {
 
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
-		mNetMode = RS_NET_MODE_UDP;
-
-		mUseExtAddrFinder = true;
 		mExtAddrFinder = new ExtAddrFinder();
-		mNetInitTS = 0;
 	
 		mNetFlags = pqiNetStatus();
 		mOldNetFlags = pqiNetStatus();
@@ -133,10 +134,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
 		mLocalAddr.ss_family = AF_INET;
 		mExtAddr.ss_family = AF_INET;
 
-		// default to full.
-		mVsDisc = RS_VS_DISC_FULL;
-		mVsDht = RS_VS_DHT_FULL;
-
 	}
 	
 #ifdef NETMGR_DEBUG
@@ -235,6 +232,10 @@ void p3NetMgrIMPL::netReset()
 	rslog(RSL_ALERT, p3netmgrzone, "p3NetMgr::netReset() Called");
 
 	shutdown(); /* blocking shutdown call */
+	{
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
+		mNetStatus = RS_NET_UNKNOWN;
+	}
 
 	// Will initiate a new call for determining the external ip.
 	if (mUseExtAddrFinder)
@@ -242,7 +243,7 @@ void p3NetMgrIMPL::netReset()
 #ifdef NETMGR_DEBUG_RESET
 		std::cerr << "p3NetMgrIMPL::netReset() restarting AddrFinder" << std::endl;
 #endif
-		mExtAddrFinder->reset() ;
+		mExtAddrFinder->reset(true) ;
 	}
 	else
 	{
@@ -265,7 +266,7 @@ void p3NetMgrIMPL::netReset()
 		*      as it calls back to p3ConnMgr.
 		*/
 
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		struct sockaddr_storage iaddr = mLocalAddr;
 		
@@ -283,8 +284,7 @@ void p3NetMgrIMPL::netReset()
 	}
 
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
-        	mNetStatus = RS_NET_UNKNOWN;
 		netStatusReset_locked();
 	}
 
@@ -311,8 +311,8 @@ bool p3NetMgrIMPL::shutdown() /* blocking shutdown call */
 	std::cerr << std::endl;
 #endif
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
-		mNetStatus = RS_NET_UNKNOWN;
+		mNetStatus = RS_NET_SHUTDOWN;
 		mNetInitTS = time(NULL);
 		netStatusReset_locked();
 	}
@@ -347,7 +347,7 @@ void p3NetMgrIMPL::netStartup()
 	 */
 
 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 	mNetInitTS = time(NULL);
 	netStatusReset_locked();
@@ -460,35 +460,51 @@ void p3NetMgrIMPL::netStatusTick()
 
 	uint32_t netStatus = 0;
 	rstime_t   age = 0;
+	bool needExtFinderUpdate = false;
 	{
-		RsStackMutex stack(mNetMtx);   /************** LOCK MUTEX ***************/
+		RS_STACK_MUTEX(mNetMtx);   /************** LOCK MUTEX ***************/
 
 		netStatus = mNetStatus;
 		age = time(NULL) - mNetInitTS;
 
+		needExtFinderUpdate = netStatus == RS_NET_DONE;
+		needExtFinderUpdate &= mNetExtAddrFinderTs < time(nullptr);
+		if(needExtFinderUpdate)
+			mNetExtAddrFinderTs = time(nullptr) + MIN_TIME_EXT_FINDER_UPDATE;
 	}
 
-    if(netStatus <= RS_NET_UPNP_SETUP && mUseExtAddrFinder)
+	if( mUseExtAddrFinder
+	   && ( netStatus <= RS_NET_UPNP_SETUP
+	        || needExtFinderUpdate) )
 	{
-        sockaddr_storage tmpip = mLocalAddr;	// copies local port and correctly inits the IP family
+		sockaddr_storage tmpip;
+		sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-        std::cerr << "Asking ExtAddrFinder for IP. Initializing port with " << sockaddr_storage_port(tmpip) << std::endl;
+		RS_DBG("Asking ExtAddrFinder for IP. Initializing port with ", sockaddr_storage_port(tmpip));
 #endif
 
-        if(mExtAddrFinder->hasValidIP(tmpip) && sockaddr_storage_ipv6_to_ipv4(tmpip) && !sockaddr_storage_same(tmpip,mExtAddr))
+		if(mExtAddrFinder->hasValidIPV4(tmpip))
+		{
+			if(!sockaddr_storage_same(tmpip,mExtAddr))
 			{
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-            std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
+				RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
-#endif
-
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-            std::cerr << "p3NetMgrIMPL::netExtCheck() ";
-            std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
-            std::cerr << std::endl;
 #endif
 				setExtAddress(tmpip);
 			}
 		}
+		else if(mExtAddrFinder->hasValidIPV6(tmpip))
+		{
+			if(!sockaddr_storage_same(tmpip,mExtAddr))
+			{
+				//Only if no IPv4 else, reset connections on setExtAddress()
+#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
+				RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
+#endif
+				setExtAddress(tmpip);
+			}
+		}
+	}
 
 	switch(netStatus)
 	{
@@ -578,7 +594,7 @@ void p3NetMgrIMPL::netDhtInit()
 	
 	uint32_t vs = 0;
 	{
-		RsStackMutex stack(mNetMtx); /*********** LOCKED MUTEX ************/
+		RS_STACK_MUTEX(mNetMtx); /*********** LOCKED MUTEX ************/
 		vs = mVsDht;
 	}
 	
@@ -745,25 +761,16 @@ void p3NetMgrIMPL::netExtCheck()
 			if (mUseExtAddrFinder)
 			{
 #if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-				std::cerr << "p3NetMgrIMPL::netExtCheck() checking ExtAddrFinder" << std::endl;
+				RS_DBG("checking ExtAddrFinder");
 #endif
-                sockaddr_storage tmpip = mLocalAddr;	// copies local port and correctly inits the IP family
+				sockaddr_storage tmpip;
+				sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
 
-                bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
+				// Test for IPv4 first to be compatible with older versions.
-
+				if (mExtAddrFinder->hasValidIPV4(tmpip))
-				if (extFinderOk && sockaddr_storage_ipv6_to_ipv4(tmpip))
 				{
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-					std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
-#endif
 					sockaddr_storage_setport(tmpip, guessNewExtPort());
 
-#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
-					std::cerr << "p3NetMgrIMPL::netExtCheck() ";
-					std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
-					std::cerr << std::endl;
-#endif
-
 					mNetFlags.mExtAddrOk = true;
 
 					address_votes[tmpip].n++ ;
@@ -773,10 +780,22 @@ void p3NetMgrIMPL::netExtCheck()
 					 * (which it is not normally) */
 					mNetFlags.mExtAddrStableOk = true;
 
-					std::cerr << __PRETTY_FUNCTION__ << " ExtAddrFinder "
+					RS_DBG("Reported external IPv4 address ", sockaddr_storage_iptostring(tmpip));
-					          << " reported external address "
+				}
-					          << sockaddr_storage_iptostring(tmpip)
+				else if (mExtAddrFinder->hasValidIPV6(tmpip))
-					          << std::endl;
+				{
+					sockaddr_storage_setport(tmpip, guessNewExtPort());
+
+					mNetFlags.mExtAddrOk = true;
+
+					address_votes[tmpip].n++ ;
+
+					/* XXX HACK TO FIX drbob: ALLOWING
+					 * ExtAddrFinder -> ExtAddrStableOk = true
+					 * (which it is not normally) */
+					mNetFlags.mExtAddrStableOk = true;
+
+					RS_DBG("Reported external IPv6 address ", sockaddr_storage_iptostring(tmpip));
 				}
 			}
 		}
@@ -967,7 +986,7 @@ void p3NetMgrIMPL::netExtCheck()
 
 	if (netSetupDone)
 	{
-		std::cerr << "p3NetMgrIMPL::netExtCheck() netSetupDone" << std::endl;
+		RS_DBG("netSetupDone");
 
 		/* Setup NetStateBox with this info */
 		updateNetStateBox_startup();
@@ -978,12 +997,14 @@ void p3NetMgrIMPL::netExtCheck()
 			mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
 		}
 
-		/* inform DHT about our external address */
+		/* inform DHT about our external IPV4 address, it doesn't support IPv6 for now.*/
+		if(sockaddr_storage_ipv6_to_ipv4(mExtAddr))
+		{
 			RsPeerId fakeId;
 			netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
+		}
 
-		std::cerr << __PRETTY_FUNCTION__ << " Network Setup Complete"
+		RS_INFO("Network Setup Complete");
-		          << std::endl;
 	}
 }
 
@@ -995,6 +1016,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 {
 	bool addrChanged = false;
 	bool validAddr = false;
+	bool needOwnAddrUpdate = false;
 	
 	sockaddr_storage prefAddr;
 	sockaddr_storage oldAddr;
@@ -1067,8 +1089,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 
 	if (!validAddr)
 	{
-		RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
+		RS_ERR("no valid local network address found. Report to developers.");
-		        <<" found. Report to developers." << std::endl;
 		print_stacktrace();
 
 		return false;
@@ -1109,6 +1130,7 @@ bool p3NetMgrIMPL::checkNetAddress()
             sockaddr_storage_setport(mExtAddr, port);	// this accounts for when the port was updated
             addrChanged = true;
 		}
+
 	} // RS_STACK_MUTEX(mNetMtx);
 
     if (addrChanged)
@@ -1123,11 +1145,24 @@ bool p3NetMgrIMPL::checkNetAddress()
             rsEvents->postEvent(ev);
         }
 
-        if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
+        needOwnAddrUpdate = true;
 
         netReset();
     }
 
+	if (mPeerMgr)
+	{
+		// Retrieve last known IP, if none, update own addresse to get current.
+		peerState ps;
+		mPeerMgr->getOwnNetStatus(ps);
+		needOwnAddrUpdate |= ps.ipAddrs.mLocal.mAddrs.empty();
+		needOwnAddrUpdate |= ps.ipAddrs.mExt.mAddrs.empty();
+		if (needOwnAddrUpdate)
+		{
+			mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
+		}
+	}
+
 	return true;
 }
 
@@ -1139,7 +1174,7 @@ bool p3NetMgrIMPL::checkNetAddress()
 /* to allow resets of network stuff */
 void    p3NetMgrIMPL::addNetListener(pqiNetListener *listener)
 {
-        RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+        RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
         mNetListeners.push_back(listener);
 }
 
@@ -1149,7 +1184,7 @@ bool    p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
 {
 	bool changed = false;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		if (!sockaddr_storage_same(mLocalAddr, addr))
 		{
 			changed = true;
@@ -1171,7 +1206,7 @@ bool    p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
 }
 bool    p3NetMgrIMPL::getExtAddress(struct sockaddr_storage& addr)
 {
-        RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+        RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
         if(mNetFlags.mExtAddrOk)
         {
@@ -1186,7 +1221,7 @@ bool    p3NetMgrIMPL::setExtAddress(const struct sockaddr_storage &addr)
 {
 	bool changed = false;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		if (!sockaddr_storage_same(mExtAddr, addr))
 		{
 			changed = true;
@@ -1220,7 +1255,7 @@ bool    p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
 {
 	uint32_t oldNetMode;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		/* only change TRY flags */
 
 		oldNetMode = mNetMode;
@@ -1266,7 +1301,7 @@ bool    p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
 
 bool    p3NetMgrIMPL::setVisState(uint16_t vs_disc, uint16_t vs_dht)
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	mVsDisc = vs_disc;
     mVsDht = vs_dht;
 
@@ -1628,7 +1663,7 @@ void	p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
 	uint32_t netsize = 0, rsnetsize = 0;
 	netAssistConnectStats(netsize, rsnetsize);
 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 	/* quick update of the stuff that can change! */
 	mNetFlags.mDhtOk = dhtOk;
@@ -1652,7 +1687,7 @@ void	p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
 
 bool  p3NetMgrIMPL::getIPServersEnabled() 
 { 
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mUseExtAddrFinder;
 }
 
@@ -1661,15 +1696,32 @@ void  p3NetMgrIMPL::getIPServersList(std::list<std::string>& ip_servers)
 	mExtAddrFinder->getIPServersList(ip_servers);
 }
 
+void  p3NetMgrIMPL::getCurrentExtIPList(std::list<std::string>& ip_list)
+{
+	ip_list.clear();
+	sockaddr_storage addr;
+	if(mExtAddrFinder->hasValidIPV4(addr))
+		ip_list.push_back(sockaddr_storage_iptostring(addr));
+	if(mExtAddrFinder->hasValidIPV6(addr))
+		ip_list.push_back(sockaddr_storage_iptostring(addr));
+}
+
 void p3NetMgrIMPL::setIPServersEnabled(bool b)
 {
+	if (mUseExtAddrFinder != b)
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		mExtAddrFinder->reset(true);
+		if (b)
+			mExtAddrFinder->start_request();
+	}
+
+	{
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		mUseExtAddrFinder = b;
 	}
 
 #ifdef NETMGR_DEBUG
-	std::cerr << "p3NetMgr: setIPServers to " << b << std::endl ; 
+	RS_DBG("set mUseExtAddrFinder to ", b);
 #endif
 
 }
@@ -1682,31 +1734,31 @@ void p3NetMgrIMPL::setIPServersEnabled(bool b)
 
 RsNetState p3NetMgrIMPL::getNetStateMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNetStateMode();
 }
 
 RsNetworkMode p3NetMgrIMPL::getNetworkMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNetworkMode();
 }
 
 RsNatTypeMode p3NetMgrIMPL::getNatTypeMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNatTypeMode();
 }
 
 RsNatHoleMode p3NetMgrIMPL::getNatHoleMode()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getNatHoleMode();
 }
 
 RsConnectModes p3NetMgrIMPL::getConnectModes()
 {
-	RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+	RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 	return mNetStateBox.getConnectModes();
 }
 
@@ -1730,7 +1782,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
         	/* input network bits */
        		if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
 		{
-			RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+			RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 			mNetStateBox.setAddressStunDht(tmpaddr, isstable);
 			
 #ifdef	NETMGR_DEBUG_STATEBOX
@@ -1749,7 +1801,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
         	/* input network bits */
        		if (mProxyStunner->getExternalAddr(tmpaddr, isstable))
 		{
-			RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+			RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 			mNetStateBox.setAddressStunProxy(tmpaddr, isstable);
 
 #ifdef	NETMGR_DEBUG_STATEBOX
@@ -1768,7 +1820,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
 		bool dhtOn = netAssistConnectEnabled();
 		bool dhtActive = netAssistConnectActive();
 
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 		mNetStateBox.setDhtState(dhtOn, dhtActive);
 	}
 
@@ -1777,7 +1829,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
 
 #ifdef	NETMGR_DEBUG_STATEBOX
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
         auto netstate = mNetStateBox.getNetStateMode();
         auto netMode = mNetStateBox.getNetworkMode();
@@ -1823,7 +1875,7 @@ void p3NetMgrIMPL::updateNatSetting()
 	RsNatTypeMode natType = RsNatTypeMode::UNKNOWN;
 	RsNatHoleMode natHole = RsNatHoleMode::UNKNOWN;
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		natType = mNetStateBox.getNatTypeMode();
 		natHole = mNetStateBox.getNatHoleMode();
@@ -1925,7 +1977,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 	std::cerr << std::endl;
 #endif
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		/* fill in the data */
 		struct sockaddr_storage tmpip;
@@ -1974,7 +2026,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 		if (mUseExtAddrFinder)
 		{
 			tmpip = mLocalAddr;
-            bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
+			bool extFinderOk = mExtAddrFinder->hasValidIPV4(tmpip);
 
 			if (extFinderOk)
 			{
@@ -2020,7 +2072,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
 void p3NetMgrIMPL::updateNetStateBox_reset()
 {
 	{
-		RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
+		RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
 
 		mNetStateBox.reset();
 

libretroshare/src/pqi/p3netmgr.h

@@ -120,6 +120,7 @@ virtual bool	shutdown() = 0; /* blocking shutdown call */
 virtual bool  getIPServersEnabled() = 0;
 virtual void  setIPServersEnabled(bool b)  = 0;
 virtual void  getIPServersList(std::list<std::string>& ip_servers)  = 0;
+virtual void  getCurrentExtIPList(std::list<std::string>& ip_list)  = 0;
 
 	// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
 virtual void 	getNetStatus(pqiNetStatus &status) = 0;
@@ -171,6 +172,7 @@ virtual bool	shutdown(); /* blocking shutdown call */
 virtual bool  getIPServersEnabled();
 virtual void  setIPServersEnabled(bool b);
 virtual void  getIPServersList(std::list<std::string>& ip_servers);
+virtual void  getCurrentExtIPList(std::list<std::string>& ip_list);
 
 	// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
 virtual void 	getNetStatus(pqiNetStatus &status);
@@ -313,6 +315,7 @@ void 	netStatusReset_locked();
 	bool     mStatusChanged;
 
 	bool     mUseExtAddrFinder;
+	rstime_t mNetExtAddrFinderTs;
 
 	/* network status flags (read by rsiface) */
 	pqiNetStatus mNetFlags;

libretroshare/src/pqi/pqissl.cc

@@ -1742,6 +1742,13 @@ bool 	pqissl::moretoread(uint32_t usec)
 	{
 		rslog(RSL_ALERT, pqisslzone, 
 			"pqissl::moretoread() Select ERROR!");
+		RS_WARN(strerror(errno));
+
+		if (errno == EBADF) {
+			// happens when SAM is shut down
+			rslog(RSL_ALERT, pqisslzone, "pqissl::moretoread() -> calling reset()");
+			reset_locked();
+		}
 		return 0;
 	}
 

libretroshare/src/pqi/pqissli2pbob.cpp

@@ -1,52 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/pqi: pqissli2pbob.cc                                      *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#include "pqissli2pbob.h"
-
-bool pqissli2pbob::connect_parameter(uint32_t type, const std::string &value)
-{
-	if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
-	{
-		RS_STACK_MUTEX(mSslMtx);
-		// a new line must be appended!
-		mI2pAddr = value + '\n';
-		return true;
-	}
-
-	return pqissl::connect_parameter(type, value);
-}
-
-int pqissli2pbob::Basic_Connection_Complete()
-{
-	int ret;
-
-	if ((ret = pqissl::Basic_Connection_Complete()) != 1)
-	{
-		// basic connection not complete.
-		return ret;
-	}
-
-	// send addr. (new line is already appended)
-	ret = send(sockfd, mI2pAddr.c_str(), mI2pAddr.length(), 0);
-	if (ret != (int)mI2pAddr.length())
-		return -1;
-	return 1;
-}

libretroshare/src/pqi/pqissli2pbob.h

@@ -1,52 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/pqi: pqissli2pbob.h                                       *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#ifndef PQISSLI2PBOB_H
-#define PQISSLI2PBOB_H
-
-#include "pqi/pqissl.h"
-
-/*
- * This class is a minimal varied version of pqissl to work with I2P BOB tunnels.
- * The only difference is that the [.b32].i2p addresses must be sent first.
- *
- * Everything else is untouched.
- */
-
-class pqissli2pbob : public pqissl
-{
-public:
-	pqissli2pbob(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
-	    : pqissl(l, parent, lm) {}
-
-	// NetInterface interface
-public:
-	bool connect_parameter(uint32_t type, const std::string &value);
-
-	// pqissl interface
-protected:
-	int Basic_Connection_Complete();
-
-private:
-	std::string mI2pAddr;
-};
-
-#endif // PQISSLI2PBOB_H

libretroshare/src/pqi/pqissli2psam3.cpp

@@ -0,0 +1,261 @@
+#include "pqissli2psam3.h"
+
+#include <libsam3.h>
+
+RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
+static constexpr int pqiDone  =  1;
+static constexpr int pqiWait  =  0;
+static constexpr int pqiError = -1;
+
+pqissli2psam3::pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
+    : pqissl(l, parent, lm), mState(pqisslSam3State::NONE), mI2pAddrB32(), mI2pAddrLong()
+{
+	RS_DBG4();
+	mConn = nullptr;
+}
+
+bool pqissli2psam3::connect_parameter(uint32_t type, const std::string &value)
+{
+	RS_DBG4();
+
+	if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
+	{
+		RS_DBG1("got addr:", value);
+		RS_STACK_MUTEX(mSslMtx);
+		mI2pAddrB32 = value;
+		return true;
+	}
+
+	return pqissl::connect_parameter(type, value);
+}
+
+int pqissli2psam3::Initiate_Connection()
+{
+	RS_DBG4();
+
+	if(waiting != WAITING_DELAY)
+	{
+		RS_ERR("Already Attempt in Progress!");
+		return pqiError;
+	}
+
+	switch (mState) {
+	case(pqisslSam3State::NONE):
+		RS_DBG2("NONE");
+	{
+		if(mConn) {
+			// how did we end up here?
+			RS_ERR("state is NONE but a connection is existing?!");
+		}
+		mConn = 0;
+		// get SAM session
+		mConn = 0;
+		samSettings ss;
+		ss.session = nullptr;
+		rsAutoProxyMonitor::taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, static_cast<void*>(&ss));
+
+		if (!!ss.session) {
+			RS_DBG3("NONE->DO_LOOKUP");
+			mState = pqisslSam3State::DO_LOOKUP;
+		} else {
+			RS_DBG3("NONE->DO_LOOKUP NOPE", ss.session);
+		}
+	}
+		break;
+	case(pqisslSam3State::DO_LOOKUP):
+		RS_DBG1("DO_LOOKUP");
+
+		if (!mI2pAddrLong.empty()) {
+			// skip lookup, it is highly unlikely/impossible for a public key to change (isn't it?)
+			mState = pqisslSam3State::WAIT_LOOKUP;
+			break;
+		}
+
+	{
+		i2p::address *addr = new i2p::address;
+		addr->clear();
+		addr->base32 = mI2pAddrB32;
+		rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::lookupKey, this, static_cast<void*>(addr));
+	}
+		mState = pqisslSam3State::WAIT_LOOKUP;
+		break;
+	case(pqisslSam3State::DO_CONNECT):
+		RS_DBG2("DO_CONNECT");
+
+	{
+		auto wrapper = new samEstablishConnectionWrapper();
+		wrapper->address.clear();
+		wrapper->address.publicKey = mI2pAddrLong;
+		wrapper->connection = nullptr;
+
+		rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::establishConnection, this, static_cast<void*>(wrapper));
+	}
+		mState = pqisslSam3State::WAIT_CONNECT;
+		break;
+	case(pqisslSam3State::DONE):
+		RS_DBG2("DONE");
+
+		if (setupSocket())
+			return pqiDone;
+		return pqiError;
+
+	/* waiting */
+	case(pqisslSam3State::WAIT_LOOKUP):
+		RS_DBG3("WAIT_LOOKUP");
+		break;
+	case(pqisslSam3State::WAIT_CONNECT):
+		RS_DBG3("WAIT_CONNECT");
+		break;
+	}
+	return pqiWait;
+}
+
+int pqissli2psam3::net_internal_close(int fd)
+{
+	RS_DBG4();
+
+	// sanity check
+	if (mConn && fd != mConn->fd) {
+		// this should never happen!
+		RS_ERR("fd != mConn");
+//		sam3CloseConnection(mConn);
+	}
+
+	// now to the actuall closing
+	int ret = pqissl::net_internal_close(fd);
+
+	rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::closeConnection, this, mConn);
+
+	// finally cleanup
+	mConn = 0;
+	mState = pqisslSam3State::NONE;
+
+	return ret;
+}
+
+void pqissli2psam3::taskFinished(taskTicket *&ticket)
+{
+	RS_DBG4();
+
+	switch (ticket->task) {
+	case autoProxyTask::lookupKey:
+	{
+		auto addr = static_cast<i2p::address*>(ticket->data);
+
+		RS_STACK_MUTEX(mSslMtx);
+		if (ticket->result == autoProxyStatus::ok) {
+			mI2pAddrLong = addr->publicKey;
+			mState = pqisslSam3State::DO_CONNECT;
+		} else {
+			waiting = WAITING_FAIL_INTERFACE;
+		}
+
+		delete addr;
+		ticket->data = nullptr;
+		addr = nullptr;
+	}
+		break;
+	case autoProxyTask::establishConnection:
+	{
+		auto wrapper = static_cast<struct samEstablishConnectionWrapper*>(ticket->data);
+
+		RS_STACK_MUTEX(mSslMtx);
+		if (ticket->result == autoProxyStatus::ok) {
+			mConn = wrapper->connection;
+			mState = pqisslSam3State::DONE;
+		} else {
+			waiting = WAITING_FAIL_INTERFACE;
+		}
+
+		delete wrapper;
+		ticket->data = nullptr;
+		wrapper = nullptr;
+	}
+		break;
+	case autoProxyTask::closeConnection:
+		// nothing to do here
+		break;
+	default:
+		RS_WARN("unkown task", ticket->task);
+	}
+
+	// clean up!
+	delete ticket;
+	ticket = nullptr;
+}
+
+bool pqissli2psam3::setupSocket()
+{
+	/*
+	 * This function contains the generis part from pqissl::Initiate_Connection()
+	 */
+	int err;
+	int osock = mConn->fd;
+
+	err = unix_fcntl_nonblock(osock);
+	if (err < 0)
+	{
+		RS_ERR("Cannot make socket NON-Blocking:", err);
+
+		waiting = WAITING_FAIL_INTERFACE;
+		net_internal_close(osock);
+		return false;
+	}
+
+#ifdef WINDOWS_SYS
+	/* Set TCP buffer size for Windows systems */
+
+	int sockbufsize = 0;
+	int size = sizeof(int);
+
+	err = getsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, &size);
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: Current TCP receive buffer size " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error getting TCP receive buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	sockbufsize = 0;
+
+	err = getsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, &size);
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: Current TCP send buffer size " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error getting TCP send buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	sockbufsize = WINDOWS_TCP_BUFFER_SIZE;
+
+	err = setsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, sizeof(sockbufsize));
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: TCP receive buffer size set to " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error setting TCP receive buffer size. Error " << err << std::endl;
+	}
+#endif
+
+	err = setsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, sizeof(sockbufsize));
+#ifdef PQISSL_DEBUG
+	if (err == 0) {
+		std::cerr << "pqissl::Initiate_Connection: TCP send buffer size set to " << sockbufsize << std::endl;
+	} else {
+		std::cerr << "pqissl::Initiate_Connection: Error setting TCP send buffer size. Error " << err << std::endl;
+	}
+#endif
+#endif // WINDOWS_SYS
+
+
+	mTimeoutTS = time(NULL) + mConnectTimeout;
+	//std::cerr << "Setting Connect Timeout " << mConnectTimeout << " Seconds into Future " << std::endl;
+
+	waiting = WAITING_SOCK_CONNECT;
+	sockfd = osock;
+
+	return true;
+}

libretroshare/src/pqi/pqissli2psam3.h

@@ -0,0 +1,47 @@
+#ifndef PQISSLI2PSAM3_H
+#define PQISSLI2PSAM3_H
+
+#include "pqi/pqissl.h"
+#include "services/autoproxy/rsautoproxymonitor.h"
+#include "services/autoproxy/p3i2psam3.h"
+
+// Use a state machine as the whole pqi code is designed around them and some operation (like lookup) might be blocking
+enum class pqisslSam3State : uint8_t {
+	NONE = 0,
+	DO_LOOKUP,
+	WAIT_LOOKUP,
+	DO_CONNECT,
+	WAIT_CONNECT,
+	DONE
+};
+
+class pqissli2psam3 : public pqissl, public autoProxyCallback
+{
+public:
+	pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm);
+
+	// NetInterface interface
+public:
+	bool connect_parameter(uint32_t type, const std::string &value);
+
+	// pqissl interface
+protected:
+	int Initiate_Connection();
+	int net_internal_close(int fd);
+
+	// autoProxyCallback interface
+public:
+	void taskFinished(taskTicket *&ticket);
+
+private:
+	bool setupSocket();
+
+private:
+	pqisslSam3State mState;
+	std::string mI2pAddrB32;
+	std::string mI2pAddrLong;
+
+	Sam3Connection *mConn;
+};
+
+#endif // PQISSLI2PSAM3_H

libretroshare/src/pqi/pqissllistener.h

@@ -186,8 +186,8 @@ public:
 	virtual int finaliseConnection(int fd, SSL *ssl, const RsPeerId& peerId,
 								   const sockaddr_storage &raddr);
 
+	RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
 private:
 	std::map<RsPeerId, pqissl*> listenaddr;
-
-	RS_SET_CONTEXT_DEBUG_LEVEL(2)
 };

libretroshare/src/pqi/pqisslpersongrp.cc

@@ -46,7 +46,7 @@ static struct RsLog::logInfo pqipersongrpzoneInfo = {RsLog::Default, "pqipersong
 #endif
 
 #include "pqi/pqisslproxy.h"
-#include "pqi/pqissli2pbob.h"
+#include "pqi/pqissli2psam3.h"
 
 pqilistener * pqisslpersongrp::locked_createListener(const struct sockaddr_storage &laddr)
 {
@@ -74,24 +74,26 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
         std::cerr << std::endl;
 #endif
 
-		// Use pqicI2PBOB for I2P
+		// Use pqicI2P for I2P
-		pqiconnect *pqicSOCKSProxy, *pqicI2PBOB;
+		pqiconnect *pqicSOCKSProxy, *pqicI2P;
 		{
 			pqisslproxy  *pqis = new pqisslproxy((pqissllistener *) listener, pqip, mLinkMgr);
 			RsSerialiser *rss  = new RsSerialiser();
 			rss->addSerialType(new RsRawSerialiser());
 			pqicSOCKSProxy = new pqiconnect(pqip, rss, pqis);
 		}
-		if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PBOB))
+#ifdef RS_USE_I2P_SAM3
+		if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PSAM3))
 		{
-			pqissli2pbob *pqis = new pqissli2pbob((pqissllistener *) listener, pqip, mLinkMgr);
+			pqissli2psam3 *pqis = new pqissli2psam3((pqissllistener *) listener, pqip, mLinkMgr);
 			RsSerialiser *rss  = new RsSerialiser();
 			rss->addSerialType(new RsRawSerialiser());
 
-			pqicI2PBOB = new pqiconnect(pqip, rss, pqis);
+			pqicI2P = new pqiconnect(pqip, rss, pqis);
-		} else {
-			pqicI2PBOB = pqicSOCKSProxy;
 		}
+		else
+#endif // def RS_USE_I2P_SAM3
+			pqicI2P = pqicSOCKSProxy;
 
 
 		/* first select type based on peer */
@@ -101,7 +103,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
 			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_TOR_TCP, pqicSOCKSProxy);
 			break;
 		case RS_HIDDEN_TYPE_I2P:
-			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
+			pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
 			break;
 		default:
 			/* peer is not a hidden one but we are */
@@ -109,7 +111,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
 			uint32_t typeOwn = mPeerMgr->getHiddenType(AuthSSL::getAuthSSL()->OwnId());
 			switch (typeOwn) {
 			case RS_HIDDEN_TYPE_I2P:
-				pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
+				pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
 				break;
 			default:
 				/* this case shouldn't happen! */

libretroshare/src/retroshare/rsevents.h

@@ -91,7 +91,7 @@ enum class RsEventType : uint32_t
     /// @see RsGxsPostedEvent
     GXS_IDENTITY                                            = 12,
 
-    /// @see RsFiles
+	/// @see RsFiles @deprecated
 	SHARED_DIRECTORIES                                      = 13,
 
     /// @see RsFiles
@@ -103,6 +103,9 @@ enum class RsEventType : uint32_t
 	/// @see rspeers.h
 	NETWORK                                                 = 16,
 
+	/** Emitted to update library clients about file hashing being completed */
+	FILE_HASHING_COMPLETED                                  = 20,
+
     /// @see rspeers.h
     TOR_MANAGER                                             = 17,
 

libretroshare/src/retroshare/rsfiles.h

@@ -194,7 +194,8 @@ enum class RsFileTransferEventCode: uint8_t {
     COMPLETED_FILES_REMOVED     = 0x02,	//
 };
 
-struct RsSharedDirectoriesEvent: RsEvent
+struct RS_DEPRECATED_FOR("Packing arbitrary data into an std::string is bad idea")
+RsSharedDirectoriesEvent: RsEvent
 {
 	RsSharedDirectoriesEvent()  : RsEvent(RsEventType::SHARED_DIRECTORIES), mEventCode(RsSharedDirectoriesEventCode::UNKNOWN) {}
 	~RsSharedDirectoriesEvent() override = default;
@@ -212,6 +213,31 @@ struct RsSharedDirectoriesEvent: RsEvent
     std::string mMessage;
 };
 
+struct RsFileHashingCompletedEvent: RsEvent
+{
+	RsFileHashingCompletedEvent():
+	    RsEvent(RsEventType::FILE_HASHING_COMPLETED), mHashingSpeed(0) {}
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+		RS_SERIAL_PROCESS(mFilePath);
+		RS_SERIAL_PROCESS(mFileHash);
+		RS_SERIAL_PROCESS(mHashingSpeed);
+	}
+
+	/// Complete path of the file being hashed
+	std::string mFilePath;
+
+	/// File hash, null if error occurred
+	RsFileHash mFileHash;
+
+	/// Hashing speed in MB/s
+	double mHashingSpeed;
+};
+
 struct RsFileTransferEvent: RsEvent
 {
 	RsFileTransferEvent()  : RsEvent(RsEventType::FILE_TRANSFER), mFileTransferEventCode(RsFileTransferEventCode::UNKNOWN) {}

libretroshare/src/retroshare/rsgxscircles.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
 enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
 {
 	UNKNOWN            = 0,	/// Used to detect uninizialized values.
-	PUBLIC             = 1,	/// Public distribution, based on GxsIds
+	PUBLIC             = 1,	/// Public distribution
 	EXTERNAL           = 2,	/// Restricted to an external circle, based on GxsIds
 
 	NODES_GROUP        = 3,	/// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

libretroshare/src/retroshare/rsgxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -118,6 +118,10 @@ enum class RsForumEventCode: uint8_t
 	SYNC_PARAMETERS_UPDATED  = 0x0a, /// sync and storage times have changed
 	PINNED_POSTS_CHANGED     = 0x0b, /// some posts where pinned or un-pinned
 	DELETED_FORUM            = 0x0c, /// forum was deleted by cleaning
+	DELETED_POST             = 0x0d,  /// Post deleted (usually by cleaning)
+
+	/// Distant search result received
+	DISTANT_SEARCH_RESULT    = 0x0e
 };
 
 struct RsGxsForumEvent: RsEvent
@@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
 		RS_SERIAL_PROCESS(mForumEventCode);
 		RS_SERIAL_PROCESS(mForumGroupId);
 		RS_SERIAL_PROCESS(mForumMsgId);
-		RS_SERIAL_PROCESS(mForumMsgId);
 		RS_SERIAL_PROCESS(mModeratorsAdded);
 		RS_SERIAL_PROCESS(mModeratorsRemoved);
 	}
@@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
 	~RsGxsForumEvent() override;
 };
 
+/** This event is fired once distant search results are received */
+struct RsGxsForumsDistantSearchEvent: RsEvent
+{
+	RsGxsForumsDistantSearchEvent():
+	    RsEvent(RsEventType::GXS_FORUMS),
+	    mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
+
+	RsForumEventCode mForumEventCode;
+	TurtleRequestId mSearchId;
+	std::vector<RsGxsSearchResult> mSearchResults;
+
+	///* @see RsEvent @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RsEvent::serial_process(j, ctx);
+
+		RS_SERIAL_PROCESS(mForumEventCode);
+		RS_SERIAL_PROCESS(mSearchId);
+		RS_SERIAL_PROCESS(mSearchResults);
+	}
+};
+
 class RsGxsForums: public RsGxsIfaceHelper
 {
 public:
@@ -385,6 +411,50 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) = 0;
 
+	/**
+	 * @brief Get forum content summaries
+	 * @jsonapi{development}
+	 * @param[in] forumId id of the forum of which the content is requested
+	 * @param[in] contentIds ids of requested contents, if empty summaries of
+	 *	all messages are reqeusted
+	 * @param[out] summaries storage for summaries
+	 * @return success or error details if something failed
+	 */
+	virtual std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) = 0;
+
+	/**
+	 * @brief Search the whole reachable network for matching forums and
+	 *	posts
+	 * @jsonapi{development}
+	 * An @see RsGxsForumsDistantSearchEvent is emitted when matching results
+	 *	arrives from the network
+	 * @param[in] matchString string to search into the forum and posts
+	 * @param[out] searchId storage for search id, useful to track search events
+	 *	and retrieve search results
+	 * @return success or error details
+	 */
+	virtual std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) = 0;
+
+	/**
+	 * @brief Search the local index for matching forums and posts
+	 * @jsonapi{development}
+	 * @param[in] matchString string to search into the index
+	 * @param[out] searchResults storage for searchr esults
+	 * @return success or error details
+	 */
+	virtual std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) = 0;
+
+
+	////////////////////////////////////////////////////////////////////////////
+	/* Following functions are deprecated and should not be considered a stable
+	 * to use API */
+
 	/**
 	 * @brief Create forum. Blocking API.
 	 * @jsonapi{development}

libretroshare/src/retroshare/rsgxsiface.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012  Christopher Evi-Parker                                  *
- * Copyright (C) 2019  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -31,13 +32,63 @@
 #include "rsitems/rsserviceids.h"
 #include "retroshare/rsevents.h"
 
+/*!
+ * This structure is used to transport GXS search results.
+ * It contains the group information as well as a context string to tell where
+ * the information was found.
+ * Keep it small as to make search responses as light as possible.
+ * It differs from RsGxsGroupSearchResults because it supports also results from
+ * message matches not just groups.
+ */
+struct RsGxsSearchResult : RsSerializable
+{
+	RsGxsSearchResult(): mPublishTs(0) {}
+
+	/** Id of the group which match*/
+	RsGxsGroupId mGroupId;
+
+	/** Title of the group which match */
+	std::string mGroupName;
+
+	/** Optional message id if the search match is against a message */
+	RsGxsMessageId mMsgId;
+
+	/** Optional message title if the search match is against a message */
+	std::string mMsgName;
+
+	/** Author id of the element which matched (group or message) */
+	RsGxsId mAuthorId;
+
+	/** Publish timestamp of the element which matched (group or message) */
+	rstime_t mPublishTs;
+
+	/** A snippet of content around the exact match */
+	std::string mSearchContext;
+
+	/// @see RsSerializable::serial_process
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mGroupId);
+		RS_SERIAL_PROCESS(mGroupName);
+		RS_SERIAL_PROCESS(mMsgId);
+		RS_SERIAL_PROCESS(mMsgName);
+		RS_SERIAL_PROCESS(mAuthorId);
+		RS_SERIAL_PROCESS(mPublishTs);
+		RS_SERIAL_PROCESS(mSearchContext);
+	}
+
+	virtual ~RsGxsSearchResult() = default;
+};
+
 /*!
  * This structure is used to transport group summary information when a GXS
  * service is searched. It contains the group information as well as a context
  * string to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSummary : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSummary : RsSerializable
 {
 	RsGxsGroupSummary() :
 	    mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
  * strings to tell where the information was found. It is more compact than a
  * GroupMeta object, so as to make search responses as light as possible.
  */
-struct RsGxsGroupSearchResults : RsSerializable
+struct RS_DEPRECATED_FOR(RsGxsSearchResult)
+RsGxsGroupSearchResults : RsSerializable
 {
+	/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
+	 * do we really need both? */
+
 	RsGxsGroupSearchResults()
         : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
     {}
@@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
 	virtual ~RsGxsGroupSearchResults() = default;
 };
 
+
 /*!
  * Stores ids of changed gxs groups and messages.
  * It is used to notify about GXS changes.

libretroshare/src/retroshare/rsgxsifacehelper.h

@@ -4,7 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2011  Christopher Evi-Parker                                  *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -208,6 +209,58 @@ public:
 		return mGxs.getGroupStatistic(token, stats);
 	}
 
+	/*!
+	 * @brief Get default maximum storage time for GXS messages
+	 * @jsonapi{development}
+	 * @return storage time in seconds
+	 */
+	uint32_t getDefaultStoragePeriod()
+	{ return mGxs.getDefaultStoragePeriod(); }
+
+	/*!
+	 * @brief Get maximum storage time of GXS messages for the given group
+	 * @jsonapi{development}
+	 * @param[in] groupId Id of the group
+	 * @return storage time in seconds
+	 */
+	uint32_t getStoragePeriod(const RsGxsGroupId& groupId)
+	{ return mGxs.getStoragePeriod(groupId); }
+
+	/*!
+	 * @brief Set GXS messages maximum storage time for the given group
+	 * @jsonapi{development}
+	 * @param[in] groupId Id of the group
+	 * @param[in] storageSecs storage time in seconds
+	 */
+	void setStoragePeriod(const RsGxsGroupId& groupId, uint32_t storageSecs)
+	{ mGxs.setStoragePeriod(groupId, storageSecs); }
+
+	/*!
+	 * @brief Get default maximum syncronization age for GXS messages
+	 * @jsonapi{development}
+	 * @return age in seconds
+	 */
+	uint32_t getDefaultSyncPeriod()
+	{ return mGxs.getDefaultSyncPeriod(); }
+
+	/*!
+	 * @brief Get maximum syncronization age of GXS messages for the given group
+	 * @jsonapi{development}
+	 * @param[in] groupId Id of the group
+	 * @return age in seconds
+	 */
+	uint32_t getSyncPeriod(const RsGxsGroupId& groupId)
+	{ return mGxs.getSyncPeriod(groupId); }
+
+	/*!
+	 * @brief Set GXS messages maximum syncronization age for the given group
+	 * @jsonapi{development}
+	 * @param[in] groupId Id of the group
+	 * @param[in] syncAge age in seconds
+	 */
+	void setSyncPeriod(const RsGxsGroupId& groupId, uint32_t syncAge)
+	{ mGxs.setSyncPeriod(groupId, syncAge); }
+
 	/*!
 	 * This determines the reputation threshold messages need to surpass in order
 	 * for it to be accepted by local user from remote source
@@ -221,34 +274,6 @@ public:
 		return mGxs.setGroupReputationCutOff(token, grpId, CutOff);
 	}
 
-    /*!
-     * @return storage/sync time of messages in secs
-     */
-    uint32_t getDefaultStoragePeriod()
-    {
-		return mGxs.getDefaultStoragePeriod();
-    }
-    uint32_t getStoragePeriod(const RsGxsGroupId& grpId)
-    {
-		return mGxs.getStoragePeriod(grpId);
-    }
-    void setStoragePeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs)
-    {
-		mGxs.setStoragePeriod(grpId,age_in_secs);
-    }
-    uint32_t getDefaultSyncPeriod()
-    {
-		return mGxs.getDefaultSyncPeriod();
-    }
-    uint32_t getSyncPeriod(const RsGxsGroupId& grpId)
-    {
-		return mGxs.getSyncPeriod(grpId);
-    }
-    void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs)
-    {
-		mGxs.setSyncPeriod(grpId,age_in_secs);
-    }
-
 	RsReputationLevel minReputationForForwardingMessages(
 	        uint32_t group_sign_flags, uint32_t identity_flags )
     {

libretroshare/src/retroshare/rsgxsifacetypes.h

@@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
 	    mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
 	    mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
 
-    virtual ~RsGroupMetaData() {}
+	virtual ~RsGroupMetaData() = default;
 
     void operator =(const RsGxsGrpMetaData& rGxsMeta);
     RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

libretroshare/src/retroshare/rsinit.h

@@ -197,7 +197,7 @@ public:
 	/*
 	 * Setup Hidden Location;
 	 */
-	static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob);
+	static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p);
 
 	static bool LoadPassword(const std::string& passwd) ;
 

libretroshare/src/retroshare/rsmsgs.h

@@ -906,8 +906,9 @@ virtual void getOwnAvatarData(unsigned char *& data,int& size) = 0 ;
 	 * @brief denyLobbyInvite deny a chat lobby invite
 	 * @jsonapi{development}
 	 * @param[in] id chat lobby id you were invited into
+     * @return true on success
      */
-	virtual void denyLobbyInvite(const ChatLobbyId &id) = 0 ;
+    virtual bool denyLobbyInvite(const ChatLobbyId &id) = 0 ;
 
 	/**
 	 * @brief getPendingChatLobbyInvites get a list of all pending chat lobby invites

libretroshare/src/retroshare/rspeers.h

@@ -788,6 +788,7 @@ public:
 	virtual bool setProxyServer(const uint32_t type, const std::string &addr, const uint16_t port) = 0;
 
 	virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
+	virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
 	virtual void allowServerIPDetermination(bool) = 0;
 	virtual bool resetOwnExternalAddressList() = 0;
 	virtual bool getAllowServerIPDetermination() = 0 ;

libretroshare/src/retroshare/rstypes.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2004-2006  Robert Fernie <retroshare@lunamutt.com>            *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -135,17 +137,10 @@ struct PeerBandwidthLimits : RsSerializable
 	}
 };
 
-//class SearchRequest // unused stuff.
-//{
-//	public:
-//	int searchId;
-//	RsCertId toId;  /* all zeros for everyone! */
-//	std::list<Condition> tests;
-//};
-
-
 /********************** For FileCache Interface *****************/
 
+/* G10h4ck: Having this kind of stuff on public headers is pretty dangerous for
+ * name space pollution, a C++11 enum class should be used instad ASAP */
 #define DIR_TYPE_UNKNOWN	    0x00
 #define DIR_TYPE_ROOT		0x01
 #define DIR_TYPE_PERSON  	0x02
@@ -258,6 +253,9 @@ struct DirStub : RsSerializable
 
 	uint8_t type;
 	std::string name;
+
+	/* G10h4ck do we still need to keep this as void* instead of uint64_t for
+	 * retroshare-gui sake? */
 	void* ref;
 
 	/// @see RsSerializable
@@ -267,29 +265,12 @@ struct DirStub : RsSerializable
 		RS_SERIAL_PROCESS(type);
 		RS_SERIAL_PROCESS(name);
 
-#if defined(__GNUC__) && !defined(__clang__)
+		/* Enforce serialization as uint64_t because void* changes size (usually
-#	pragma GCC diagnostic ignored "-Wstrict-aliasing"
+		 * 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
-#endif // defined(__GNUC__) && !defined(__clang__)
+		 * architectures and make JSON API behave inconsistenly. */
-		// (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
+		uint64_t handle = reinterpret_cast<uint64_t>(ref);
-        // preferable to force type serialization to the correct size rather than letting the compiler choose for us.
-        // /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
-
-		if(sizeof(ref) == 4)
-		{
-			std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
 		RS_SERIAL_PROCESS(handle);
-		}
+		ref = reinterpret_cast<void*>(handle);
-		else if(sizeof(ref) == 8)
-		{
-			std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
-			RS_SERIAL_PROCESS(handle);
-		}
-		else
-			std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#	pragma GCC diagnostic pop
-#endif // defined(__GNUC__) && !defined(__clang__)
 	}
 };
 
@@ -330,9 +311,9 @@ struct DirDetails : RsSerializable
 	void serial_process(RsGenericSerializer::SerializeJob j,
 	                    RsGenericSerializer::SerializeContext& ctx) override
 	{
-		/* Enforce serialization as uint64_t because void* changes size
+		/* Enforce serialization as uint64_t because void* changes size (usually
-		 * depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
+		 * 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
-		 */
+		 * architectures and make JSON API behave inconsistenly. */
 		uint64_t handle = reinterpret_cast<uint64_t>(ref);
 		RS_SERIAL_PROCESS(handle);
 		ref = reinterpret_cast<void*>(handle);

libretroshare/src/rsitems/rsgxsforumitems.h

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com>              *
+ * Copyright (C) 2012  Robert Fernie <retroshare@lunamutt.com>                 *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -19,8 +21,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RS_GXS_FORUM_ITEMS_H
+#pragma once
-#define RS_GXS_FORUM_ITEMS_H
 
 #include <map>
 
@@ -31,7 +32,18 @@
 
 #include "retroshare/rsgxsforums.h"
 
+enum class RsGxsForumsItems : uint8_t
+{
+	GROUP_ITEM     = 0x02,
+	MESSAGE_ITEM   = 0x03,
+	SEARCH_REQUEST = 0x04,
+	SEARCH_REPLY   = 0x05,
+};
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM   = 0x02;
+
+RS_DEPRECATED_FOR(RsGxsForumsItems)
 const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
 
 class RsGxsForumGroupItem : public RsGxsGrpItem
@@ -61,6 +73,48 @@ public:
 	RsGxsForumMsg mMsg;
 };
 
+struct RsGxsForumsSearchRequest : RsSerializable
+{
+	RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Store search match string
+	std::string mQuery;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mQuery);
+	}
+
+	~RsGxsForumsSearchRequest() override = default;
+};
+
+struct RsGxsForumsSearchReply : RsSerializable
+{
+	RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
+
+	/// Just for easier back and forward compatibility
+	RsGxsForumsItems mType;
+
+	/// Results storage
+	std::vector<RsGxsSearchResult> mResults;
+
+	/// @see RsSerializable
+	void serial_process( RsGenericSerializer::SerializeJob j,
+	                     RsGenericSerializer::SerializeContext& ctx ) override
+	{
+		RS_SERIAL_PROCESS(mType);
+		RS_SERIAL_PROCESS(mResults);
+	}
+
+	~RsGxsForumsSearchReply() override = default;
+};
+
 class RsGxsForumSerialiser : public RsServiceSerializer
 {
 public:
@@ -69,5 +123,3 @@ public:
 
 	virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
 };
-
-#endif /* RS_GXS_FORUM_ITEMS_H */

libretroshare/src/rsitems/rsitem.h

@@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
 	virtual void serial_process(RsGenericSerializer::SerializeJob,
 	                            RsGenericSerializer::SerializeContext&)// = 0;
 	{
-		std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
+		RS_ERR( "called by an item using new serialization system without "
-		          << "new serialization classes, but not derived! Class is "
+		        "overriding Class is: ", typeid(*this).name() );
-		          << typeid(*this).name() << std::endl;
 		print_stacktrace();
 	}
 

libretroshare/src/rsitems/rsserviceids.h

@@ -28,7 +28,9 @@
 
 enum class RsServiceType : uint16_t
 {
-	NONE                       = 0, /// To detect non-initialized reads
+	/// To detect non-initialized items
+	NONE                       = 0,
+
 	GOSSIP_DISCOVERY           = 0x0011,
 	CHAT                       = 0x0012,
 	MSG                        = 0x0013,
@@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
 	GWEMAIL_MAIL               = 0x0025,
 	SERVICE_CONTROL            = 0x0026,
 	DISTANT_CHAT               = 0x0027,
+
+	/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
 	GXS_TUNNEL                 = 0x0028,
+
 	BANLIST                    = 0x0101,
 	STATUS                     = 0x0102,
 	NXS                        = 0x0200,
@@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
 	POSTED                     = 0x0216,
 	CHANNELS                   = 0x0217,
 	GXSCIRCLE                  = 0x0218,
+
 	/// not gxs, but used with identities.
 	REPUTATION                 = 0x0219,
 	GXS_RECOGN                 = 0x0220,
@@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
 	CHANNELS_CONFIG            = 0x0317,
 	RTT                        = 0x1011, /// Round Trip Time
 
-
-	/***************** IDS ALLOCATED FOR PLUGINS ******************/
-	// 2000+
 	PLUGIN_ARADO_ID            = 0x2001,
 	PLUGIN_QCHESS_ID           = 0x2002,
 	PLUGIN_FEEDREADER          = 0x2003,
 
+	/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
+	GXS_DISTANT                = 0x2233,
+
 	/// Reserved for packet slicing probes.
 	PACKET_SLICING_PROBE       = 0xAABB,
 

libretroshare/src/rsserver/p3face.h

@@ -43,7 +43,7 @@
 
 class p3heartbeat;
 class p3discovery2;
-class p3I2pBob;
+class p3I2pSam3;
 
 /* GXS Classes - just declare the classes.
    so we don't have to totally recompile to switch */
@@ -161,8 +161,8 @@ public:
 		p3ChatService *chatSrv;
 		p3StatusService *mStatusSrv;
 		p3GxsTunnelService *mGxsTunnels;
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-		p3I2pBob *mI2pBob;
+		p3I2pSam3 *mI2pSam3;
 #endif
 
         // This list contains all threaded services. It will be used to shut them down properly.

libretroshare/src/rsserver/p3msgs.cc

@@ -530,9 +530,9 @@ bool p3Msgs::acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& gxs_id)
 {
     return mChatSrv->acceptLobbyInvite(id,gxs_id) ;
 }
-void p3Msgs::denyLobbyInvite(const ChatLobbyId& id) 
+bool p3Msgs::denyLobbyInvite(const ChatLobbyId& id)
 {
-	mChatSrv->denyLobbyInvite(id) ;
+    return mChatSrv->denyLobbyInvite(id) ;
 }
 void p3Msgs::getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) 
 {

libretroshare/src/rsserver/p3msgs.h

@@ -64,66 +64,66 @@ public:
 	  /*!
 	   * @param msgList ref to list summarising client's msgs
 	   */
-	  virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList);
+      virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList)override ;
-	  virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg);
+      virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg)override ;
-	  virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox);
+      virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox)override ;
 
 	RS_DEPRECATED_FOR(sendMail)
-	  virtual bool MessageSend(Rs::Msgs::MessageInfo &info);
+      virtual bool MessageSend(Rs::Msgs::MessageInfo &info)override ;
-	  virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag);
+      virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag)override ;
-	  virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId);
+      virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId)override ;
-	  virtual bool MessageToTrash(const std::string &mid, bool bTrash);
+      virtual bool MessageToTrash(const std::string &mid, bool bTrash)override ;
-	  virtual bool MessageDelete(const std::string &mid);
+      virtual bool MessageDelete(const std::string &mid)override ;
-	  virtual bool MessageRead(const std::string &mid, bool unreadByUser);
+      virtual bool MessageRead(const std::string &mid, bool unreadByUser)override ;
-	  virtual bool MessageReplied(const std::string &mid, bool replied);
+      virtual bool MessageReplied(const std::string &mid, bool replied)override ;
-	  virtual bool MessageForwarded(const std::string &mid, bool forwarded);
+      virtual bool MessageForwarded(const std::string &mid, bool forwarded)override ;
-	  virtual bool MessageStar(const std::string &mid, bool star);
+      virtual bool MessageStar(const std::string &mid, bool star)override ;
-	  virtual bool MessageJunk(const std::string &mid, bool junk);
+      virtual bool MessageJunk(const std::string &mid, bool junk)override ;
-	  virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load);
+      virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load)override ;
-	  virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId);
+      virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId)override ;
 
-	  virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags);
+      virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags)override ;
-	  virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color);
+      virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color)override ;
-	  virtual bool removeMessageTagType(uint32_t tagId);
+      virtual bool removeMessageTagType(uint32_t tagId)override ;
 
-	  virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info);
+      virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info)override ;
 	  /* set == false && tagId == 0 --> remove all */
-	  virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set);
+      virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set)override ;
 
-	  virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags);
+      virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags)override ;
 
-	  virtual uint32_t getDistantMessagingPermissionFlags() ;
+      virtual uint32_t getDistantMessagingPermissionFlags() override ;
-	  virtual void setDistantMessagingPermissionFlags(uint32_t flags) ;
+      virtual void setDistantMessagingPermissionFlags(uint32_t flags) override ;
 
 	  /*!
 	   * gets avatar from peer, image data in jpeg format
 	   */
-	  virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size);
+      virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size)override ;
 
 	  /*!
 	   * sets clients avatar, image data should be in jpeg format
 	   */
-	  virtual void setOwnAvatarData(const unsigned char *data,int size);
+      virtual void setOwnAvatarData(const unsigned char *data,int size)override ;
 
 	  /*!
 	   * retrieve clients avatar, image data in jpeg format
 	   */
-	  virtual void getOwnAvatarData(unsigned char *& data,int& size);
+      virtual void getOwnAvatarData(unsigned char *& data,int& size)override ;
 
 	  /*!
 	   * sets clients custom status (e.g. "i'm tired")
 	   */
-	  virtual void setCustomStateString(const std::string&  status_string) ;
+      virtual void setCustomStateString(const std::string&  status_string) override ;
 
 	  /*!
 	   * retrieves client's custom status
 	   */
-	  virtual std::string getCustomStateString() ;
+      virtual std::string getCustomStateString() override ;
 
 	  /*!
 	   * retrieves peer's custom status
 	   */
-	  virtual std::string getCustomStateString(const RsPeerId& peer_id) ;
+      virtual std::string getCustomStateString(const RsPeerId& peer_id) override ;
 	  
 
 	  /*!
@@ -132,56 +132,56 @@ public:
        * @param msg the message
        * @see ChatId
 	   */
-      virtual bool sendChat(ChatId destination, std::string msg) ;
+      virtual bool sendChat(ChatId destination, std::string msg) override ;
 
 	  /*!
 	   * Return the max message size for security forwarding
 	   */
-	  virtual uint32_t getMaxMessageSecuritySize(int type);
+      virtual uint32_t getMaxMessageSecuritySize(int type)override ;
 
     /*!
      * sends immediate status string to a specific peer, e.g. in a private chat
      * @param chat_id chat id to send status string to
      * @param status_string immediate status to send
      */
-    virtual void    sendStatusString(const ChatId& id, const std::string& status_string) ;
+    virtual void    sendStatusString(const ChatId& id, const std::string& status_string) override ;
 
     /**
      * @brief clearChatLobby: Signal chat was cleared by GUI.
      * @param id: Chat id cleared.
      */
-    virtual void clearChatLobby(const ChatId &id);
+    virtual void clearChatLobby(const ChatId &id)override ;
 
 	  /****************************************/
 
-      virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) ;
+      virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) override ;
-	  virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) ;
+      virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) override ;
-      virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) ;
+      virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) override ;
-      virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) ;
+      virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) override ;
-      virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) ;
+      virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) override ;
-      virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) ;
+      virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) override ;
-	  virtual void denyLobbyInvite(const ChatLobbyId& id) ;
+      virtual bool denyLobbyInvite(const ChatLobbyId& id) override ;
-	  virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
+      virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) override ;
-	  virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;
+      virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) override ;
-	  virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id);
+      virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id)override ;
-      virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) ;
+      virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) override ;
-      virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) ;
+      virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) override ;
-      virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) ;
+      virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) override ;
-      virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) ;
+      virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) override ;
-    virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe);
+    virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe)override ;
-    virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id);
+    virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id)override ;
-      virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) ;
+      virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) override ;
 
 	virtual bool initiateDistantChatConnexion(
 		          const RsGxsId& to_gxs_id, const RsGxsId& from_gxs_id,
 		          DistantChatPeerId &pid, uint32_t& error_code,
-		          bool notify = true );
+                  bool notify = true )override ;
 
-      virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info);
+      virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info)override ;
-      virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) ;
+      virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) override ;
 
-    virtual uint32_t getDistantChatPermissionFlags() ;
+    virtual uint32_t getDistantChatPermissionFlags() override ;
-    virtual bool setDistantChatPermissionFlags(uint32_t flags) ;
+    virtual bool setDistantChatPermissionFlags(uint32_t flags) override ;
     
    private:
 

libretroshare/src/rsserver/p3peers.cc

@@ -373,7 +373,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
 			sockaddr_storage_ipv6_to_ipv4(it->mAddr);
 			std::string toto;
 			toto += sockaddr_storage_tostring(it->mAddr);
-			rs_sprintf_append(toto, "    %ld sec", time(NULL) - it->mSeenTime);
+			rs_sprintf_append(toto, "    %ld sec loc", time(NULL) - it->mSeenTime);
 			d.ipAddressList.push_back(toto);
 		}
 		for(it = ps.ipAddrs.mExt.mAddrs.begin(); it != ps.ipAddrs.mExt.mAddrs.end(); ++it)
@@ -381,7 +381,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
 			sockaddr_storage_ipv6_to_ipv4(it->mAddr);
 			std::string toto;
 			toto += sockaddr_storage_tostring(it->mAddr);
-			rs_sprintf_append(toto, "    %ld sec", time(NULL) - it->mSeenTime);
+			rs_sprintf_append(toto, "    %ld sec ext", time(NULL) - it->mSeenTime);
 			d.ipAddressList.push_back(toto);
 		}
 	}
@@ -859,8 +859,14 @@ void p3Peers::getIPServersList(std::list<std::string>& ip_servers)
 {
 	mNetMgr->getIPServersList(ip_servers) ;
 }
+void p3Peers::getCurrentExtIPList(std::list<std::string>& ip_list)
+{
+	mNetMgr->getCurrentExtIPList(ip_list) ;
+}
 bool p3Peers::resetOwnExternalAddressList()
 {
+	//TODO Phenom 2021-10-30: Need to call something like mNetMgr->netReset();
+	// to update this addresslist.
 	return mPeerMgr->resetOwnExternalAddressList();
 }
 void p3Peers::allowServerIPDetermination(bool b)
@@ -1287,7 +1293,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
         }
 #else
         sockaddr_storage tLocal;
-        if(sockaddr_storage_inet_pton(tLocal, tDetails.localAddr) && sockaddr_storage_isValidNet(tLocal)  && sockaddr_storage_ipv6_to_ipv4(tLocal) && tDetails.localPort )
+        bool validLoc =   sockaddr_storage_inet_pton(tLocal, tDetails.localAddr)
+                       && sockaddr_storage_isValidNet(tLocal)
+                       && tDetails.localPort;
+        bool isLocIpv4 = sockaddr_storage_ipv6_to_ipv4(tLocal);
+        if(validLoc && isLocIpv4)
         {
             uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tLocal).sin_addr.s_addr;
 
@@ -1305,7 +1315,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
         }
 
         sockaddr_storage tExt;
-        if(sockaddr_storage_inet_pton(tExt, tDetails.extAddr) && sockaddr_storage_isValidNet(tExt)  && sockaddr_storage_ipv6_to_ipv4(tExt) && tDetails.extPort )
+        bool validExt =   sockaddr_storage_inet_pton(tExt, tDetails.extAddr)
+                       && sockaddr_storage_isValidNet(tExt)
+                       && tDetails.extPort;
+        bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
+        if(validExt && isExtIpv4)
         {
             uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tExt).sin_addr.s_addr;
 
@@ -1321,6 +1335,17 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
 
             offset += 4+2;
         }
+        else if(validExt && !isExtIpv4)
+        {
+            // External address is IPv6, save it on LOCATOR
+            sockaddr_storage_setport(tExt,tDetails.extPort);
+            std::string tLocator = sockaddr_storage_tostring(tExt);
+
+            addPacketHeader(RsShortInviteFieldType::LOCATOR, tLocator.size(),buf,offset,buf_size);
+            memcpy(&buf[offset],tLocator.c_str(),tLocator.size());
+
+            offset += tLocator.size();
+        }
 #endif
 
     }
@@ -1595,9 +1620,27 @@ std::string p3Peers::GetRetroshareInvite( const RsPeerId& sslId, RetroshareInvit
 
 	if (getPeerDetails(ssl_id, detail))
 	{
-        if(!(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY) || detail.isHiddenNode)
+		if(   !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
+		   || detail.isHiddenNode)
 			detail.ipAddressList.clear();
 
+		//Check if external address is IPv6, then move it to ipAddressList as RsCertificate only allow 4 numbers.
+		sockaddr_storage tExt;
+		bool validExt =   sockaddr_storage_inet_pton(tExt, detail.extAddr)
+		               && sockaddr_storage_isValidNet(tExt)
+		               && detail.extPort;
+		bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
+
+		if(   !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
+		   && !detail.isHiddenNode
+		   && validExt && !isExtIpv4)
+		{
+			sockaddr_storage_setport(tExt,detail.extPort);
+			detail.ipAddressList.push_front(sockaddr_storage_tostring(tExt) + " "); // Space needed to later parse.
+			detail.extAddr = ""; //Clear it to not trigg error.
+			detail.extPort = 0;
+		}
+
 		unsigned char *mem_block = nullptr;
 		size_t mem_block_size = 0;
 

libretroshare/src/rsserver/p3peers.h

@@ -132,6 +132,7 @@ public:
 	virtual bool isProxyAddress(const uint32_t type, const sockaddr_storage &addr);
 
     virtual void getIPServersList(std::list<std::string>& ip_servers) override;
+    virtual void getCurrentExtIPList(std::list<std::string>& ip_list) override;
     virtual void allowServerIPDetermination(bool) override;
     virtual bool getAllowServerIPDetermination() override;
     virtual bool resetOwnExternalAddressList() override;

libretroshare/src/rsserver/rsinit.cc

@@ -2,7 +2,8 @@
  * libretroshare/src/retroshare: rsinit.cc                                     *
  *                                                                             *
  * Copyright (C) 2004-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2016-2019  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2016-2021  Gioacchino Mazzurco <gio@altermundi.net>           *
+ * Copyright (C) 2021       Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -173,7 +174,7 @@ struct RsInitConfig
 		std::string hiddenNodeAddress;
 		uint16_t    hiddenNodePort;
 
-		bool        hiddenNodeI2PBOB;
+		bool        hiddenNodeI2P;
 
 		/* Logging */
 		bool haveLogFile;
@@ -667,13 +668,13 @@ void RsInit::setAutoLogin(bool autoLogin){
 }
 
 /* Setup Hidden Location; */
-void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob)
+void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p)
 {
 	/* parse the bugger (todo) */
 	rsInitConfig->hiddenNodeSet = true;
 	rsInitConfig->hiddenNodeAddress = hiddenaddress;
 	rsInitConfig->hiddenNodePort = port;
-	rsInitConfig->hiddenNodeI2PBOB = useBob;
+	rsInitConfig->hiddenNodeI2P = useI2p;
 }
 
 
@@ -720,7 +721,7 @@ RsGRouter *rsGRouter = NULL ;
 #	include "rs_upnp/upnphandler_miniupnp.h"
 #endif // def RS_USE_LIBUPNP
 
-#include "services/autoproxy/p3i2pbob.h"
+#include "services/autoproxy/p3i2psam3.h"
 #include "services/autoproxy/rsautoproxymonitor.h"
 
 #include "services/p3gxsreputation.h"
@@ -926,9 +927,9 @@ int RsServer::StartupRetroShare()
 	mNetMgr->setManagers(mPeerMgr, mLinkMgr);
 
 	rsAutoProxyMonitor *autoProxy = rsAutoProxyMonitor::instance();
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	mI2pBob = new p3I2pBob(mPeerMgr);
+	mI2pSam3 = new p3I2pSam3(mPeerMgr);
-	autoProxy->addProxy(autoProxyType::I2PBOB, mI2pBob);
+	autoProxy->addProxy(autoProxyType::I2PSAM3, mI2pSam3);
 #endif
 
 	//load all the SSL certs as friends
@@ -1339,23 +1340,27 @@ int RsServer::StartupRetroShare()
     mWiki->setNetworkExchangeService(wiki_ns) ;
 #endif
 
-        /**** Forum GXS service ****/
+	/************************* Forum GXS service ******************************/
 
-        RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
+	RsGeneralDataService* gxsforums_ds = new RsDataService(
-                                                            RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
+	            currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
+	            nullptr, rsInitConfig->gxs_passwd );
 
+	p3GxsForums* mGxsForums = new p3GxsForums(
+	            gxsforums_ds, nullptr, mGxsIdService );
 
-        p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
+	RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
+#ifdef RS_DEEP_FORUMS_INDEX
+	gxsForumsTunnelService = mGxsNetTunnel;
+#endif
 
-        // create GXS photo service
 	RsGxsNetService* gxsforums_ns = new RsGxsNetService(
-                        RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
+	            RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
-                        mGxsForums, mGxsForums->getServiceInfo(),
+	            mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
-			mReputations, mGxsCircles,mGxsIdService,
+	            mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
-		    pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
-
 	mGxsForums->setNetworkExchangeService(gxsforums_ns);
 
+
         /**** Channel GXS service ****/
 
         RsGeneralDataService* gxschannels_ds = new RsDataService(currGxsDir + "/", "gxschannels_db",
@@ -1602,6 +1607,9 @@ int RsServer::StartupRetroShare()
     // Turtle search for GXS services
 
 	mGxsNetTunnel->registerSearchableService(gxschannels_ns);
+#ifdef RS_DEEP_FORUMS_INDEX
+	mGxsNetTunnel->registerSearchableService(gxsforums_ns);
+#endif
 
 	/**************************************************************************/
 
@@ -1658,8 +1666,9 @@ int RsServer::StartupRetroShare()
 	mConfigMgr->addConfiguration("wire.cfg", wire_ns);
 #endif
 #endif //RS_ENABLE_GXS
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pBob);
+	// to make migration easiert, SAM will use BOBs configuration, as they are compatible / the same.
+	mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pSam3);
 #endif
 
 	mPluginsManager->addConfigurations(mConfigMgr) ;
@@ -1712,34 +1721,33 @@ int RsServer::StartupRetroShare()
 	{
 		std::cout << "RsServer::StartupRetroShare setting up hidden locations" << std::endl;
 
-		if (rsInitConfig->hiddenNodeI2PBOB) {
+		if (rsInitConfig->hiddenNodeI2P) {
-			std::cout << "RsServer::StartupRetroShare setting up BOB" << std::endl;
+			std::cout << "RsServer::StartupRetroShare setting up SAMv3" << std::endl;
 
 			// we need a local port!
 			mNetMgr->checkNetAddress();
 
 			// add i2p proxy
-			// bob will use this address
 			sockaddr_storage i2pInstance;
 			sockaddr_storage_ipv4_aton(i2pInstance, rsInitConfig->hiddenNodeAddress.c_str());
 			mPeerMgr->setProxyServerAddress(RS_HIDDEN_TYPE_I2P, i2pInstance);
 
 			std::string addr; // will be set by auto proxy service
-			uint16_t port = rsInitConfig->hiddenNodePort; // unused by bob
+			uint16_t port; // unused by SAM
 
-			bool r = autoProxy->initialSetup(autoProxyType::I2PBOB, addr, port);
+			bool r = autoProxy->initialSetup(autoProxyType::I2PSAM3, addr, port);
 
 			if (r && !addr.empty()) {
 				mPeerMgr->setupHiddenNode(addr, port);
 
-				// now enable bob
+				// now enable SAM
-				bobSettings bs;
+				samSettings ss;
-				autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::getSettings, &bs);
+				autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, &ss);
-				bs.enable = true;
+				ss.enable = true;
-				autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &bs);
+				autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::setSettings, &ss);
 			} else {
 				std::cerr << "RsServer::StartupRetroShare failed to receive keys" << std::endl;
-				/// TODO add notify for failed bob setup
+				/// TODO add notify for failed i2p setup
 			}
 		} else {
 			mPeerMgr->setupHiddenNode(rsInitConfig->hiddenNodeAddress, rsInitConfig->hiddenNodePort);
@@ -1761,19 +1769,17 @@ int RsServer::StartupRetroShare()
 	if (rsInitConfig->hiddenNodeSet) {
 		// newly created location
 		// mNetMgr->checkNetAddress() will setup ports for us
+
+#if 0 // this was used for BOB but is not requires for SAMv3
 		// trigger updates for auto proxy services
 		std::vector<autoProxyType::autoProxyType_enum> types;
-
-		// i2p bob need to rebuild its command map
-		types.push_back(autoProxyType::I2PBOB);
-
 		rsAutoProxyMonitor::taskSync(types, autoProxyTask::reloadConfig);
+#endif
 	}
 
 	/**************************************************************************/
 	/* startup (stuff dependent on Ids/peers is after this point) */
 	/**************************************************************************/
-
 	autoProxy->startAll();
 
 	pqih->init_listener();
@@ -1806,8 +1812,8 @@ int RsServer::StartupRetroShare()
 	/**************************************************************************/
 
 	// auto proxy threads
-#ifdef RS_USE_I2P_BOB
+#ifdef RS_USE_I2P_SAM3
-	startServiceThread(mI2pBob, "I2P-BOB");
+	startServiceThread(mI2pSam3, "I2P-SAM3");
 #endif
 
 #ifdef RS_ENABLE_GXS

libretroshare/src/services/autoproxy/p3i2pbob.h

@@ -1,256 +0,0 @@
-/*******************************************************************************
- * libretroshare/src/services/autoproxy: p3i2pbob.h                            *
- *                                                                             *
- * libretroshare: retroshare core library                                      *
- *                                                                             *
- * Copyright 2016 by Sehraf                                                    *
- *                                                                             *
- * This program is free software: you can redistribute it and/or modify        *
- * it under the terms of the GNU Lesser General Public License as              *
- * published by the Free Software Foundation, either version 3 of the          *
- * License, or (at your option) any later version.                             *
- *                                                                             *
- * This program is distributed in the hope that it will be useful,             *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
- * GNU Lesser General Public License for more details.                         *
- *                                                                             *
- * You should have received a copy of the GNU Lesser General Public License    *
- * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
- *                                                                             *
- *******************************************************************************/
-#ifndef P3I2PBOB_H
-#define P3I2PBOB_H
-
-#include <map>
-#include <queue>
-#include <sys/types.h>
-#include "util/rstime.h"
-#ifndef WINDOWS_SYS
-	#include <sys/socket.h>
-#endif
-
-#include "pqi/p3cfgmgr.h"
-#include "services/autoproxy/rsautoproxymonitor.h"
-#include "util/rsthreads.h"
-#include "util/i2pcommon.h"
-
-/*
- * This class implements I2P BOB (BASIC OPEN BRIDGE) communication to allow RS
- * to automatically remote control I2P to setup the needed tunnel.
- * BOB is a simple text-based interface: https://geti2p.net/en/docs/api/bob
- *
- * Note 1:
- *	One tunnel is enough even for hidden locations since it can be used
- *	bidirectional. (In contrast to what RS I2P users had to set up manually.)
- *
- * Note 2:
- *	BOB tunnels are no SOCKS tunnel. Therefore pqissli2pbob implements a simplified
- *	proxy specially for BOB tunnels.
- *
- * Note 3:
- *	BOB needs a unique name as an ID for each tunnel.
- *	We use 'RetroShare-' + 8 random base32 characters.
- *
- * Design:
- *	The service uses three state machines to manage its task:
- *		int stateMachineBOB();
- *			mBOBState
- *		int stateMachineController();
- *			mState
- *			mTask
- *
- * stateMachineBOB:
- *	This state machine manages the low level communication with BOB. It basically has a linked
- *	list (currently a implemented as a std::map) that contains a command and the next
- *	state.
- *	Each high level operation (start up / shut down / get keys) is represented by a
- *	chain of states. E.g. the chain to retrieve new keys:
- *		mCommands[bobState::setnickN] = {setnick, bobState::newkeysN};
- *		mCommands[bobState::newkeysN] = {newkeys, bobState::getkeys};
- *		mCommands[bobState::getkeys]  = {getkeys, bobState::clear};
- *		mCommands[bobState::clear]    = {clear,   bobState::quit};
- *		mCommands[bobState::quit]     = {quit,    bobState::cleared};
- *
- * stateMachineController:
- *	This state machine manages the high level tasks.
- *	It is controlled by mState and mTask.
- *
- *		mTast:
- *			Tracks the high level operation (like start up).
- *			It will keep its value even when a task is done to track
- *			the requested BOB state.
- *			When other operations are performed like a conection check
- *			the last task gets backed up and is later restored again
- *
- *		mState:
- *			This state lives only for one operation an manages the communication
- *			with the BOB instance. This is basically connecting, starting BOB
- *			protocol and disconnecting
- *
- * How a task looks like:
- *	1) RS sets task using the ticket system
- *	2) stateMachineController connects to BOBs control port, sets mBobState to a lists head
- *	3) stateMachineBOB processes command chain
- *	4) stateMachineBOB is done and sets mBobState to cleared signaling that the connection
- *		is cleared and can be closed
- *	5) stateMachineController disconnects from BOBs control port and updates mState
- */
-
-///
-/// \brief The controllerState enum
-/// States for the controller to keep track of what he is currently doing
-enum controllerState {
-	csIdel,
-	csDoConnect,
-	csConnected,
-	csWaitForBob,
-	csDoDisconnect,
-	csDisconnected,
-	csError
-};
-
-///
-/// \brief The controllerTask enum
-/// This state tracks the controllers tast (e.g. setup a BOB tunnel or shut down
-/// an existing one).
-enum controllerTask {
-	ctIdle,
-	ctRunSetUp,
-	ctRunShutDown,
-	ctRunGetKeys,
-	ctRunCheck
-};
-
-///
-/// \brief The bobState enum
-/// One state for each message
-///
-enum bobState {
-	bsCleared,
-	bsSetnickC, // chain head for only client tunnel
-	bsSetnickN, // chain head for getting new (server) keys
-	bsSetnickS, // chain head for client and server tunnel
-	bsGetnick,
-	bsNewkeysC, // part of chain for only client tunnel
-	bsNewkeysN, // part of chain for getting new (server) keys
-	bsGetkeys,
-	bsSetkeys,
-	bsInhost,
-	bsOuthost,
-	bsInport,
-	bsOutport,
-	bsInlength,
-	bsOutlength,
-	bsInvariance,
-	bsOutvariance,
-	bsInquantity,
-	bsOutquantity,
-	bsQuiet,
-	bsStart,
-	bsStop,
-	bsClear,
-	bsList, // chain head for 'list' command
-	bsQuit
-};
-
-///
-/// \brief The bobStateInfo struct
-/// State machine with commands
-/// \todo This could be replaced by a linked list instead of a map
-struct bobStateInfo {
-	std::string command;
-	bobState    nextState;
-};
-
-struct bobSettings : i2p::settings {};
-
-///
-/// \brief The bobStates struct
-/// This container struct is used to pass all states.
-/// Additionally, the tunnel name is included to to show it in the GUI.
-/// The advantage of a struct is that it can be forward declared.
-struct bobStates {
-	bobState bs;
-	controllerState cs;
-	controllerTask ct;
-
-	std::string tunnelName;
-};
-
-class p3PeerMgr;
-
-class p3I2pBob : public RsTickingThread, public p3Config, public autoProxyService
-{
-public:
-	explicit p3I2pBob(p3PeerMgr *peerMgr);
-
-	// autoProxyService interface
-public:
-	bool isEnabled();
-	bool initialSetup(std::string &addr, uint16_t &);
-	void processTaskAsync(taskTicket *ticket);
-	void processTaskSync(taskTicket *ticket);
-
-	void threadTick() override; /// @see RsTickingThread
-
-private:
-	int stateMachineBOB();
-	int stateMachineBOB_locked_failure(const std::string &answer, const bobStateInfo &currentState);
-
-	int stateMachineController();
-	int stateMachineController_locked_idle();
-	int stateMachineController_locked_connected();
-	int stateMachineController_locked_disconnected();
-	int stateMachineController_locked_error();
-
-	// p3Config interface
-protected:
-	RsSerialiser *setupSerialiser();
-	bool saveList(bool &cleanup, std::list<RsItem *> &lst);
-	bool loadList(std::list<RsItem *> &load);
-
-private:
-	// helpers
-	void getBOBSettings(bobSettings *settings);
-	void setBOBSettings(const bobSettings *settings);
-	void getStates(bobStates *bs);
-
-	std::string executeCommand(const std::string &command);
-	bool connectI2P();
-	bool disconnectI2P();
-
-	void finalizeSettings_locked();
-	void updateSettings_locked();
-
-	std::string recv();
-
-	// states for state machines
-	controllerState mState;
-	controllerTask	mTask;
-	// used to store old state when in error state
-	// mStateOld is also used as a flag when an error occured in BOB protocol
-	controllerState mStateOld;
-	// mTaskOld is used to keep the previous task (start up / shut down) when requesting keys or checking the connection
-	controllerTask	mTaskOld;
-	bobSettings mSetting;
-	bobState	mBOBState;
-
-	// used variables
-	p3PeerMgr *mPeerMgr;
-	bool mConfigLoaded;
-	int mSocket;
-	rstime_t mLastProxyCheck;
-	sockaddr_storage mI2PProxyAddr;
-	std::map<bobState, bobStateInfo> mCommands;
-	std::string mErrorMsg;
-	std::string mTunnelName;
-
-	std::queue<taskTicket *> mPending;
-	taskTicket *mProcessing;
-
-	// mutex
-	RsMutex mLock;
-};
-
-#endif // P3I2PBOB_H

libretroshare/src/services/autoproxy/p3i2psam3.cpp

@@ -0,0 +1,798 @@
+#include "p3i2psam3.h"
+
+#include <libsam3.h>
+
+#include "pqi/p3peermgr.h"
+#include "rsitems/rsconfigitems.h"
+
+
+static const std::string kConfigKeySAM3Enable  = "SAM3_ENABLE";
+
+static const std::string kConfigKeyDestPriv    = "DEST_PRIV";
+
+static const std::string kConfigKeyInLength    = "IN_LENGTH";
+static const std::string kConfigKeyInQuantity  = "IN_QUANTITY";
+static const std::string kConfigKeyInVariance  = "IN_VARIANCE";
+static const std::string kConfigKeyInBackupQuantity = "IN_BACKUPQUANTITY";
+
+static const std::string kConfigKeyOutLength   = "OUT_LENGTH";
+static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY";
+static const std::string kConfigKeyOutVariance = "OUT_VARIANCE";
+static const std::string kConfigKeyOutBackupQuantity = "OUT_BACKUPQUANTITY";
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+// used for migration from BOB to SAM
+static const std::string kConfigKeyBOBEnable   = "BOB_ENABLE";
+static const std::string kConfigKeyBOBKey      = "BOB_KEY";
+static const std::string kConfigKeyBOBAddr     = "BOB_ADDR";
+#endif
+
+static constexpr bool   kDefaultSAM3Enable = false;
+
+RS_SET_CONTEXT_DEBUG_LEVEL(2)
+
+static void inline doSleep(std::chrono::duration<long, std::ratio<1,1000>> timeToSleepMS) {
+	std::this_thread::sleep_for(timeToSleepMS);
+}
+
+p3I2pSam3::p3I2pSam3(p3PeerMgr *peerMgr) :
+    mConfigLoaded(false), mPeerMgr(peerMgr), mPending(), mLock("p3i2p-sam3")
+#ifdef RS_USE_I2P_SAM3_LIBSAM3
+  , mLockSam3Access("p3i2p-sam3-access")
+#endif
+{
+	RS_DBG4();
+
+	// set defaults
+	mSetting.initDefault();
+	mSetting.enable  = kDefaultSAM3Enable;
+	mSetting.session = nullptr;
+
+	libsam3_debug = 0;
+}
+
+bool p3I2pSam3::isEnabled()
+{
+	RS_STACK_MUTEX(mLock);
+	return mSetting.enable;
+}
+
+bool p3I2pSam3::initialSetup(std::string &addr, uint16_t &/*port*/)
+{
+	RS_DBG4();
+
+	RS_STACK_MUTEX(mLock);
+
+	if (!mSetting.address.publicKey.empty() || !mSetting.address.privateKey.empty())
+		RS_WARN("overwriting keys!");
+
+	bool success = generateKey(mSetting.address.publicKey, mSetting.address.privateKey);
+
+	if (!success) {
+		RS_WARN("failed to retrieve keys");
+		return false;
+	} else {
+		std::string s, c;
+		i2p::getKeyTypes(mSetting.address.publicKey, s, c);
+		RS_INFO("received key ", s, " ", c);
+		RS_INFO("public key:      ", mSetting.address.publicKey);
+		RS_INFO("private key:     ", mSetting.address.privateKey);
+		RS_INFO("address:         ", i2p::keyToBase32Addr(mSetting.address.publicKey));
+
+		// sanity check
+		auto pub = i2p::publicKeyFromPrivate(mSetting.address.privateKey);
+		RS_INFO("pub key derived: ", pub);
+		RS_INFO("address:         ", i2p::keyToBase32Addr(pub));
+		if (pub != mSetting.address.publicKey) {
+			RS_WARN("public key does not match private key! fixing ...");
+			mSetting.address.publicKey = pub;
+		}
+
+		mSetting.address.base32 = i2p::keyToBase32Addr(mSetting.address.publicKey);
+
+		IndicateConfigChanged();
+	}
+
+	addr = mSetting.address.base32;
+	return true;
+}
+
+void p3I2pSam3::processTaskAsync(taskTicket *ticket)
+{
+	RS_DBG4();
+
+	switch (ticket->task) {
+	case autoProxyTask::stop: [[fallthrough]];
+	case autoProxyTask::start: [[fallthrough]];
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey: [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck: [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
+	{
+		RS_STACK_MUTEX(mLock);
+		mPending.push(ticket);
+	}
+		break;
+	case autoProxyTask::status: [[fallthrough]];
+	case autoProxyTask::getSettings: [[fallthrough]];
+	case autoProxyTask::setSettings: [[fallthrough]];
+	case autoProxyTask::getErrorInfo: [[fallthrough]];
+	case autoProxyTask::reloadConfig:
+		// These are supposed to be sync!
+		RS_DBG("unknown task or sync one!");
+		rsAutoProxyMonitor::taskError(ticket);
+		break;
+	}
+}
+
+void p3I2pSam3::processTaskSync(taskTicket *ticket)
+{
+//	RS_DBG4();
+
+	const bool data = !!ticket->data;
+
+	switch (ticket->task) {
+	case autoProxyTask::status:
+	{
+		samStatus *ss = static_cast<struct samStatus *>(ticket->data);
+		RS_STACK_MUTEX(mLock);
+		ss->state = mState;
+		if (mSetting.session)
+			ss->sessionName = mSetting.session->channel;
+		else
+			ss->sessionName = "none";
+	}
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+
+		break;
+	case autoProxyTask::getSettings:
+		// check if everything needed is set
+		if (!data) {
+			RS_DBG("autoProxyTask::getSettings data is missing");
+			rsAutoProxyMonitor::taskError(ticket);
+			break;
+		}
+
+		// get settings
+	    {
+		    RS_STACK_MUTEX(mLock);
+			*static_cast<struct samSettings *>(ticket->data) = mSetting;
+	    }
+
+		// finish task
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::setSettings:
+		// check if everything needed is set
+		if (!data) {
+			RS_DBG("autoProxyTask::setSettings data is missing");
+			rsAutoProxyMonitor::taskError(ticket);
+			break;
+		}
+
+		// set settings
+	    {
+		    RS_STACK_MUTEX(mLock);
+			mSetting = *static_cast<struct samSettings *>(ticket->data);
+			updateSettings_locked();
+	    }
+
+		// finish task
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::getErrorInfo:
+		*static_cast<std::string *>(ticket->data) = mSetting.session->error;
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::reloadConfig:
+	    {
+		    RS_STACK_MUTEX(mLock);
+			updateSettings_locked();
+	    }
+		rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		break;
+	case autoProxyTask::stop:
+#if 0 // doesn't seem to work, socket stays "CLOSE_WAIT"
+		// there can be a case where libsam3 will block forever because for some reason it does not detect that the socket it has is dead
+		// as a workaroung kill it from here
+		if (mState == samStatus::samState::connectSession || mState == samStatus::samState::connectForward) {
+			// lock should be held by the main thread
+			if (!mTmpSession) {
+				// now it's getting weird
+				RS_WARN("session is nullptr but mState says it is connecting.");
+				// no break! just ignore for now ...
+			} else {
+				// just close it from here, libsam3 is not thread safe.
+				// a bit of a hack but should do the trick
+//				sam3CloseSession(mSetting.session);
+				sam3tcpDisconnect(mTmpSession->fd);
+				// no break! continue as usual to keep everything in line
+			}
+		}
+#endif
+		[[fallthrough]];
+	case autoProxyTask::start: [[fallthrough]];
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey: [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck: [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
+		// These are supposed to be async!
+		RS_WARN("unknown task or async one!");
+		rsAutoProxyMonitor::taskError(ticket);
+		break;
+	}
+}
+
+void p3I2pSam3::threadTick()
+{
+//	{
+//		RS_STACK_MUTEX(mLock);
+//		Dbg4() << __PRETTY_FUNCTION__ << " mPending: " << mPending.size() << std::endl;
+//	}
+
+	if(mPending.empty()) {
+		// sleep outisde of lock!
+		doSleep(std::chrono::milliseconds(250));
+		return;
+	}
+
+	// get task
+	taskTicket *tt = nullptr;
+	{
+		RS_STACK_MUTEX(mLock);
+		tt = mPending.front();
+		mPending.pop();
+	}
+
+	switch (tt->task) {
+	case autoProxyTask::stop:
+		mState = samStatus::samState::offline;
+		stopForwarding();
+		stopSession();
+		rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::offline);
+		break;
+
+	case autoProxyTask::start:
+	{
+		if (!mSetting.enable) {
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::disabled);
+			break;
+		}
+
+		// create main session
+		mState = samStatus::samState::connectSession;
+		bool ret = startSession();
+		if (!ret) {
+			mState = samStatus::samState::offline;
+
+			rsAutoProxyMonitor::taskError(tt);
+			break;
+		}
+
+		// start forwarding
+		mState = samStatus::samState::connectForward;
+		ret = startForwarding();
+
+		// finish ticket
+		if (ret) {
+			mState = samStatus::samState::online;
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::online);
+		} else {
+			mState = samStatus::samState::offline;
+			rsAutoProxyMonitor::taskError(tt);
+		}
+	}
+		break;
+
+	case autoProxyTask::receiveKey:
+	{
+		i2p::address *addr = static_cast<i2p::address *>(tt->data);
+		if (generateKey(addr->publicKey, addr->privateKey)) {
+			addr->base32 = i2p::keyToBase32Addr(addr->publicKey);
+			rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::ok);
+		} else {
+			rsAutoProxyMonitor::taskError(tt);
+		}
+	}
+		break;
+
+	case autoProxyTask::lookupKey:
+		lookupKey(tt);
+		break;
+
+	case autoProxyTask::proxyStatusCheck:
+	{
+		// TODO better detection of status
+		bool ok;
+		ok = !!mSetting.session->fd;
+		ok &= !!mSetting.session->fwd_fd;
+		*static_cast<bool*>(tt->data) = ok;
+		rsAutoProxyMonitor::taskDone(tt, ok ? autoProxyStatus::ok : autoProxyStatus::error);
+	}
+		break;
+
+	case autoProxyTask::establishConnection:
+		establishConnection(tt);
+		break;
+	case autoProxyTask::closeConnection:
+		closeConnection(tt);
+		break;
+	case autoProxyTask::status: [[fallthrough]];
+	case autoProxyTask::getSettings: [[fallthrough]];
+	case autoProxyTask::setSettings: [[fallthrough]];
+	case autoProxyTask::getErrorInfo: [[fallthrough]];
+	case autoProxyTask::reloadConfig:
+		RS_ERR("unable to handle! This is a bug! task:", tt->task);
+		rsAutoProxyMonitor::taskError(tt);
+		break;
+	}
+	tt = nullptr;
+
+	// give i2p backend some time
+	doSleep(std::chrono::milliseconds(100));
+}
+
+RsSerialiser *p3I2pSam3::setupSerialiser()
+{
+	RsSerialiser* rsSerialiser = new RsSerialiser();
+	rsSerialiser->addSerialType(new RsGeneralConfigSerialiser());
+
+	return rsSerialiser;
+}
+
+#define addKVS(_key, _value) \
+	kv.key = _key;\
+	kv.value = _value;\
+	vitem->tlvkvs.pairs.push_back(kv);
+
+#define addKVSInt(_key, _value) \
+	kv.key = _key;\
+	rs_sprintf(kv.value, "%d", _value);\
+	vitem->tlvkvs.pairs.push_back(kv);
+
+bool p3I2pSam3::saveList(bool &cleanup, std::list<RsItem *> &lst)
+{
+	RS_DBG4();
+
+	cleanup = true;
+	RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet;
+	RsTlvKeyValue kv;
+
+	RS_STACK_MUTEX(mLock);
+	addKVS(kConfigKeySAM3Enable, mSetting.enable ? "TRUE" : "FALSE")
+	addKVS(kConfigKeyDestPriv,   mSetting.address.privateKey);
+
+	addKVSInt(kConfigKeyInLength,    mSetting.inLength)
+	addKVSInt(kConfigKeyInQuantity,  mSetting.inQuantity)
+	addKVSInt(kConfigKeyInVariance,  mSetting.inVariance)
+	addKVSInt(kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
+
+	addKVSInt(kConfigKeyOutLength,   mSetting.outLength)
+	addKVSInt(kConfigKeyOutQuantity, mSetting.outQuantity)
+	addKVSInt(kConfigKeyOutVariance, mSetting.outVariance)
+	addKVSInt(kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+	// these allow SAMv3 users to switch back to BOB
+	// remove after some time
+	addKVS(kConfigKeyBOBEnable, mSetting.enable ? "TRUE" : "FALSE")
+	addKVS(kConfigKeyBOBKey,    mSetting.address.privateKey)
+	addKVS(kConfigKeyBOBAddr,   mSetting.address.base32)
+#endif
+	lst.push_back(vitem);
+	return true;
+}
+
+#undef addKVS
+#undef addKVSUInt
+
+#define getKVSUInt(_kit, _key, _value) \
+	else if (_kit->key == _key) {\
+	    std::istringstream is(_kit->value);\
+	    int tmp;\
+	    is >> tmp;\
+	    _value = (int8_t)tmp;\
+	}
+
+bool p3I2pSam3::loadList(std::list<RsItem *> &load)
+{
+	RS_DBG4();
+
+	std::string priv;
+	priv.clear();
+
+	for(std::list<RsItem*>::const_iterator it = load.begin(); it!=load.end(); ++it) {
+		RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it);
+		if(vitem != NULL) {
+			RS_STACK_MUTEX(mLock);
+			for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) {
+				if      (kit->key == kConfigKeySAM3Enable)
+					mSetting.enable = kit->value == "TRUE";
+				else if (kit->key == kConfigKeyDestPriv)
+					priv = kit->value;
+				getKVSUInt(kit, kConfigKeyInLength,    mSetting.inLength)
+				getKVSUInt(kit, kConfigKeyInQuantity,  mSetting.inQuantity)
+				getKVSUInt(kit, kConfigKeyInVariance,  mSetting.inVariance)
+				getKVSUInt(kit, kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
+
+				getKVSUInt(kit, kConfigKeyOutLength,   mSetting.outLength)
+				getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity)
+				getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance)
+				getKVSUInt(kit, kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
+
+#ifdef RS_I2P_SAM3_BOB_COMPAT
+				// import BOB settings
+				else if (kit->key == kConfigKeyBOBEnable)
+				    mSetting.enable = kit->value == "TRUE";
+				else if (kit->key == kConfigKeyBOBKey) {
+					// don't overwirte, just import when not set already!
+					if (priv.empty())
+						priv = kit->value;
+				}
+#endif
+				else
+					RS_INFO("unknown key:", kit->key);
+			}
+		}
+		delete vitem;
+	}
+
+	// get the pub key
+	std::string pub = i2p::publicKeyFromPrivate(priv);
+	if (pub.empty() || priv.empty())
+		RS_DBG("no destination to load");
+	else {
+		RS_STACK_MUTEX(mLock);
+
+		mSetting.address.publicKey = pub;
+		mSetting.address.privateKey = priv;
+		mSetting.address.base32 = i2p::keyToBase32Addr(pub);
+	}
+
+	RS_STACK_MUTEX(mLock);
+	mConfigLoaded = true;
+
+	return true;
+}
+
+#undef getKVSUInt
+
+bool p3I2pSam3::startSession()
+{
+	RS_DBG4();
+
+	constexpr size_t len = 8;
+	const std::string location = RsRandom::alphaNumeric(len);
+	const std::string nick = "RetroShare-" + location;
+
+	std::vector<std::string> params;
+	{
+		RS_STACK_MUTEX(mLock);
+
+		// length
+		params.push_back(i2p::makeOption("inbound.length", mSetting.inLength));
+		params.push_back(i2p::makeOption("outbound.length", mSetting.outLength));
+		// variance
+		params.push_back(i2p::makeOption("inbound.lengthVariance", + mSetting.inVariance));
+		params.push_back(i2p::makeOption("outbound.lengthVariance", + mSetting.outVariance));
+		// quantity
+		params.push_back(i2p::makeOption("inbound.quantity", + mSetting.inQuantity));
+		params.push_back(i2p::makeOption("outbound.quantity", + mSetting.outQuantity));
+		// backup quantity
+		params.push_back(i2p::makeOption("inbound.backupQuantity", + mSetting.inBackupQuantity));
+		params.push_back(i2p::makeOption("outbound.backupQuantity", + mSetting.outBackupQuantity));
+	}
+
+	std::string paramsStr;
+	for (auto &&p : params)
+		paramsStr.append(p + " ");
+	// keep trailing space for easier extending when necessary
+
+	int ret;
+
+	if (mSetting.session) {
+		stopSession();
+	}
+
+	auto session = (Sam3Session*)rs_malloc(sizeof (Sam3Session));
+
+	// add nick
+	paramsStr.append("inbound.nickname=" + nick); // leading space is already there
+
+	{
+		RS_STACK_MUTEX(mLockSam3Access);
+
+		if(!mSetting.address.privateKey.empty()) {
+			RS_DBG3("with destination");
+			ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, mSetting.address.privateKey.c_str(), Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
+		} else {
+			RS_DBG("without destination");
+			ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, SAM3_DESTINATION_TRANSIENT, Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
+		}
+	}
+
+	if (ret != 0) {
+		delete session;
+		session = nullptr;
+		return false;
+	}
+
+#if 0 // this check is useless. For non i2p hidden locations the public key is temporal anyway and for i2p hidden ones, it is part of the (fixed) private key.
+	if (!mSetting.address.publicKey.empty() && mSetting.address.publicKey != session->pubkey)
+		// This should be ok for non hidden locations. This should be a problem for hidden i2p locations...
+		RS_DBG("public key changed! Yet unsure if this is ok or a problem. Should be fine for non i2p hidden locations or clear net.");
+#endif
+	/*
+	 * Note: sam3CreateSession will issue a name looup of "ME" to receive its public key, thus it is always correct.
+	 *       No need to use i2p::publicKeyFromPrivate()
+	 */
+	RS_STACK_MUTEX(mLock);
+	mSetting.session = session;
+	mSetting.address.publicKey = session->pubkey;
+	mSetting.address.base32 = i2p::keyToBase32Addr(session->pubkey);
+	// do not overwrite the private key, if any!!
+
+	RS_DBG1("nick: ", nick, " address: ", mSetting.address.base32);
+	RS_DBG2("  myDestination.pub  ", mSetting.address.publicKey);
+	RS_DBG2("  myDestination.priv ", mSetting.address.privateKey);
+	return true;
+}
+
+bool p3I2pSam3::startForwarding()
+{
+	RS_DBG4();
+
+	if(mSetting.address.privateKey.empty()) {
+		RS_DBG3("no private key set");
+		// IMPORANT: return true here!
+		// since there is no forward session for non hidden nodes, this funtion is successfull by doing nothing
+		return true;
+	}
+
+	if (!mSetting.session) {
+		RS_WARN("no session found!");
+		return false;
+	}
+
+	peerState ps;
+	mPeerMgr->getOwnNetStatus(ps);
+
+	RS_STACK_MUTEX(mLockSam3Access);
+
+	mSetting.session->silent = true;
+	int ret = sam3StreamForward(mSetting.session, sockaddr_storage_iptostring(ps.localaddr).c_str(), sockaddr_storage_port(ps.localaddr));
+	if (ret < 0) {
+		RS_DBG("forward failed, due to", mSetting.session->error);
+		return false;
+	}
+
+	return true;
+}
+
+void p3I2pSam3::stopSession()
+{
+	RS_DBG4();
+
+	{
+		RS_STACK_MUTEX(mLock);
+		if (!mSetting.session)
+			return;
+
+		// swap connections
+		mInvalidConnections = mValidConnections;
+		mValidConnections.clear();
+
+		RS_STACK_MUTEX(mLockSam3Access);
+		sam3CloseSession(mSetting.session);
+		free(mSetting.session);
+
+		mSetting.session = nullptr;
+		mState = samStatus::samState::offline;
+	}
+
+	// At least i2pd doesn't like to instantaniously stop and (re)start a session, wait here just a little bit.
+	// Not ideal but does the trick.
+	// (This happens when using the "restart" button in the settings.)
+	doSleep(std::chrono::seconds(10));
+}
+
+void p3I2pSam3::stopForwarding()
+{
+	// nothing to do here, forwarding is stop when closing the seassion
+}
+
+bool p3I2pSam3::generateKey(std::string &pub, std::string &priv)
+{
+	RS_DBG4();
+
+	pub.clear();
+	priv.clear();
+
+	// The session is only usef for transporting the data
+	Sam3Session ss;
+
+	if (0 > sam3GenerateKeys(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, Sam3SigType::EdDSA_SHA512_Ed25519)) {
+		RS_DBG("got error: ", ss.error);
+		return false;
+	}
+
+	pub = std::string(ss.pubkey);
+	priv = std::string(ss.privkey);
+
+	// sanity check
+	auto p = i2p::publicKeyFromPrivate(priv);
+	if (p != pub) {
+		RS_WARN("public key does not match private key! fixing ...");
+		pub = p;
+	}
+
+	RS_DBG2("publuc key / address ", pub);
+	RS_DBG2("private key ", priv);
+
+	return true;
+}
+
+void p3I2pSam3::lookupKey(taskTicket *ticket)
+{
+	// this can be called independend of the main SAM session!
+
+	auto addr = static_cast<i2p::address*>(ticket->data);
+	if (addr->base32.empty()) {
+		RS_ERR("lookupKey: called with empty address");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	RsThread::async([ticket]()
+	{
+		auto addr = static_cast<i2p::address*>(ticket->data);
+
+		// The session is only usef for transporting the data
+		Sam3Session ss;
+		int ret = sam3NameLookup(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, addr->base32.c_str());
+		if (ret < 0) {
+			// get error
+			RS_DBG("key:       ", addr->base32);
+			RS_DBG("got error: ", ss.error);
+			rsAutoProxyMonitor::taskError(ticket);
+		} else {
+			addr->publicKey = ss.destkey;
+			rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+			RS_DBG1("success");
+		}
+	});
+}
+
+void p3I2pSam3::establishConnection(taskTicket *ticket)
+{
+	if (mState != samStatus::samState::online || !mSetting.session) {
+		RS_WARN("no session found!");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	samEstablishConnectionWrapper *wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
+	if (wrapper->address.publicKey.empty()) {
+		RS_ERR("no public key given");
+		rsAutoProxyMonitor::taskError(ticket);
+		return;
+	}
+
+	RsThread::async([ticket, this]() {
+		auto wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
+
+		struct Sam3Connection *connection;
+		{
+			auto l = this->mLockSam3Access;
+			RS_STACK_MUTEX(l);
+			mSetting.session->silent = false;
+			connection = sam3StreamConnect(this->mSetting.session, wrapper->address.publicKey.c_str());
+		}
+
+		if (!connection) {
+			// get error
+			RS_DBG("got error:", this->mSetting.session->error);
+			rsAutoProxyMonitor::taskError(ticket);
+		} else {
+			wrapper->connection = connection;
+			{
+				auto l = this->mLockSam3Access;
+				RS_STACK_MUTEX(l);
+				this->mValidConnections.push_back(connection);
+			}
+			RS_DBG1("success");
+			rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+		}
+	});
+}
+
+void p3I2pSam3::closeConnection(taskTicket *ticket)
+{
+	Sam3Connection *conn = static_cast<Sam3Connection*>(ticket->data);
+
+	if (mState == samStatus::samState::offline || !mSetting.session) {
+		// no session found, sam was likely stopped
+		RS_DBG2("no session found");
+
+		auto it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
+		if (it != mInvalidConnections.end()) {
+			// this is the expected case
+			mInvalidConnections.erase(it);
+		} else {
+			// this is unexpected but not a big deal, just warn
+			RS_WARN("cannot find connection in mInvalidConnections");
+
+			it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
+			if (it != mValidConnections.end()) {
+				mValidConnections.erase(it);
+
+				// now it is getting even weirder, still not a big deal, just warn
+				RS_WARN("found connection in mValidConnections");
+			}
+		}
+
+		// when libsam3 has already handled closing of the connection - which should be the case here - the memory has been freed already (-> pointer is invalid)
+		conn = nullptr;
+	} else {
+		RS_STACK_MUTEX(mLock);
+
+		bool callClose = true;
+		// search in current connections
+		auto it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
+		if (it != mValidConnections.end()) {
+			RS_DBG2("found valid connection");
+			mValidConnections.erase(it);
+		} else {
+			// search in old connections
+			it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
+			if (it != mInvalidConnections.end()) {
+				// old connection, just ignore. *should* be freed already
+				mInvalidConnections.erase(it);
+
+				RS_DBG2("found old (invalid) connection");
+
+				callClose = false;
+				conn = nullptr;
+			} else {
+				// weird
+				RS_WARN("could'n find connection!");
+
+				// best thing we can do here
+				callClose = false;
+				conn = nullptr;
+			}
+		}
+
+		if (callClose) {
+			RS_DBG2("closing connection");
+
+			RS_STACK_MUTEX(mLockSam3Access);
+			sam3CloseConnection(conn);
+			conn = nullptr; // freed by above call
+		}
+	}
+
+	if (conn) {
+		free(conn);
+		conn = nullptr;
+	}
+
+	ticket->data = nullptr;
+	rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
+	return;
+}
+
+void p3I2pSam3::updateSettings_locked()
+{
+	RS_DBG4();
+	IndicateConfigChanged();
+
+#if 0 // TODO recreat session when active, can we just recreat it?
+	if (mSs) {
+		stopSession();
+		startSession();
+	}
+#endif
+}

libretroshare/src/services/autoproxy/p3i2psam3.h

@@ -0,0 +1,110 @@
+#ifndef P3I2PSAM3_H
+#define P3I2PSAM3_H
+
+#include <queue>
+#include <list>
+
+#include "services/autoproxy/rsautoproxymonitor.h"
+#include "pqi/p3cfgmgr.h"
+#include "util/i2pcommon.h"
+#include "util/rsthreads.h"
+
+/*
+ * This class implements I2P SAMv3 (Simple Anonymous Messaging) to allow RS
+ * to automatically setup tunnel to and from I2P.
+ * SAMv3 is a simple text-based interface: https://geti2p.net/de/docs/api/samv3
+ *
+ * For the actual SAM commands / low level stuff libsam3 (https://github.com/i2p/libsam3)
+ * is used with some minor adjustments, for exmaple, the FORWARD session is always silent.
+ *
+ * SAM in a nutshell works like this:
+ * 1) setup main/control session which configures everything (destination ID, tunnel number, hops number, and so on)
+ * 2) setup a forward session, so that I2P will establish a connection to RS for each incoming connection to our i2p destination
+ * 3a) query/lookup the destination (public key) for a given i2p address
+ * 3b) connect to the given destination
+ *
+ * An established connection (both incoming or outgoing) are then handed over to RS.
+ * The lifetime of a session (and its subordinates connections) is bound to their tcp socket. When the socket closes, the session is closed, too.
+ *
+ */
+
+class p3PeerMgr;
+
+class Sam3Session;
+class Sam3Connection;
+
+typedef Sam3Session samSession;
+
+struct samSettings : i2p::settings {
+	samSession *session;
+};
+
+struct samEstablishConnectionWrapper {
+	i2p::address address;
+	Sam3Connection *connection;
+};
+
+struct samStatus {
+	std::string sessionName;
+	enum samState {
+		offline,
+		connectSession,
+		connectForward,
+		online
+	} state; // the name is kinda redundant ...
+};
+
+class p3I2pSam3 : public RsTickingThread, public p3Config, public autoProxyService
+{
+public:
+	p3I2pSam3(p3PeerMgr *peerMgr);
+
+	// autoProxyService interface
+public:
+	bool isEnabled();
+	bool initialSetup(std::string &addr, uint16_t &port);
+	void processTaskAsync(taskTicket *ticket);
+	void processTaskSync(taskTicket *ticket);
+
+	// RsTickingThread interface
+public:
+	void threadTick();	/// @see RsTickingThread
+
+	// p3Config interface
+protected:
+	RsSerialiser *setupSerialiser();
+	bool saveList(bool &cleanup, std::list<RsItem *> &);
+	bool loadList(std::list<RsItem *> &load);
+
+private:
+	bool startSession();
+	bool startForwarding();
+	void stopSession();
+	void stopForwarding();
+
+	bool generateKey(std::string &pub, std::string &priv);
+	void lookupKey(taskTicket *ticket);
+	void establishConnection(taskTicket *ticket);
+	void closeConnection(taskTicket *ticket);
+	void updateSettings_locked();
+
+	bool mConfigLoaded;
+
+	samSettings mSetting;
+	p3PeerMgr *mPeerMgr;
+	std::queue<taskTicket *> mPending;
+
+	// Used to report the state to the gui
+	// (Since the create session call/will can block and there is no easy way from outside the main thread to see
+	// what is going on, it is easier to store the current state in an extra variable independen from the main thread)
+	samStatus::samState mState;
+
+	// used to keep track of connections, libsam3 does it internally but it can be unreliable since pointers are shared
+	std::list<Sam3Connection *> mValidConnections, mInvalidConnections;
+
+	// mutex
+	RsMutex mLock;
+	RsMutex mLockSam3Access; // libsam3 is not thread safe! (except for key lookup)
+};
+
+#endif // P3I2PSAM3_H

libretroshare/src/services/autoproxy/rsautoproxymonitor.cc

@@ -329,14 +329,22 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType
 
 bool rsAutoProxyMonitor::isAsyncTask(autoProxyTask::autoProxyTask_enum t)
 {
+	// Explicit list all values, so that missing ones will be detected by the compiler.
 	switch (t) {
-	case autoProxyTask::start:
+	case autoProxyTask::start:      [[fallthrough]];
-	case autoProxyTask::stop:
+	case autoProxyTask::stop:       [[fallthrough]];
-	case autoProxyTask::receiveKey:
+	case autoProxyTask::receiveKey: [[fallthrough]];
+	case autoProxyTask::lookupKey:  [[fallthrough]];
+	case autoProxyTask::establishConnection: [[fallthrough]];
+	case autoProxyTask::closeConnection:
 		return true;
-		break;
+	case autoProxyTask::status:           [[fallthrough]];
-	default:
+	case autoProxyTask::getSettings:      [[fallthrough]];
-		break;
+	case autoProxyTask::setSettings:      [[fallthrough]];
+	case autoProxyTask::getErrorInfo:     [[fallthrough]];
+	case autoProxyTask::reloadConfig:     [[fallthrough]];
+	case autoProxyTask::proxyStatusCheck:
+		return false;
 	}
 	return false;
 }

libretroshare/src/services/autoproxy/rsautoproxymonitor.h

@@ -31,7 +31,8 @@ class autoProxyCallback;
 
 namespace autoProxyType {
     enum autoProxyType_enum {
-		I2PBOB
+//		I2PBOB,
+		I2PSAM3
 	};
 }
 
@@ -41,7 +42,10 @@ namespace autoProxyTask {
 		start,               ///< start up proxy
 		stop,                ///< shut down proxy
 		receiveKey,          ///< renew proxy key (if any)
+		lookupKey,           ///< look up a base32 addr
 		proxyStatusCheck,    ///< use to check if the proxy is still running
+		establishConnection, ///< create a connection to a given public key or base32 address
+		closeConnection,	 ///< closes a connection
 		/* sync tasks */
 		status,       ///< get status from auto proxy
 		getSettings,  ///< get setting from auto proxy

libretroshare/src/services/p3gxsforums.cc

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
     RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
                    RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
     RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
-    mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
+    mGenActive(false), mGenCount(0),
+    mKnownForumsMutex("GXS forums known forums timestamp cache")
+#ifdef RS_DEEP_FORUMS_INDEX
+    , mDeepIndex(DeepForumsIndex::dbDefaultPath())
+#endif
 {
 	// Test Data disabled in Repo.
 	//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@@ -192,158 +196,197 @@ RsSerialiser* p3GxsForums::setupSerialiser()
 
 void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
 {
-#ifdef GXSFORUMS_DEBUG
+	RS_DBG2(changes.size(), " changes to notify");
-	std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
+
+	for(RsGxsNotify* gxsChange: changes)
+	{
+		// Let the compiler delete the change for us
+		std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
+
+		switch(gxsChange->getType())
+		{
+		case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
+		case RsGxsNotify::TYPE_PUBLISHED:
+		{
+			auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
+
+			if(msgChange) /* Message received */
+			{
+				uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
+				switch(static_cast<RsGxsForumsItems>(msgSubtype))
+				{
+				case RsGxsForumsItems::MESSAGE_ITEM:
+				{
+					auto newForumMessageItem =
+					        dynamic_cast<RsGxsForumMsgItem*>(
+					            msgChange->mNewMsgItem );
+
+					if(!newForumMessageItem)
+					{
+						RS_ERR("Received message change with mNewMsgItem type "
+						       "mismatching or null");
+						print_stacktrace();
+						return;
+					}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+					RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
+					tmpPost.mMeta = newForumMessageItem->meta;
+					mDeepIndex.indexForumPost(tmpPost);
 #endif
-
-	std::vector<RsGxsNotify *>::iterator it;
-	for(it = changes.begin(); it != changes.end(); ++it)
-	{
-		RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
-
-		if (msgChange)
-		{
-			if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
-				if (rsEvents)
-				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
 					ev->mForumMsgId = msgChange->mMsgId;
 					ev->mForumGroupId = msgChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
 					rsEvents->postEvent(ev);
-				}
-
-#ifdef NOT_USED_YET
-			if (!msgChange->metaChange())
-			{
-#ifdef GXSCHANNELS_DEBUG
-				std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
-				std::cerr << std::endl;
-#endif
-
-				std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
-				for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
-				{
-#ifdef GXSCHANNELS_DEBUG
-					std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
-					std::cerr << std::endl;
-#endif
-					bool enabled = false;
-					if (autoDownloadEnabled(mit->first, enabled) && enabled)
-					{
-#ifdef GXSCHANNELS_DEBUG
-						std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
-						std::cerr << std::endl;
-#endif
-
-						/* problem is most of these will be comments and votes,
-						 * should make it occasional - every 5mins / 10minutes TODO */
-						unprocessedGroups.push_back(mit->first);
-					}
-				}
-			}
-#endif
-		}
-		else
-		{
-			if (rsEvents)
-			{
-				RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
-				if (grpChange)
-				{
-					switch (grpChange->getType())
-					{
-					case RsGxsNotify::TYPE_PROCESSED:	// happens when the group is subscribed
-					{
-							auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
-							ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
-							rsEvents->postEvent(ev);
-					}
 					break;
-
-                    case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
-                    {
-                            auto ev = std::make_shared<RsGxsForumEvent>();
-                            ev->mForumGroupId = grpChange->mGroupId;
-                            ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
-                            rsEvents->postEvent(ev);
 				}
+				default:
+					RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
 					break;
+				}
+			}
 
-                    case RsGxsNotify::TYPE_PUBLISHED:
+			auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-					case RsGxsNotify::TYPE_RECEIVED_NEW:
+			if(groupChange) /* Group received */
 			{
-						/* group received */
-
 				bool unknown;
 				{
 					RS_STACK_MUTEX(mKnownForumsMutex);
-                            unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
+					unknown = ( mKnownForums.find(gxsChange->mGroupId)
-                            mKnownForums[grpChange->mGroupId] = time(nullptr);
+					            == mKnownForums.end() );
+					mKnownForums[gxsChange->mGroupId] = time(nullptr);
 					IndicateConfigChanged();
 				}
 
 				if(unknown)
 				{
 					auto ev = std::make_shared<RsGxsForumEvent>();
-							ev->mForumGroupId = grpChange->mGroupId;
+					ev->mForumGroupId = gxsChange->mGroupId;
 					ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
 					rsEvents->postEvent(ev);
 				}
-						else
+
-							RsInfo() << __PRETTY_FUNCTION__
+#ifdef RS_DEEP_FORUMS_INDEX
-							         << " Not notifying already known forum "
+				uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
-							         << grpChange->mGroupId << std::endl;
+				switch(static_cast<RsGxsForumsItems>(itemType))
+				{
+				case RsGxsForumsItems::GROUP_ITEM:
+				{
+					auto newForumGroupItem =
+					        static_cast<RsGxsForumGroupItem*>(
+					            groupChange->mNewGroupItem );
+					mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
+					break;
+				}
+				default:
+					RS_WARN("Got unknown gxs group subtype: ", itemType);
+					break;
+				}
+#endif // def RS_DEEP_FORUMS_INDEX
+
 			}
 			break;
-
+		}
-                    case RsGxsNotify::TYPE_GROUP_DELETED:
+		case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-                        ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
+		{
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
+			ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
+			rsEvents->postEvent(ev);
+			break;
+		}
+		case RsGxsNotify::TYPE_MESSAGE_DELETED:
+		{
+			auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
+			if(!delChange)
+			{
+				RS_ERR( "Got mismatching notification type: ",
+				        gxsChange->getType() );
+				print_stacktrace();
+				break;
+			}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumPostFromIndex(
+			            delChange->mGroupId, delChange->messageId);
+#endif
+
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumEventCode = RsForumEventCode::DELETED_POST;
+			ev->mForumGroupId = delChange->mGroupId;
+			ev->mForumMsgId = delChange->messageId;
+			break;
+		}
+		case RsGxsNotify::TYPE_GROUP_DELETED:
+		{
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
+#endif
+			auto ev = std::make_shared<RsGxsForumEvent>();
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
 			rsEvents->postEvent(ev);
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_STATISTICS_CHANGED:
 		{
 			auto ev = std::make_shared<RsGxsForumEvent>();
-						ev->mForumGroupId = grpChange->mGroupId;
+			ev->mForumGroupId = gxsChange->mGroupId;
 			ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
 			rsEvents->postEvent(ev);
 
 			RS_STACK_MUTEX(mKnownForumsMutex);
-                        mKnownForums[grpChange->mGroupId] = time(nullptr);
+			mKnownForums[gxsChange->mGroupId] = time(nullptr);
 			IndicateConfigChanged();
-                    }
 			break;
-
+		}
 		case RsGxsNotify::TYPE_UPDATED:
 		{
-						// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
+			/* Happens when the group data has changed. In this case we need to
+			 * analyse the old and new group in order to detect possible
+			 * notifications for clients */
 
-						RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
-						RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
+
+			RsGxsForumGroupItem* old_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
+			RsGxsForumGroupItem* new_forum_grp_item =
+			        dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
 
 			if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
 			{
-                            RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
+				RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
-							delete grpChange;
+				        "mNewGroup not of type RsGxsForumGroupItem or NULL. "
-							continue;
+				        "This is inconsistent!");
+				print_stacktrace();
+				break;
 			}
 
-						// First of all, we check if there is a difference between the old and new list of moderators
+#ifdef RS_DEEP_FORUMS_INDEX
+			mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
+#endif
+
+			/* First of all, we check if there is a difference between the old
+			 * and new list of moderators */
 
 			std::list<RsGxsId> added_mods, removed_mods;
-
 			for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
-							if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == old_forum_grp_item->mGroup.mAdminList.ids.end() )
 					added_mods.push_back(gxs_id);
 
 			for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
-							if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
+				if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
+				        == new_forum_grp_item->mGroup.mAdminList.ids.end() )
 					removed_mods.push_back(gxs_id);
 
 			if(!added_mods.empty() || !removed_mods.empty())
@@ -359,24 +402,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			}
 
 			// check the list of pinned posts
-
 			std::list<RsGxsMessageId> added_pins, removed_pins;
 
 			for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					added_pins.push_back(msg_id);
 
 			for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
-                            if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
+				if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
+				        == new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
 					removed_pins.push_back(msg_id);
 
 			if(!added_pins.empty() || !removed_pins.empty())
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
-
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
-
 				rsEvents->postEvent(ev);
 			}
 
@@ -384,31 +426,24 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
 			        || old_forum_grp_item->meta.mGroupName  != new_forum_grp_item->meta.mGroupName
 			        || old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
 			        || old_forum_grp_item->meta.mAuthorId   != new_forum_grp_item->meta.mAuthorId
-                          || old_forum_grp_item->meta.mCircleId      != new_forum_grp_item->meta.mCircleId
+			        || old_forum_grp_item->meta.mCircleId   != new_forum_grp_item->meta.mCircleId )
-                             )
 			{
 				auto ev = std::make_shared<RsGxsForumEvent>();
 				ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
 				ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
 				rsEvents->postEvent(ev);
 			}
-					}
-                        break;
 
+			break;
+		}
 
 		default:
-                        RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
+			RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
+			        " Currently not handled." );
 			break;
 		}
 	}
 }
-		}
-
-		/* shouldn't need to worry about groups - as they need to be subscribed to */
-
-        delete *it;
-	}
-}
 
 void	p3GxsForums::service_tick()
 {
@@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
 	forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
 	        | GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
 
+	/* This flag have always this value even for circle restricted forums due to
+	 * how GXS distribute/verify groups */
 	forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
 
 	forum.mMeta.mCircleId.clear();
@@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
 	        id == mMeta.mAuthorId;
 }
 
+std::error_condition p3GxsForums::getContentSummaries(
+        const RsGxsGroupId& forumId,
+        const std::set<RsGxsMessageId>& contentIds,
+        std::vector<RsMsgMetaData>& summaries )
+{
+	uint32_t token;
+	RsTokReqOptions opts;
+	opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
+
+	GxsMsgReq msgReq;
+	msgReq[forumId] = contentIds;
+
+
+	if(!requestMsgInfo(token, opts, msgReq))
+	{
+		RS_ERR("requestMsgInfo failed");
+		return std::errc::invalid_argument;
+	}
+
+	switch(waitToken(token, std::chrono::seconds(5)))
+	{
+	case RsTokenService::COMPLETE:
+	{
+		GxsMsgMetaMap metaMap;
+		if(!RsGenExchange::getMsgMeta(token, metaMap))
+			return std::errc::result_out_of_range;
+		summaries = metaMap[forumId];
+		return std::error_condition();
+	}
+	case RsTokenService::PARTIAL: // [[fallthrough]];
+	case RsTokenService::PENDING:
+		return std::errc::timed_out;
+	default:
+		return std::errc::not_supported;
+	}
+}
+
+#ifdef RS_DEEP_FORUMS_INDEX
+std::error_condition p3GxsForums::handleDistantSearchRequest(
+        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RS_DBG1("");
+
+	RsGxsForumsSearchRequest request;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(request.mType) );
+		return std::errc::bad_message;
+	}
+
+	RsGxsForumsSearchReply reply;
+	auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
+	if(mErr || reply.mResults.empty()) return mErr;
+
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(reply);
+		resultSize = ctx.mOffset;
+	}
+
+	resultData = rs_malloc<uint8_t>(resultSize);
+	RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+	RsGenericSerializer::SerializeJob j =
+	        RsGenericSerializer::SerializeJob::SERIALIZE;
+	RS_SERIAL_PROCESS(reply);
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string& matchString, TurtleRequestId& searchId )
+{
+	RsGxsForumsSearchRequest request;
+	request.mQuery = matchString;
+
+	uint32_t requestSize;
+	{
+		RsGenericSerializer::SerializeContext ctx;
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
+		RS_SERIAL_PROCESS(request);
+		requestSize = ctx.mOffset;
+	}
+
+	std::error_condition ec;
+	auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
+	if(!requestData) return ec;
+	{
+		RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::SERIALIZE;
+		RS_SERIAL_PROCESS(request);
+	}
+
+	return netService()->distantSearchRequest(
+	            requestData, requestSize,
+	            static_cast<RsServiceType>(serviceType()), searchId );
+}
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string& matchString,
+        std::vector<RsGxsSearchResult>& searchResults )
+{ return prepareSearchResults(matchString, false, searchResults); }
+
+std::error_condition p3GxsForums::prepareSearchResults(
+        const std::string& matchString, bool publicOnly,
+        std::vector<RsGxsSearchResult>& searchResults )
+{
+	std::vector<DeepForumsSearchResult> results;
+	auto mErr = mDeepIndex.search(matchString, results);
+	if(mErr) return mErr;
+
+	searchResults.clear();
+	for(auto uRes: results)
+	{
+		RsUrl resUrl(uRes.mUrl);
+		const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
+		if(!forumIdStr)
+		{
+			RS_ERR( "Forum URL retrieved from deep index miss ID. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::address_not_available;
+		}
+
+		std::vector<RsGxsForumGroup> forumsInfo;
+		RsGxsGroupId forumId(*forumIdStr);
+		if(forumId.isNull())
+		{
+			RS_ERR( "Forum ID retrieved from deep index is invalid. ",
+			        "Should never happen! ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::bad_address;
+		}
+
+		if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
+		        forumsInfo.empty() )
+		{
+			RS_ERR( "Forum just parsed from deep index link not found. "
+			        "Should never happen! ", forumId, " ", uRes.mUrl );
+			print_stacktrace();
+			return std::errc::identifier_removed;
+		}
+
+		RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
+
+		// Avoid leaking sensitive information to unkown peers
+		if( publicOnly &&
+		        ( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
+		          RsGxsCircleType::PUBLIC ) ) continue;
+
+		RsGxsSearchResult res;
+		res.mGroupId = forumId;
+		res.mGroupName = fMeta.mGroupName;
+		res.mAuthorId = fMeta.mAuthorId;
+		res.mPublishTs = fMeta.mPublishTs;
+		res.mSearchContext = uRes.mSnippet;
+
+		auto postIdStr =
+		        resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
+		if(postIdStr)
+		{
+			RsGxsMessageId msgId(*postIdStr);
+			if(msgId.isNull())
+			{
+				RS_ERR( "Post just parsed from deep index link is invalid. "
+				        "Should never happen! ", postIdStr, " ", uRes.mUrl );
+				print_stacktrace();
+				return std::errc::bad_address;
+			}
+
+			std::vector<RsMsgMetaData> msgSummaries;
+			auto errc = getContentSummaries(
+			            forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
+			if(errc) return errc;
+
+			if(msgSummaries.size() != 1)
+			{
+				RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
+				        "should never happen!" );
+				return std::errc::result_out_of_range;
+			}
+
+			RsMsgMetaData& msgMeta(msgSummaries[0]);
+			res.mMsgId = msgMeta.mMsgId;
+			res.mMsgName = msgMeta.mMsgName;
+			res.mAuthorId = msgMeta.mAuthorId;
+		}
+
+		RS_DBG4(res);
+		searchResults.push_back(res);
+	}
+
+	return std::error_condition();
+}
+
+std::error_condition p3GxsForums::receiveDistantSearchResult(
+        const TurtleRequestId requestId,
+        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
+{
+	RsGxsForumsSearchReply reply;
+	{
+		RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
+		RsGenericSerializer::SerializeJob j =
+		        RsGenericSerializer::SerializeJob::DESERIALIZE;
+		RS_SERIAL_PROCESS(reply);
+	}
+	free(resultData);
+
+	if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
+	{
+		// If more types are implemented we would put a switch on mType instead
+		RS_WARN( "Got search request with unkown type: ",
+		         static_cast<uint32_t>(reply.mType) );
+		return std::errc::bad_message;
+	}
+
+	auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
+	event->mSearchId = requestId;
+	event->mSearchResults = reply.mResults;
+	rsEvents->postEvent(event);
+	return std::error_condition();
+}
+
+#else  // def RS_DEEP_FORUMS_INDEX
+
+std::error_condition p3GxsForums::distantSearchRequest(
+        const std::string&, TurtleRequestId& )
+{ return std::errc::function_not_supported; }
+
+std::error_condition p3GxsForums::localSearch(
+        const std::string&,
+        std::vector<RsGxsSearchResult>& )
+{ return std::errc::function_not_supported; }
+
+#endif // def RS_DEEP_FORUMS_INDEX
+
 /*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
         "retroshare:///forums";
 /*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

libretroshare/src/services/p3gxsforums.h

@@ -4,8 +4,8 @@
  * libretroshare: retroshare core library                                      *
  *                                                                             *
  * Copyright (C) 2012-2014  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2018-2020  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2019-2020  Asociación Civil Altermundi <info@altermundi.net>  *
+ * Copyright (C) 2019-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -32,6 +32,10 @@
 #include "util/rstickevent.h"
 #include "util/rsdebug.h"
 
+#ifdef RS_DEEP_FORUMS_INDEX
+#include "deep_search/forumsindex.hpp"
+#endif
+
 
 class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
 	public RsTickEvent	/* only needed for testing - remove after */
@@ -144,6 +148,33 @@ public:
 	        const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
 	        bool keepForever ) override;
 
+	/// @see RsGxsForums
+	std::error_condition getContentSummaries(
+	        const RsGxsGroupId& forumId,
+	        const std::set<RsGxsMessageId>& contentIds,
+	        std::vector<RsMsgMetaData>& summaries ) override;
+
+	/// @see RsGxsForums
+	std::error_condition distantSearchRequest(
+	        const std::string& matchString, TurtleRequestId& searchId ) override;
+
+	/// @see RsGxsForums
+	std::error_condition localSearch(
+	        const std::string& matchString,
+	        std::vector<RsGxsSearchResult>& searchResults ) override;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	/// @see RsNxsObserver
+	std::error_condition handleDistantSearchRequest(
+	        rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+
+	/// @see RsNxsObserver
+	std::error_condition receiveDistantSearchResult(
+	        const TurtleRequestId requestId,
+	        rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
+#endif
+
     /// implementation of rsGxsGorums
     ///
 	bool getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups) override;
@@ -155,6 +186,17 @@ public:
 
 	bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
 
+protected:
+#ifdef RS_DEEP_FORUMS_INDEX
+	/** Internal usage
+	 * @param[in] publicOnly if true is passed only results pertaining to
+	 * publicly shared forums are returned
+	 */
+	std::error_condition prepareSearchResults(
+	        const std::string& matchString, bool publicOnly,
+	        std::vector<RsGxsSearchResult>& searchResults );
+#endif //def RS_DEEP_FORUMS_INDEX
+
 private:
 
 static uint32_t forumsAuthenPolicy();
@@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
     std::map<RsGxsGroupId,rstime_t> mKnownForums ;
 	
 	RsMutex mKnownForumsMutex;
+
+#ifdef RS_DEEP_FORUMS_INDEX
+	DeepForumsIndex mDeepIndex;
+#endif
 };

libretroshare/src/turtle/p3turtle.cc

@@ -3,7 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright 2009-2018 by Cyril Soler <csoler@users.sourceforge.net>           *
+ * Copyright (C) 2009-2018  Cyril Soler <csoler@users.sourceforge.net>         *
+ * Copyright (C) 2018-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ * Copyright (C) 2021  Asociación Civil Altermundi <info@altermundi.net>       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -26,6 +28,10 @@
 #include <stdexcept>
 #include <stdlib.h>
 #include <assert.h>
+#include <iostream>
+#include <cerrno>
+#include <cmath>
+#include <cstdio>
 
 #include "rsserver/p3face.h"
 #include "crypto/rscrypto.h"
@@ -39,13 +45,7 @@
 #include "ft/ftcontroller.h"
 
 #include "p3turtle.h"
-
+#include "util/cxx17retrocompat.h"
-#include <iostream>
-#include <errno.h>
-#include <cmath>
-
-#include <stdio.h>
-
 #include "util/rsdebug.h"
 #include "util/rsprint.h"
 #include "util/rsrandom.h"
@@ -1975,7 +1975,8 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
 //
 
 
-void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
+void RsTurtleStringSearchRequestItem::search(
+        std::list<TurtleFileInfo>& result ) const
 {
 	/* call to core */
 	std::list<DirDetails> initialResults;
@@ -1988,17 +1989,19 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
 	std::cerr << "Performing rsFiles->search()" << std::endl ;
 #endif
 	// now, search!
-    rsFiles->SearchKeywords(words, initialResults,RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE,PeerId());
+	rsFiles->SearchKeywords(
+	            words, initialResults,
+	            RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE, PeerId() );
 
 #ifdef P3TURTLE_DEBUG
 	std::cerr << initialResults.size() << " matches found." << std::endl ;
 #endif
 	result.clear() ;
 
-	for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
+	for(auto& it: std::as_const(initialResults))
 	{
 		// retain only file type
-		if (it->type == DIR_TYPE_DIR)
+		if (it.type == DIR_TYPE_DIR)
 		{
 #ifdef P3TURTLE_DEBUG
 			std::cerr << "  Skipping directory " << it->name << std::endl ;
@@ -2007,9 +2010,9 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
 		}
 
 		TurtleFileInfo i;
-		i.hash = it->hash ;
+		i.hash = it.hash;
-        i.size = it->size ;
+		i.size = it.size;
-		i.name = it->name ;
+		i.name = it.name;
 
 		result.push_back(i);
 	}

libretroshare/src/turtle/turtleclientservice.h

@@ -19,23 +19,25 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-
-// This class is the parent class for any service that will use the turtle router to distribute its packets.
-// Typical representative clients include:
-//
-// 	p3ChatService:		opens tunnels to distant peers for chatting
-// 	ftServer:			searches and open tunnels to distant sources for file transfer
-//
 #pragma once
 
 #include <string>
 #include <stdlib.h>
-#include <serialiser/rsserial.h>
+
-#include <turtle/rsturtleitem.h>
+#include "serialiser/rsserial.h"
+#include "turtle/rsturtleitem.h"
+#include "util/rsdebug.h"
 
 struct RsItem;
 class p3turtle ;
 
+/** This class is the parent class for any service that will use the turtle
+ * router to distribute its packets.
+ * Typical representative clients include:
+ * 	p3ChatService: opens tunnels to distant peers for chatting
+ * 	ftServer:      searches and open tunnels to distant sources for file
+ * 	               transfer
+ */
 class RsTurtleClientService
 {
 	public:
@@ -88,27 +90,32 @@ class RsTurtleClientService
 		}
 
 	/*!
-         * \brief receiveSearchRequest
+	* This method is called by the turtle router to notify the client of a
-         * 			This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
+	* search request in the form generic data.
-         * 			result contains the serialised generic result returned by the client.
+	* The returned result contains the serialised generic result returned by the
-         *
+	* client service.
 	* The turtle router keeps the memory ownership over search_request_data
-         *
 	* \param search_request_data generic serialized search data
 	* \param search_request_data_len  length of the serialized search data
 	* \param search_result_data generic serialized search result data
 	* \param search_result_data_len length of the serialized search result data
-         * \param max_allowed_hits         max number of hits allowed to be sent back and forwarded
+	* \param max_allowed_hits max number of hits allowed to be sent back and
-         *
+	* forwarded
-         * \return true if the search is successful.
+	* \return true if matching results are available, false otherwise.
 	*/
-    	virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
+	virtual bool receiveSearchRequest(
-                                          uint32_t /*search_request_data_len*/,
+	        unsigned char *search_request_data, uint32_t search_request_data_len,
-                                          unsigned char *& /*search_result_data*/,
+	        unsigned char *& search_result_data, uint32_t& search_result_data_len,
-                                          uint32_t& /*search_result_data_len*/,
+	        uint32_t& max_allows_hits )
-                                          uint32_t& /* max_allows_hits */)
 	{
-			std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
+		/* Suppress unused warning this way and not commenting the param names
+		 * so doxygen match documentation against params */
+		(void) search_request_data; (void) search_request_data_len;
+		(void) search_result_data; (void) search_result_data_len;
+		(void) max_allows_hits;
+
+		RS_WARN( "Received search request from turtle router, but the client "
+		         "is not handling it!" );
 		return false;
 	}
 

libretroshare/src/use_libretroshare.pri

@@ -90,6 +90,15 @@ rs_broadcast_discovery {
     win32-g++|win32-clang-g++:dLibs *= wsock32
 }
 
+rs_sam3_libsam3 {
+    LIBSAM3_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/libsam3/)
+    LIBSAM3_BUILD_PATH=$$clean_path($${RS_BUILD_PATH}/supportlibs/libsam3/)
+    INCLUDEPATH *= $$clean_path($${LIBSAM3_SRC_PATH}/src/libsam3/)
+    DEPENDPATH *= $$clean_path($${LIBSAM3_BUILD_PATH})
+    QMAKE_LIBDIR *= $$clean_path($${LIBSAM3_BUILD_PATH})
+    LIBS *= -L$$clean_path($${LIBSAM3_BUILD_PATH}) -lsam3
+}
+
 static {
     sLibs *= $$mLibs
 } else {

libretroshare/src/util/dnsresolver.cc

@@ -41,8 +41,6 @@
 const rstime_t MAX_TIME_BEFORE_RETRY 	=	300 ; /* seconds before retrying an ip address */
 const rstime_t MAX_KEEP_DNS_ENTRY 		= 3600 ; /* seconds during which a DNS entry is considered valid */
 
-static const std::string ADDR_AGENT  = "Mozilla/5.0";
-
 void *solveDNSEntries(void *p)
 {
 	bool more_to_go = true ;

libretroshare/src/util/extaddrfinder.cc

@@ -19,9 +19,13 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
+
+//#define EXTADDRSEARCH_DEBUG
+
 #include "extaddrfinder.h"
 
 #include "pqi/pqinetwork.h"
+#include "rsdebug.h"
 #include "util/rsstring.h"
 #include "util/rsmemory.h"
 
@@ -38,286 +42,237 @@
 #include <stdio.h>
 #include "util/rstime.h"
 
+#include <map>
+
 const uint32_t MAX_IP_STORE = 300; /* seconds ip address timeout */
 
-//#define EXTADDRSEARCH_DEBUG
+class ZeroInt
-
-static const std::string ADDR_AGENT  = "Mozilla/5.0";
-
-static std::string scan_ip(const std::string& text)
 {
-	std::set<unsigned char> digits ;
+public:
-	digits.insert('0') ; digits.insert('3') ; digits.insert('6') ;
+	ZeroInt() : n(0) {}
-	digits.insert('1') ; digits.insert('4') ; digits.insert('7') ;
+	uint32_t n ;
-	digits.insert('2') ; digits.insert('5') ; digits.insert('8') ;
+};
-	digits.insert('9') ; 
 
-	for(int i=0;i<(int)text.size();++i)
+void ExtAddrFinder::run()
-	{
-		while(i < (int)text.size() && digits.find(text[i])==digits.end()) ++i ;
-
-		if(i>=(int)text.size())
-			return "" ;
-
-		unsigned int a,b,c,d ;
-
-		if(sscanf(text.c_str()+i,"%u.%u.%u.%u",&a,&b,&c,&d) != 4)
-			continue ;
-
-		if(a < 256 && b<256 && c<256 && d<256)
-		{
-			std::string s ;
-			rs_sprintf(s, "%u.%u.%u.%u", a, b, c, d) ;
-			return s;
-		}
-	}
-	return "" ;
-}
-
-static void getPage(const std::string& server_name,std::string& page)
-{
-	page = "" ;
-	int sockfd,n=0;                   // socket descriptor
-	struct sockaddr_in serveur;       // server's parameters
-	memset(&serveur.sin_zero, 0, sizeof(serveur.sin_zero));
-
-	char buf[1024];
-	char request[1024];
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cout << "ExtAddrFinder: connecting to " << server_name << std::endl ;
-#endif
-	// socket creation
-
-	sockfd = unix_socket(PF_INET,SOCK_STREAM,0);
-	if (sockfd < 0)
-	{
-		std::cerr << "ExtAddrFinder: Failed to create socket" << std::endl;
-		return ;
-	}
-
-	serveur.sin_family = AF_INET;
-
-	// get server's ipv4 adress
-
-    	in_addr in ;
-        
-	if(!rsGetHostByName(server_name.c_str(),in))  /* l'hôte n'existe pas */
-	{
-		std::cerr << "ExtAddrFinder: Unknown host " << server_name << std::endl;
-		unix_close(sockfd);
-		return ;
-	}
-	serveur.sin_addr = in ;
-	serveur.sin_port = htons(80);
-
-#ifdef EXTADDRSEARCH_DEBUG
-	printf("Connection attempt\n");
-#endif
-    	std::cerr << "ExtAddrFinder: resolved hostname " << server_name << " to " << rs_inet_ntoa(in) << std::endl;
-
-	sockaddr_storage server;
-	sockaddr_storage_setipv4(server, &serveur);
-	sockaddr_storage_setport(server, 80);
-	if(unix_connect(sockfd, server) == -1)
-	{
-		std::cerr << "ExtAddrFinder: Connection error to " << server_name << std::endl ;
-		unix_close(sockfd);
-		return ;
-	}
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Connection established to " << server_name << std::endl ;
-#endif
-
-	// envoi 
-	if(snprintf( request, 
-			1024,
-			"GET / HTTP/1.0\r\n"
-			"Host: %s:%d\r\n"
-			"Connection: Close\r\n"
-			"\r\n", 
-			server_name.c_str(), 80) > 1020)
-	{
-		std::cerr << "ExtAddrFinder: buffer overrun. The server name \"" << server_name << "\" is too long. This is quite unexpected." << std::endl;
-		unix_close(sockfd);
-		return ;
-	}
-
-	if(send(sockfd,request,strlen(request),0)== -1)
-	{
-		std::cerr << "ExtAddrFinder: Could not send request to " << server_name << std::endl ;
-		unix_close(sockfd);
-		return ;
-	}
-	// recéption 
-
-	while((n = recv(sockfd, buf, sizeof buf - 1, 0)) > 0)
-	{
-		buf[n] = '\0';
-		page += std::string(buf,n) ;
-	}
-	// fermeture de la socket
-
-	unix_close(sockfd);
-#ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Got full page from " << server_name << std::endl ;
-#endif
-}
-
-
-void* doExtAddrSearch(void *p)
 {
 	
 	std::vector<std::string> res ;
 
-	ExtAddrFinder *af = (ExtAddrFinder*)p ;
+	for(auto& it : _ip_servers)
-
-	for(std::list<std::string>::const_iterator it(af->_ip_servers.begin());it!=af->_ip_servers.end();++it)
 	{
-		std::string page ;
+		std::string ip = "";
-
+		rsGetHostByNameSpecDNS(it,"myip.opendns.com",ip,2);
-		getPage(*it,page) ;
-		std::string ip = scan_ip(page) ;
-
 		if(ip != "")
 			res.push_back(ip) ;
 #ifdef EXTADDRSEARCH_DEBUG
-		std::cout << "ip found through " << *it << ": \"" << ip << "\"" << std::endl ;
+		RS_DBG("ip found through DNS ", it, ": \"", ip, "\"");
 #endif
 	}
 
 	if(res.empty())
 	{
-		// thread safe copy results.
+		reset();
-		//
+		return ;
+	}
+
+	std::map<sockaddr_storage,ZeroInt> addrV4_votes;
+	std::map<sockaddr_storage,ZeroInt> addrV6_votes;
+	std::string all_addrV4_Found;
+	std::string all_addrV6_Found;
+
+	for(auto curRes : res)
 	{
-			RsStackMutex mtx(af->mAddrMtx) ;
+		sockaddr_storage addr;
-
+		sockaddr_storage_clear(addr);
-			af->mFound = false ;
+		bool validIP =   sockaddr_storage_inet_pton(addr, curRes)
-			af->mFoundTS = time(NULL) ;
+		              && sockaddr_storage_isValidNet(addr);
-			af->mSearching = false ;
+		bool isIPv4 = sockaddr_storage_ipv6_to_ipv4(addr);
-		}
+		if( validIP && isIPv4 )
-		return NULL ;
-	}
-
-	sort(res.begin(),res.end()) ; // eliminates outliers.
-
-
-
-	if(!sockaddr_storage_ipv4_aton(af->mAddr, res[res.size()/2].c_str()))
 		{
-		std::cerr << "ExtAddrFinder: Could not convert " << res[res.size()/2] << " into an address." << std::endl ;
+			addr.ss_family = AF_INET;
+			addrV4_votes[addr].n++ ;
+			all_addrV4_Found += sockaddr_storage_tostring(addr) + "\n";
+		}
+		else if( validIP && !isIPv4)
 		{
-			RsStackMutex mtx(af->mAddrMtx) ;
+			addr.ss_family = AF_INET6;
-			af->mFound = false ;
+			addrV6_votes[addr].n++ ;
-			af->mFoundTS = time(NULL) ;
+			all_addrV6_Found += sockaddr_storage_tostring(addr) + "\n";
-			af->mSearching = false ;
 		}
-		return NULL ;
+		else
+			RS_ERR("Invalid addresse reported: ", curRes) ;
+
 	}
 
+	if( (0 == addrV4_votes.size()) && (0 == addrV6_votes.size()) )
+	{
+		RS_ERR("Could not find any external address.");
+		reset();
+		return ;
+	}
+
+	if( 1 < addrV4_votes.size() )
+		RS_ERR("Multiple external IPv4 addresses reported: "
+		      , all_addrV4_Found ) ;
+
+	if( 1 < addrV6_votes.size() )
+		RS_ERR("Multiple external IPv6 addresses reported: "
+		      , all_addrV6_Found ) ;
+
 	{
-		RsStackMutex mtx(af->mAddrMtx) ;
+		RS_STACK_MUTEX(mAddrMtx);
-		af->mFound = true ;
+
-		af->mFoundTS = time(NULL) ;
+		mSearching = false ;
-		af->mSearching = false ;
+		mFoundTS = time(NULL) ;
+
+		// Only save more reported address if not only once.
+		uint32_t admax = 0 ;
+		sockaddr_storage_clear(mAddrV4);
+		for (auto it : addrV4_votes)
+			if (admax < it.second.n)
+			{
+				mAddrV4 = it.first ;
+				mFoundV4 = true ;
+				admax = it.second.n ;
 			}
 
-	return NULL ;
+		admax = 0 ;
+		sockaddr_storage_clear(mAddrV6);
+		for (auto it : addrV6_votes)
+			if (admax < it.second.n)
+			{
+				mAddrV6 = it.first ;
+				mFoundV6 = true ;
+				admax = it.second.n ;
 			}
 
+	}
+
+	return ;
+}
 
 void ExtAddrFinder::start_request()
 {
-	void *data = (void *)this;
+	if (!isRunning())
-	pthread_t tid ;
+		start("ExtAddrFinder");
-    
-    	if(! pthread_create(&tid, 0, &doExtAddrSearch, data))
-		pthread_detach(tid); /* so memory is reclaimed in linux */
-        else
-            	std::cerr << "(EE) Could not start ExtAddrFinder thread." << std::endl;
 }
 
-bool ExtAddrFinder::hasValidIP(struct sockaddr_storage &addr)
+bool ExtAddrFinder::hasValidIPV4(struct sockaddr_storage &addr)
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Getting ip." << std::endl ;
+	RS_DBG("Getting ip.");
 #endif
 
 	{
-		RsStackMutex mut(mAddrMtx) ;
+		RS_STACK_MUTEX(mAddrMtx) ;
-		if(mFound)
+		if(mFoundV4)
 		{
 #ifdef EXTADDRSEARCH_DEBUG
-			std::cerr << "ExtAddrFinder: Has stored ip: responding with this ip." << std::endl ;
+			RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV4)) ;
 #endif
-            sockaddr_storage_copyip(addr,mAddr);	// just copy the IP so we dont erase the port.
+			sockaddr_storage_copyip(addr,mAddrV4);	// just copy the IP so we dont erase the port.
 		}
 	}
-	rstime_t delta;
+
+	testTimeOut();
+
+	RS_STACK_MUTEX(mAddrMtx) ;
+	return mFoundV4;
+}
+
+bool ExtAddrFinder::hasValidIPV6(struct sockaddr_storage &addr)
 {
-		RsStackMutex mut(mAddrMtx) ;
+#ifdef EXTADDRSEARCH_DEBUG
+	RS_DBG("Getting ip.");
+#endif
+
+	{
+		RS_STACK_MUTEX(mAddrMtx) ;
+		if(mFoundV6)
+		{
+#ifdef EXTADDRSEARCH_DEBUG
+			RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV6)) ;
+#endif
+			sockaddr_storage_copyip(addr,mAddrV6);	// just copy the IP so we dont erase the port.
+		}
+	}
+
+	testTimeOut();
+
+	RS_STACK_MUTEX(mAddrMtx) ;
+	return mFoundV6;
+}
+
+void ExtAddrFinder::testTimeOut()
+{
+	bool timeOut;
+	{
+		RS_STACK_MUTEX(mAddrMtx) ;
 		//timeout the current ip
-		delta = time(NULL) - mFoundTS;
+		timeOut = (mFoundTS + MAX_IP_STORE < time(NULL));
 	}
-	if((uint32_t)delta > MAX_IP_STORE) {//launch a research
+	if(timeOut || mFirstTime) {//launch a research
 		if( mAddrMtx.trylock())
 		{
 			if(!mSearching)
 			{
 #ifdef EXTADDRSEARCH_DEBUG
-				std::cerr << "ExtAddrFinder: No stored ip: Initiating new search." << std::endl ;
+				RS_DBG("No stored ip: Initiating new search.");
 #endif
 				mSearching = true ;
 				start_request() ;
 			}
 #ifdef EXTADDRSEARCH_DEBUG
 			else
-				std::cerr << "ExtAddrFinder: Already searching." << std::endl ;
+				RS_DBG("Already searching.");
 #endif
+			mFirstTime = false;
 			mAddrMtx.unlock();
 		}
 #ifdef EXTADDRSEARCH_DEBUG
 		else
-			std::cerr << "ExtAddrFinder: (Note) Could not acquire lock. Busy." << std::endl ;
+			RS_DBG("(Note) Could not acquire lock. Busy.");
 #endif
 	}
-
-	RsStackMutex mut(mAddrMtx) ;
-	return mFound ;
 }
 
-void ExtAddrFinder::reset()
+void ExtAddrFinder::reset(bool firstTime /*=false*/)
 {
-	RsStackMutex mut(mAddrMtx) ;
+#ifdef EXTADDRSEARCH_DEBUG
+	RS_DBG("firstTime=", firstTime);
+#endif
+	RS_STACK_MUTEX(mAddrMtx) ;
 
-	mFound = false ;
 	mSearching = false ;
-	mFoundTS = time(NULL) - MAX_IP_STORE;
+	mFoundV4 = false ;
+	mFoundV6 = false ;
+	mFirstTime = firstTime;
+	mFoundTS = time(nullptr);
+	sockaddr_storage_clear(mAddrV4);
+	sockaddr_storage_clear(mAddrV6);
 }
 
 ExtAddrFinder::~ExtAddrFinder()
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Deleting ExtAddrFinder." << std::endl ;
+	RS_DBG("Deleting ExtAddrFinder.");
 #endif
-
 }
 
 ExtAddrFinder::ExtAddrFinder() : mAddrMtx("ExtAddrFinder")
 {
 #ifdef EXTADDRSEARCH_DEBUG
-	std::cerr << "ExtAddrFinder: Creating new ExtAddrFinder." << std::endl ;
+	RS_DBG("Creating new ExtAddrFinder.");
 #endif
-	RsStackMutex mut(mAddrMtx) ;
+	reset( true );
 
-	mFound = false;
+//https://unix.stackexchange.com/questions/22615/how-can-i-get-my-external-ip-address-in-a-shell-script
-	mSearching = false;
+	//Enter direct ip so local DNS cannot change it.
-	mFoundTS = time(NULL) - MAX_IP_STORE;
+	//DNS servers must recognize "myip.opendns.com"
-	sockaddr_storage_clear(mAddr);
+	_ip_servers.push_back(std::string( "208.67.222.222" )) ;//resolver1.opendns.com
-
+	_ip_servers.push_back(std::string( "208.67.220.220" )) ;//resolver2.opendns.com
-	_ip_servers.push_back(std::string( "checkip.dyndns.org" )) ;
+	_ip_servers.push_back(std::string( "208.67.222.220" )) ;//resolver3.opendns.com
-	_ip_servers.push_back(std::string( "www.myip.dk"   )) ;
+	_ip_servers.push_back(std::string( "208.67.220.222" )) ;//resolver4.opendns.com
-	_ip_servers.push_back(std::string( "showip.net"         )) ;
+	_ip_servers.push_back(std::string( "2620:119:35::35" )) ;//resolver1.opendns.com
-	_ip_servers.push_back(std::string( "www.displaymyip.com")) ;
+	_ip_servers.push_back(std::string( "2620:119:53::53" )) ;//resolver2.opendns.com
 }
-

libretroshare/src/util/extaddrfinder.h

@@ -30,26 +30,31 @@
 
 struct sockaddr ;
 
-class ExtAddrFinder
+class ExtAddrFinder: public RsThread
 {
 	public:
 		ExtAddrFinder() ;
 		~ExtAddrFinder() ;
 
-		bool hasValidIP(struct sockaddr_storage &addr) ;
+		bool hasValidIPV4(struct sockaddr_storage &addr) ;
+		bool hasValidIPV6(struct sockaddr_storage &addr) ;
 		void getIPServersList(std::list<std::string>& ip_servers) { ip_servers = _ip_servers ; }
 
 		void start_request() ;
 
-		void reset() ;
+		void reset(bool firstTime = false) ;
 
 	private:
-		friend void* doExtAddrSearch(void *p) ;
+		virtual void run();
+		void testTimeOut();
 
 		RsMutex mAddrMtx;
-		rstime_t   mFoundTS;
-		struct sockaddr_storage mAddr;
-		bool mFound ;
 		bool mSearching;
+		bool mFoundV4;
+		bool mFoundV6;
+		bool mFirstTime;
+		rstime_t mFoundTS;
+		struct sockaddr_storage mAddrV4;
+		struct sockaddr_storage mAddrV6;
 		std::list<std::string> _ip_servers;
 };

libretroshare/src/util/folderiterator.cc

@@ -38,8 +38,11 @@
 namespace librs { namespace util {
 
 
-FolderIterator::FolderIterator(const std::string& folderName, bool allow_symlinks, bool allow_files_from_the_future)
+FolderIterator::FolderIterator(
-    : mFolderName(folderName),mAllowSymLinks(allow_symlinks),mAllowFilesFromTheFuture(allow_files_from_the_future)
+        const std::string& folderName, bool allow_symlinks,
+        bool allow_files_from_the_future ):
+    mFolderName(folderName), mAllowSymLinks(allow_symlinks),
+    mAllowFilesFromTheFuture(allow_files_from_the_future)
 {
     is_open = false ;
     validity = false ;

libretroshare/src/util/folderiterator.h

@@ -44,7 +44,9 @@ namespace librs { namespace util {
 class FolderIterator
 {
 public:
-    FolderIterator(const std::string& folderName,bool allow_symlinks,bool allow_files_from_the_future = true);
+	FolderIterator(
+	        const std::string& folderName, bool allow_symlinks,
+	        bool allow_files_from_the_future = true );
 	~FolderIterator();
 
     enum { TYPE_UNKNOWN = 0x00,

libretroshare/src/util/i2pcommon.cpp

@@ -50,8 +50,10 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	 * https://geti2p.net/spec/common-structures#keysandcert
 	 * https://geti2p.net/spec/common-structures#certificate
 	 */
-	if (priv.length() < 884) // base64 ( = 663 bytes = KeyCert + priv Keys)
+	if (priv.length() < privKeyMinLenth_b64) {
+		RS_WARN("key to short!");
 		return std::string();
+	}
 
 	// creat a copy to work on, need to convert it to standard base64
 	auto priv_copy(priv);
@@ -70,13 +72,14 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	uint8_t certType = 0;
 	uint16_t len = 0;
 	uint16_t signingKeyType = 0;
-	uint16_t cryptKey = 0;
+	uint16_t cryptKeyType = 0;
 
 	// only used for easy break
 	do {
 		try {
 			// jump to certificate
 			p += publicKeyLen;
+
 			// try to read type and length
 			certType = *p++;
 			len = readTwoBytesBE(p);
@@ -87,7 +90,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
 				/*
 				 * CertType.Null
 				 * type null is followed by 0x00 0x00 <END>
-				 * so has to be 0!
+				 * so len has to be 0!
 				 */
 				RS_DBG("cert is CertType.Null");
 				publicKeyLen += 3; // add 0x00 0x00 0x00
@@ -119,7 +122,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
 			// likely 7
 			signingKeyType = readTwoBytesBE(p);
 
-			RS_DBG("signing pubkey type ", certType);
+			RS_DBG("signing pubkey type ", signingKeyType);
 			if (signingKeyType >= 3 && signingKeyType <= 6) {
 				RS_DBG("signing pubkey type ", certType, " has oversize");
 				// calculate oversize
@@ -137,18 +140,18 @@ std::string publicKeyFromPrivate(std::string const &priv)
 					return std::string();
 				}
 
-				publicKeyLen += values.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted
+				publicKeyLen += values.first - 128; // 128 = default DSA key length = the space that can be used before the key must be splitted
 			}
 
 			// Crypto Public Key
 			// likely 0
-			cryptKey = readTwoBytesBE(p);
+			cryptKeyType = readTwoBytesBE(p);
-			RS_DBG("crypto pubkey type ", cryptKey);
+			RS_DBG("crypto pubkey type ", cryptKeyType);
 			// info: these are all smaller than the default 256 bytes, so no oversize calculation is needed
 
 			break;
 		}  catch (const std::out_of_range &e) {
-			RS_DBG("hit exception! ", e.what());
+			RS_DBG("hit an exception! ", e.what());
 			return std::string();
 		}
 	} while(false);
@@ -160,4 +163,107 @@ std::string publicKeyFromPrivate(std::string const &priv)
 	return pub;
 }
 
+bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey)
+{
+	if (key.length() < pubKeyMinLenth_b64) {
+		RS_WARN("key to short!");
+		return false;
+	}
+
+	// creat a copy to work on, need to convert it to standard base64
+	auto key_copy(key);
+	std::replace(key_copy.begin(), key_copy.end(), '~', '/');
+	// replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too
+	// std::replace(copy.begin(), copy.end(), '-', '+');
+
+	// get raw data
+	std::vector<uint8_t> data;
+	RsBase64::decode(key_copy, data);
+
+	auto p = data.cbegin();
+
+	constexpr size_t publicKeyLen = 256 + 128; // default length (bytes)
+	uint8_t certType = 0;
+	uint16_t signingKeyType = 0;
+	uint16_t cryptKeyType = 0;
+
+	// try to read types
+	try {
+		// jump to certificate
+		p += publicKeyLen;
+
+		// try to read type and skip length
+		certType = *p++;
+		p += 2;
+
+		// only 0 and 5 are used / valid at this point
+		// check for == 0
+		if (certType == static_cast<typename std::underlying_type<CertType>::type>(CertType::Null)) {
+			RS_DBG("cert is CertType.Null");
+
+			signingKey = "DSA_SHA1";
+			cryptoKey = "ElGamal";
+			return true;
+		}
+
+		// check for != 5
+		if (certType != static_cast<typename std::underlying_type<CertType>::type>(CertType::Key)) {
+			// unsupported
+			RS_DBG("cert type ", certType, " is unsupported");
+			return false;
+		}
+
+		RS_DBG("cert is CertType.Key");
+
+		// Signing Public Key
+		// likely 7
+		signingKeyType = readTwoBytesBE(p);
+		RS_DBG("signing pubkey type ", signingKeyType);
+
+		// Crypto Public Key
+		// likely 0
+		cryptKeyType = readTwoBytesBE(p);
+		RS_DBG("crypto pubkey type ", cryptKeyType);
+	}  catch (const std::out_of_range &e) {
+		RS_DBG("hit an exception! ", e.what());
+		return false;
+	}
+
+	// now convert to string (this would be easier with c++17)
+#define HELPER(a, b, c) \
+	case static_cast<typename std::underlying_type<a>::type>(a::c): \
+	    b = #c; \
+	    break;
+
+	switch (signingKeyType) {
+	HELPER(SigningKeyType, signingKey, DSA_SHA1)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA256_P256)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA384_P384)
+	HELPER(SigningKeyType, signingKey, ECDSA_SHA512_P521)
+	HELPER(SigningKeyType, signingKey, RSA_SHA256_2048)
+	HELPER(SigningKeyType, signingKey, RSA_SHA384_3072)
+	HELPER(SigningKeyType, signingKey, RSA_SHA512_4096)
+	HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519)
+	HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519ph)
+	HELPER(SigningKeyType, signingKey, RedDSA_SHA512_Ed25519)
+	default:
+	    RsWarn("unkown signing key type:", signingKeyType);
+	    return false;
+	}
+
+	switch (cryptKeyType) {
+	HELPER(CryptoKeyType, cryptoKey, ElGamal)
+	HELPER(CryptoKeyType, cryptoKey, P256)
+	HELPER(CryptoKeyType, cryptoKey, P384)
+	HELPER(CryptoKeyType, cryptoKey, P521)
+	HELPER(CryptoKeyType, cryptoKey, X25519)
+	default:
+	    RsWarn("unkown crypto key type:", cryptKeyType);
+	    return false;
+	}
+#undef HELPER
+
+	return true;
+}
+
 } // namespace i2p

libretroshare/src/util/i2pcommon.h

@@ -186,6 +186,39 @@ static const std::array<std::pair<uint16_t, uint16_t>, 12> signingKeyLengths {
 	/*SigningKeyType::RedDSA_SHA512_Ed25519  */ std::make_pair<uint16_t, uint16_t>( 32,  32),
 };
 
+/*
+ * Key length infos:
+ *
+ * BOB private key
+ * len b64: 884
+ * len pln: 663
+ *
+ * BOB public key / destination
+ * len b64: 516
+ * len pln: 387
+ *
+ * SAMv3 private key
+ * len b64: 908
+ * len pln: 679
+ *
+ * SAMv3 public key
+ * len b64: 516
+ * len pln: 387
+ *
+ * Example:
+ * in bytes, public key only
+ *   384 (Key) + 3 (Null certificate) = 387 bytes
+ *   384 (Key) + 7 (key  certificate) = 391 bytes
+ *
+ * in bytes public + private key
+ *   384 (Key) + 3 (Null certificate) + 256 (ElGamal) + 20 (DSA_SHA1) = 663 bytes
+ *   384 (Key) + 7 (key  certificate) + 256 (ElGamal) + 32 (EdDSA_SHA512_Ed25519) = 679 bytes
+ */
+constexpr size_t pubKeyMinLenth_b64 = 516;
+constexpr size_t pubKeyMinLenth_bin = 387;
+constexpr size_t privKeyMinLenth_b64 = 884;
+constexpr size_t privKeyMinLenth_bin = 663;
+
 /**
  * @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs
  * @param lhs option to set
@@ -208,6 +241,15 @@ std::string keyToBase32Addr(const std::string &key);
  */
 std::string  publicKeyFromPrivate(const std::string &priv);
 
+/**
+ * @brief getKeyTypes returns the name of the utilized algorithms used by the key
+ * @param key public key
+ * @param signingKey name of the signing key, e.g. DSA_SHA1
+ * @param cryptoKey name of the crpyto key, e.g. ElGamal
+ * @return true on success, false otherwise
+ */
+bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey);
+
 } // namespace i2p
 
 #endif // I2PCOMMON_H

libretroshare/src/util/rsdebug.cc

@@ -3,9 +3,9 @@
  *                                                                             *
  * libretroshare: retroshare core library                                      *
  *                                                                             *
- * Copyright (C) 2004-2008 by Robert Fernie   <retroshare@lunamutt.com>        *
+ * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ * Copyright (C) 2020-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -22,6 +22,8 @@
  *                                                                             *
  *******************************************************************************/
 
+#include <iomanip>
+
 #include "util/rsdebug.h"
 
 std::ostream &operator<<(std::ostream& out, const std::error_condition& err)
@@ -36,6 +38,54 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
 	        " not available in category: " + categoryName;
 }
 
+std::error_condition rs_errno_to_condition(int errno_code)
+{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
+
+
+std::ostream& hex_dump(std::ostream& os, const void *buffer,
+                       std::size_t bufsize, bool showPrintableChars /*= true*/)
+{
+	if (buffer == nullptr) {
+		return os;
+	}
+	auto oldFormat = os.flags();
+	auto oldFillChar = os.fill();
+	constexpr std::size_t maxline{8};
+	// create a place to store text version of string
+	char renderString[maxline+1];
+	char *rsptr{renderString};
+	// convenience cast
+	const unsigned char *buf{reinterpret_cast<const unsigned char *>(buffer)};
+
+	for (std::size_t linecount=maxline; bufsize; --bufsize, ++buf) {
+		os << std::setw(2) << std::setfill('0') << std::hex
+		   << static_cast<unsigned>(*buf) << ' ';
+		*rsptr++ = std::isprint(*buf) ? *buf : '.';
+		if (--linecount == 0) {
+			*rsptr++ = '\0';  // terminate string
+			if (showPrintableChars) {
+				os << " | " << renderString;
+			}
+			os << '\n';
+			rsptr = renderString;
+			linecount = std::min(maxline, bufsize);
+		}
+	}
+	// emit newline if we haven't already
+	if (rsptr != renderString) {
+		if (showPrintableChars) {
+			for (*rsptr++ = '\0'; rsptr != &renderString[maxline+1]; ++rsptr) {
+				os << "   ";
+			}
+			os << " | " << renderString;
+		}
+		os << '\n';
+	}
+
+	os.fill(oldFillChar);
+	os.flags(oldFormat);
+	return os;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
@@ -208,3 +258,7 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
 		lineCount++;
 	}
 }
+/// All the lines before are DEPRECATED!!
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////

libretroshare/src/util/rsdebug.h

@@ -2,8 +2,8 @@
  * RetroShare debugging utilities                                              *
  *                                                                             *
  * Copyright (C) 2004-2008  Robert Fernie <retroshare@lunamutt.com>            *
- * Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org>              *
+ * Copyright (C) 2019-2021  Gioacchino Mazzurco <gio@eigenlab.org>             *
- * Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>       *
+ * Copyright (C) 2020-2021  Asociación Civil Altermundi <info@altermundi.net>  *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Lesser General Public License as              *
@@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
  * the message around */
 std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
 
+/** Convert C errno codes to modern C++11 std::error_condition, this is quite
+ * useful to use toghether with C functions used around the code like `malloc`,
+ * `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
+ */
+std::error_condition rs_errno_to_condition(int errno_code);
+
 
 template <RsLoggerCategories CATEGORY>
 struct t_RsLogger : std::ostringstream
@@ -204,6 +210,53 @@ struct RsNoDbg
 };
 
 
+//From https://codereview.stackexchange.com/a/165162
+/**
+ * @brief hex_dump: Send Hexadecimal Dump to stream
+ * @param os: Output Stream
+ * @param buffer: Buffer to send
+ * @param bufsize: Buffer's size
+ * @param showPrintableChars: If must send printable Char too
+ * @return
+ * basic string:
+ * 61 62 63 64 65 66 31 32  | abcdef12
+ * 33 34 35 36 00 7a 79 78  | 3456.zyx
+ * 77 76 75 39 38 37 36 35  | wvu98765
+ * 34 45 64 77 61 72 64 00  | 4Edward.
+ *
+ * wide string:
+ * 41 00 00 00 20 00 00 00  | A... ...
+ * 77 00 00 00 69 00 00 00  | w...i...
+ * 64 00 00 00 65 00 00 00  | d...e...
+ * 20 00 00 00 73 00 00 00  |  ...s...
+ * 74 00 00 00 72 00 00 00  | t...r...
+ * 69 00 00 00 6e 00 00 00  | i...n...
+ * 67 00 00 00 2e 00 00 00  | g.......
+ *
+ * a double
+ * 49 92 24 49 92 24 09 40  | I.$I.$.@
+ */
+std::ostream& hex_dump(std::ostream& os, const void *buffer,
+                       std::size_t bufsize, bool showPrintableChars = true);
+
+/**
+ * @brief The hexDump struct
+ * Enable to print dump calling like that:
+ * const char test[] = "abcdef123456\0zyxwvu987654Edward";
+ * RsDbg()<<hexDump(test, sizeof(test))<<std::endl;
+ */
+struct hexDump {
+	const void *buffer;
+	std::size_t bufsize;
+	hexDump(const void *buf, std::size_t bufsz) : buffer{buf}, bufsize{bufsz} {}
+	friend std::ostream &operator<<(std::ostream &out, const hexDump &hd) {
+		return hex_dump(out, hd.buffer, hd.bufsize, true);
+	}
+};
+
+
+
+
 
 ////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
@@ -312,3 +365,8 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
 #define PQL_DEBUG_ALERT RSL_DEBUG_ALERT 
 #define PQL_DEBUG_BASIC	RSL_DEBUG_BASIC
 #define PQL_DEBUG_ALL 	RSL_DEBUG_ALL
+
+/// All the lines before are DEPRECATED!!
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////

libretroshare/src/util/rsdir.cc

@@ -458,6 +458,36 @@ bool	RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d
 	return true;
 }
 
+rstime_t RsDirUtil::lastWriteTime(
+        const std::string& path,
+        std::error_condition& errc )
+{
+	if(!fileExists(path))
+	{
+		errc = std::errc::no_such_file_or_directory;
+		return 0;
+	}
+
+#ifdef WINDOWS_SYS
+	struct _stati64 buf;
+	std::wstring wPath;
+	librs::util::ConvertUtf8ToUtf16(path, wPath);
+	if ( 0 == _wstati64(wPath.c_str(), &buf))
+#else
+	struct stat64 buf;
+	if ( 0 == stat64(path.c_str(), &buf))
+#endif
+	{
+		/* errc output param is guaranted to be meaningful only if an error
+		 * happens so is not strictly necessary but we clean it anyway just
+		 * in case */
+		errc = std::error_condition();
+		return buf.st_mtime;
+	}
+
+	errc = std::error_condition(errno, std::generic_category());
+	return 0;
+}
 
 bool RsDirUtil::checkDirectory(const std::string& dir)
 {