Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into v0.6-TorControl

Browse Source
This commit is contained in:
csoler 2021-11-11 18:28:02 +01:00
commit 4d06ab2b4a
895 changed files with 19933 additions and 16253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -15,3 +15,5 @@ Thumbs.db
*.pro.user
.kdev4
*.kdev4
!supportlibs/libsam3/Makefile

80
.gitlab-ci.yml
View File

@ -3,30 +3,46 @@ image: docker:stable
services:
- docker:stable-dind
stages:
- build
- test
workflow:
rules:
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_BRANCH
build-and-test:
variables:
UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
build-ubuntu-test-image:
stage: build
script:
- >
if [ -n "$CI_MERGE_REQUEST_ID" ]; then
REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
else
REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
fi ;
export REPO_ARGS ;
echo REPO_ARGS=$REPO_ARGS ;
docker login "$CI_REGISTRY"
--username "$CI_REGISTRY_USER"
--password "$CI_REGISTRY_PASSWORD"
- mkdir Dockercontext
- >
docker build -t retroshare:testing $REPO_ARGS
docker build
-t $UBUNTU_TESTING_IMAGE_TAG
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
Dockercontext
- docker push $UBUNTU_TESTING_IMAGE_TAG
- echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
test-ubuntu:
stage: test
script:
- >
docker run --name retroshare --detach --tty retroshare:testing
docker login "$CI_REGISTRY"
--username "$CI_REGISTRY_USER"
--password "$CI_REGISTRY_PASSWORD"
- echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
- docker pull $UBUNTU_TESTING_IMAGE_TAG
- >
docker run --name retroshare --detach --tty $UBUNTU_TESTING_IMAGE_TAG
retroshare-service --jsonApiPort 9092
- apk add jq
- >
@ -51,3 +67,43 @@ build-and-test:
docker exec retroshare
curl --verbose http://127.0.0.1:9092/rsMsgs/getChatLobbyList | jq
- docker container stop retroshare
build-android-arm-apk:
stage: build
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
# see https://stackoverflow.com/a/59055906
- >
docker cp
$(docker create --rm retroshare:android_arm_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm.apk
artifacts:
paths:
- RetroShare_Android_Service-arm.apk
# Use separate runner to avoid no space left on device
build-android-arm64-apk:
stage: build
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm64_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg ANDROID_PLATFORM_VER=21
--build-arg ANDROID_NDK_ARCH=arm64
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
- >
docker cp
$(docker create --rm retroshare:android_arm64_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm64.apk
artifacts:
paths:
- RetroShare_Android_Service-arm64.apk

3
.gitmodules vendored
View File

@ -14,3 +14,6 @@
[submodule "supportlibs/rapidjson"]
path = supportlibs/rapidjson
url = https://github.com/Tencent/rapidjson.git
[submodule "supportlibs/libsam3"]
path = supportlibs/libsam3
url = https://github.com/i2p/libsam3.git

12
.travis.yml
View File

@ -8,10 +8,10 @@ language: cpp
matrix:
include:
- os: linux
dist: bionic
sudo: required
compiler: gcc
# - os: linux
# dist: bionic
# sudo: required
# compiler: gcc
- os: osx
osx_image: xcode10.2
compiler: clang
@ -68,11 +68,11 @@ before_script:
fi
script:
- if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j4; fi
- if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then travis_wait 30 make -j$(nproc); fi
after_success:
- if [ $TRAVIS_OS_NAME == osx ]; then build_scripts/OSX/travis_makeOSXPackage.sh ; fi
- if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
# - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
# branches:

4
README.asciidoc
View File

@ -9,9 +9,9 @@ RetroShare provides file sharing, chat, messages, forums, channels and more.
.Build Status
|===============================================================================
|GNU/Linux (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|GNU/Linux, macOS, (via Travis CI) | image:https://travis-ci.org/RetroShare/RetroShare.svg?branch=master[link="https://travis-ci.org/RetroShare/RetroShare"]
|GNU/Linux, Android (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|Windows (via AppVeyor) | image:https://ci.appveyor.com/api/projects/status/github/RetroShare/RetroShare?svg=true[link="https://ci.appveyor.com/project/RetroShare58622/retroshare"]
|macOS, (via Travis CI) | image:https://app.travis-ci.com/RetroShare/RetroShare.svg?branch=master[link="https://app.travis-ci.com/github/RetroShare/RetroShare"]
|===============================================================================

115
build_scripts/Android/Dockerfile Normal file
View File

@ -0,0 +1,115 @@
## To prepare an image suitable as base for Gitlab CI use
## image name must match gitlab repository name, you can play just with the tag
## the part after :
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm_base"
# docker build --squash -t "${CI_REGISTRY_IMAGE}" \
# --build-arg QT_INSTALLER_JWT_TOKEN="your qt JWT token goes here" .
#
# To build Android ARMv8 (64 bit) package pass also
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm64_base"
# --build-arg ANDROID_NDK_ARCH=arm64 --build-arg ANDROID_PLATFORM_VER=21
## --squash is very important in case of GitlabCI shared runners as they are
## limited to 25GB disk size
## To push it to gitlab CI registry you need first to login and the to push
# docker login registry.gitlab.com
# docker push ${CI_IMAGE_NAME}
## To extract the generated APK easily you can run after the build complete
# docker cp \
# $(docker create --rm ${CI_IMAGE_NAME}):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk \
# /tmp/RetroShare_Android_Service.apk
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get clean
RUN apt-get install -y -qq \
bash build-essential bzip2 cmake curl chrpath doxygen \
git p7zip python qt5-default qttools5-dev tclsh unzip wget zip
# Dependencies to create Android pkg
RUN apt-get install -y -qq \
openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle
ARG FRESHCLONE=0
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
ARG REPO_DEPTH="--depth 2000"
RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
git fetch --tags
ENV PREPARE_TOOLCHAIN="/RetroShare/build_scripts/Android/prepare-toolchain-clang.sh"
ENV NATIVE_LIBS_TOOLCHAIN_PATH="/android-toolchain/"
ARG ANDROID_PLATFORM_VER=16
ARG ANDROID_NDK_ARCH=arm
ENV ANDROID_SDK_PATH="/opt/android-sdk"
ENV ANDROID_HOME="$ANDROID_SDK_PATH"
ENV ANDROID_SDK_ROOT="$ANDROID_SDK_PATH"
ENV ANDROID_NDK_PATH="/opt/android-ndk"
ENV ANDROID_NDK_ROOT="$ANDROID_NDK_PATH"
ENV PATH="$PATH:$ANDROID_HOME/tools"
ENV PATH="$PATH:$ANDROID_HOME/platform-tools"
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN mkdir /bCache
WORKDIR /bCache
## Quick HACK to ease development
#COPY prepare-toolchain-clang.sh $PREPARE_TOOLCHAIN
RUN $PREPARE_TOOLCHAIN install_android_sdk
RUN $PREPARE_TOOLCHAIN install_android_ndk
RUN $PREPARE_TOOLCHAIN
ARG QT_INSTALLER_JWT_TOKEN
RUN $PREPARE_TOOLCHAIN install_qt_android
# Avoid Qt account details leak into the image
RUN rm -f /root/.local/share/Qt/qtaccount.ini
# Shrink image by removing unneded Qt components
RUN rm -r \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Docs/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Examples/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Tools/
RUN mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
RUN mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle
RUN rm -rf /bCache
# Clean apt cache
RUN apt-get clean && rm -rf /var/lib/apt/lists/*

1
build_scripts/Android/README.asciidoc
View File

@ -81,6 +81,7 @@ JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
CONFIG+=no_rs_service_terminal_login
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3
-------------------------------------------------------------------------------
TIP: Some versions of QtCreator try to find the Android SDK in

779
build_scripts/Android/prepare-toolchain-clang.sh
View File

@ -2,7 +2,8 @@
# Script to prepare RetroShare Android package building toolchain
#
# Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
@ -38,6 +39,15 @@ define_default_value ANDROID_PLATFORM_VER "16"
define_default_value NATIVE_LIBS_TOOLCHAIN_PATH "${HOME}/Builds/android-toolchains/retroshare-android-${ANDROID_PLATFORM_VER}-${ANDROID_NDK_ARCH}/"
define_default_value HOST_NUM_CPU $(nproc)
define_default_value ANDROID_SDK_INSTALL "false"
define_default_value ANDROID_SDK_TOOLS_VERSION "3859397"
define_default_value ANDROID_SDK_TOOLS_SHA256 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
define_default_value ANDROID_SDK_VERSION "29.0.3"
define_default_value ANDROID_NDK_INSTALL "false"
define_default_value ANDROID_NDK_VERSION "r21"
define_default_value ANDROID_NDK_SHA256 b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b
define_default_value BZIP2_SOURCE_VERSION "1.0.6"
define_default_value BZIP2_SOURCE_SHA256 a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
@ -48,16 +58,22 @@ define_default_value SQLITE_SOURCE_YEAR "2018"
define_default_value SQLITE_SOURCE_VERSION "3250200"
define_default_value SQLITE_SOURCE_SHA256 da9a1484423d524d3ac793af518cdf870c8255d209e369bd6a193e9f9d0e3181
define_default_value SQLCIPHER_SOURCE_VERSION "4.2.0"
define_default_value SQLCIPHER_SOURCE_SHA256 105c1b813f848da038c03647a8bfc9d42fb46865e6aaf4edfd46ff3b18cdccfc
define_default_value SQLCIPHER_SOURCE_VERSION "4.4.3"
define_default_value SQLCIPHER_SOURCE_SHA256 b8df69b998c042ce7f8a99f07cf11f45dfebe51110ef92de95f1728358853133
define_default_value LIBUPNP_SOURCE_VERSION "1.8.4"
define_default_value LIBUPNP_SOURCE_SHA256 976c3e4555604cdd8391ed2f359c08c9dead3b6bf131c24ce78e64d6669af2ed
define_default_value INSTALL_QT_ANDROID "false"
define_default_value QT_VERSION "5.12.0"
define_default_value QT_ANDROID_VIA_INSTALLER "false"
define_default_value QT_VERSION "5.12.11"
define_default_value QT_INSTALLER_VERSION "4.1.1"
define_default_value QT_INSTALLER_SHA256 1266ffd0d1b0e466244e3bc8422975c1aa9d96745b6bb28d422f7f92df11f34c
define_default_value QT_INSTALLER_JWT_TOKEN ""
define_default_value QT_INSTALL_PATH "${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt/"
define_default_value QT_ANDROID_INSTALLER_SHA256 a214084e2295c9a9f8727e8a0131c37255bf724bfc69e80f7012ba3abeb1f763
define_default_value RESTBED_SOURCE_REPO "https://github.com/Corvusoft/restbed.git"
define_default_value RESTBED_SOURCE_VERSION f74f9329dac82e662c1d570b7cd72c192b729eb4
define_default_value UDP_DISCOVERY_CPP_SOURCE "https://github.com/truvorskameikin/udp-discovery-cpp.git"
@ -72,6 +88,64 @@ define_default_value RAPIDJSON_SOURCE_SHA256 bf7ced29704a1e696fbccf2a2b4ea068e77
define_default_value MINIUPNPC_SOURCE_VERSION "2.1.20190625"
define_default_value MINIUPNPC_SOURCE_SHA256 8723f5d7fd7970de23635547700878cd29a5c2bb708b5e5475b2d1d2510317fb
# zlib and libpng versions walks toghether
define_default_value ZLIB_SOURCE_VERSION "1.2.11"
define_default_value ZLIB_SOURCE_SHA256 4ff941449631ace0d4d203e3483be9dbc9da454084111f97ea0a2114e19bf066
define_default_value LIBPNG_SOURCE_VERSION "1.6.37"
define_default_value LIBPNG_SOURCE_SHA256 505e70834d35383537b6491e7ae8641f1a4bed1876dbfe361201fc80868d88ca
define_default_value LIBJPEG_SOURCE_VERSION "9d"
define_default_value LIBJPEG_SOURCE_SHA256 6c434a3be59f8f62425b2e3c077e785c9ce30ee5874ea1c270e843f273ba71ee
define_default_value TIFF_SOURCE_VERSION "4.2.0"
define_default_value TIFF_SOURCE_SHA256 eb0484e568ead8fa23b513e9b0041df7e327f4ee2d22db5a533929dfc19633cb
define_default_value CIMG_SOURCE_VERSION "2.9.7"
define_default_value CIMG_SOURCE_SHA256 595dda9718431a123b418fa0db88e248c44590d47d9b1646970fa0503e27fa5c
define_default_value PHASH_SOURCE_REPO "https://gitlab.com/g10h4ck/pHash.git"
define_default_value PHASH_SOURCE_VERSION origin/android-ndk
define_default_value MVPTREE_SOURCE_REPO "https://github.com/starkdg/mvptree.git"
define_default_value MVPTREE_SOURCE_VERSION origin/master
define_default_value REPORT_DIR "$(pwd)/$(basename ${NATIVE_LIBS_TOOLCHAIN_PATH})_build_report/"
cArch=""
eABI=""
cmakeABI=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
;;
"arm64")
cArch="aarch64"
eABI=""
;;
"x86")
cArch="i686"
eABI=""
;;
"x86_64")
echo "ANDROID_NDK_ARCH=${ANDROID_NDK_ARCH} not supported yet"
exit -1
cArch="??"
eABI=""
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
# Used to instruct cmake to explicitely ignore host libraries
export HOST_IGNORE_PREFIX="/usr/"
## $1 filename, $2 sha256 hash
function check_sha256()
@ -104,159 +178,269 @@ function verified_download()
}
}
cArch=""
eABI=""
# This function is the result of reading and testing many many stuff be very
# careful editing it
function andro_cmake()
{
# Using android.toolchain.cmake as documented here
# https://developer.android.com/ndk/guides/cmake seens to break more things then
# it fixes :-\
cmakeProc=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
cmakeProc="armv7-a"
;;
"arm64")
cArch="aarch64"
eABI=""
cmakeProc="aarch64"
;;
"x86")
cArch="i686"
eABI=""
cmakeProc="i686"
;;
"x86_64")
cmakeProc="x86_64"
;;
*)
echo "Unhandled NDK architecture ${ANDROID_NDK_ARCH}"
exit -1
;;
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
_hi="$HOST_IGNORE_PREFIX"
cmake \
-DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_PREFIX_PATH="${PREFIX}" \
-DCMAKE_SYSTEM_PREFIX_PATH="${PREFIX}" \
-DCMAKE_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_SYSTEM_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_SYSTEM_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" \
-DCMAKE_IGNORE_PATH="$_hi/include;$_hi/lib;$_hi/lib64" \
$@
# It is probably ok to do not touch CMAKE_PROGRAM_PATH and
# CMAKE_SYSTEM_PROGRAM_PATH
}
function git_source_get()
{
sourceDir="$1" ; shift #$1
sourceRepo="$1" ; shift #$2
sourceVersion="$1" ; shift #$3
# extra paramethers are treated as submodules
[ -d "$sourceDir" ] &&
{
pushd "$sourceDir"
actUrl="$(git remote get-url origin)"
[ "$actUrl" != "$sourceRepo" ] && rm -rf "${sourceDir}"
popd
} || true
[ -d $sourceDir ] || git clone "$sourceRepo" "$sourceDir"
pushd $sourceDir
git fetch --all
git reset --hard ${sourceVersion}
while [ "$1" != "" ] ; do
git submodule update --init "$1"
pushd "$1"
git reset --hard
shift
popd
done
popd
}
declare -A TASK_REGISTER
function task_register()
{
TASK_REGISTER[$1]=true
}
function task_unregister()
{
# we may simply wipe them but we could benefit from keeping track of
# unregistered tasks too
TASK_REGISTER[$1]=false
}
function task_logfile()
{
echo "$REPORT_DIR/$1.log"
}
function task_run()
{
mTask="$1" ; shift
[ "${TASK_REGISTER[$mTask]}" != "true" ] &&
{
echo "Attempt to run not registered task $mTask $@"
return -1
}
logFile="$(task_logfile $mTask)"
if [ -f "$logFile" ] ; then
echo "Task $mTask already run more details at $logFile"
else
date | tee > "$logFile"
$mTask $@ |& tee --append "$logFile"
mRetval="${PIPESTATUS[0]}"
echo "Task $mTask return ${mRetval} more details at $logFile"
date | tee --append "$logFile"
return ${mRetval}
fi
}
function task_zap()
{
rm -f "$(task_logfile $1)"
}
DUPLICATED_INCLUDES_LIST_FILE="${REPORT_DIR}/duplicated_includes_list"
DUPLICATED_INCLUDES_DIR="${REPORT_DIR}/duplicated_includes/"
task_register install_android_sdk
install_android_sdk()
{
tFile="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
verified_download "${tFile}" "${ANDROID_SDK_TOOLS_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_SDK_PATH"
rm -rf "$ANDROID_SDK_PATH/tools/"
mv --verbose tools/ "$ANDROID_SDK_PATH/tools/"
# Install Android SDK
yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses && \
$ANDROID_SDK_PATH/tools/bin/sdkmanager --update
$ANDROID_SDK_PATH/tools/bin/sdkmanager "platforms;android-$ANDROID_PLATFORM_VER"
$ANDROID_SDK_PATH/tools/bin/sdkmanager "build-tools;$ANDROID_SDK_VERSION"
}
task_register install_android_ndk
install_android_ndk()
{
tFile="android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
verified_download "${tFile}" "${ANDROID_NDK_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_NDK_PATH"
rm -rf "$ANDROID_NDK_PATH"
mv --verbose "android-ndk-${ANDROID_NDK_VERSION}/" "$ANDROID_NDK_PATH/"
}
## More information available at https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html
build_toolchain()
task_register bootstrap_toolchain
bootstrap_toolchain()
{
echo "build_toolchain()
################################################################################
################################################################################
################################################################################
"
rm -rf ${NATIVE_LIBS_TOOLCHAIN_PATH}
rm -rf "${NATIVE_LIBS_TOOLCHAIN_PATH}"
${ANDROID_NDK_PATH}/build/tools/make_standalone_toolchain.py --verbose \
--arch ${ANDROID_NDK_ARCH} --install-dir ${NATIVE_LIBS_TOOLCHAIN_PATH} \
--api ${ANDROID_PLATFORM_VER}
find "${PREFIX}/include/" -not -type d > "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles"
# Avoid problems with arm64 some libraries installing on lib64
ln -s "${PREFIX}/lib/" "${PREFIX}/lib64"
pushd "${PREFIX}/include/"
find . -not -type d > "${DUPLICATED_INCLUDES_LIST_FILE}"
popd
}
## This avoid <cmath> include errors due to -isystem and -I ordering issue
delete_copied_includes()
task_register deduplicate_includes
deduplicate_includes()
{
echo "delete_copied_includes()
################################################################################
################################################################################
################################################################################
"
cat "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles" | while read delFile ; do
rm "$delFile"
done
while read delFile ; do
mNewPath="${DUPLICATED_INCLUDES_DIR}/$delFile"
mkdir --verbose --parents "$(dirname "$mNewPath")"
mv --verbose "${PREFIX}/include/$delFile" "$mNewPath"
done < "${DUPLICATED_INCLUDES_LIST_FILE}"
}
## More information available at https://gitlab.com/relan/provisioners/merge_requests/1 and http://stackoverflow.com/a/34032216
task_register reduplicate_includes
reduplicate_includes()
{
pushd "${DUPLICATED_INCLUDES_DIR}"
find . -not -type d | while read delFile ; do
mv --verbose "${delFile}" "${PREFIX}/include/$delFile"
done
popd
}
# $1 optional prefix prepended only if return value is not empty
# $2 optional suffix appended only if return value is not empty
task_register get_qt_arch
get_qt_arch()
{
local QT_VERSION_COMP="$(echo $QT_VERSION | awk -F. '{print $1*1000000+$2*1000+$3}')"
local QT_ARCH=""
# Qt >= 5.15.0 ships all Android architectures toghether
[ "$QT_VERSION_COMP" -lt "5015000" ] &&
{
case "${ANDROID_NDK_ARCH}" in
"arm")
QT_ARCH="armv7"
;;
"arm64")
QT_ARCH="arm64_v8a"
;;
"x86")
QT_ARCH="x86"
;;
esac
echo "$1$QT_ARCH$2"
}
}
task_register get_qt_dir
get_qt_dir()
{
echo "${QT_INSTALL_PATH}/${QT_VERSION}/android$(get_qt_arch _)/"
}
## More information available at https://wiki.qt.io/Online_Installer_4.x
task_register install_qt_android
install_qt_android()
{
echo "install_qt_android()
################################################################################
################################################################################
################################################################################
"
[ "$QT_INSTALLER_JWT_TOKEN" == "" ] &&
{
echo "To run Qt installer QT_INSTALLER_JWT_TOKEN environement variable \
need to be set to a valid JWT token see https://wiki.qt.io/Online_Installer_4.x"
return -1
}
QT_VERSION_CODE=$(echo $QT_VERSION | tr -d .)
QT_INSTALL_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt
QT_INSTALLER="qt-unified-linux-x64-3.0.2-online.run"
verified_download $QT_INSTALLER $QT_ANDROID_INSTALLER_SHA256 \
http://master.qt.io/archive/online_installers/3.0/${QT_INSTALLER}
QT_VERSION_CODE="$(echo $QT_VERSION | tr -d .)"
QT_INSTALLER="qt-unified-linux-x86_64-${QT_INSTALLER_VERSION}-online.run"
tMajDotMinVer="$(echo $QT_INSTALLER_VERSION | awk -F. '{print $1"."$2}')"
verified_download $QT_INSTALLER $QT_INSTALLER_SHA256 \
"https://master.qt.io/archive/online_installers/${tMajDotMinVer}/${QT_INSTALLER}"
chmod a+x ${QT_INSTALLER}
QT_INSTALLER_SCRIPT="qt_installer_script.js"
cat << EOF > "${QT_INSTALLER_SCRIPT}"
function Controller() {
installer.autoRejectMessageBoxes();
installer.installationFinished.connect(function() {
gui.clickButton(buttons.NextButton);
});
var welcomePage = gui.pageWidgetByObjectName("WelcomePage");
welcomePage.completeChanged.connect(function() {
if (gui.currentPageWidget().objectName == welcomePage.objectName)
gui.clickButton(buttons.NextButton);
});
}
Controller.prototype.WelcomePageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.CredentialsPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.IntroductionPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.TargetDirectoryPageCallback = function() {
gui.currentPageWidget().TargetDirectoryLineEdit.setText("$QT_INSTALL_PATH");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ComponentSelectionPageCallback = function() {
var widget = gui.currentPageWidget();
// You can get these component names by running the installer with the
// --verbose flag. It will then print out a resource tree.
widget.deselectComponent("qt.tools.qtcreator");
widget.deselectComponent("qt.tools.doc");
widget.deselectComponent("qt.tools.examples");
widget.selectComponent("qt.$QT_VERSION_CODE.android_armv7");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.LicenseAgreementPageCallback = function() {
gui.currentPageWidget().AcceptLicenseRadioButton.setChecked(true);
gui.clickButton(buttons.NextButton);
}
Controller.prototype.StartMenuDirectoryPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ReadyForInstallationPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.FinishedPageCallback = function() {
var checkBoxForm = gui.currentPageWidget().LaunchQtCreatorCheckBoxForm;
if (checkBoxForm && checkBoxForm.launchQtCreatorCheckBox)
checkBoxForm.launchQtCreatorCheckBox.checked = false;
gui.clickButton(buttons.FinishButton);
}
EOF
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} --script ${QT_INSTALLER_SCRIPT}
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} \
install qt.qt5.${QT_VERSION_CODE}.android$(get_qt_arch _) \
--accept-licenses --accept-obligations --confirm-command \
--default-answer --no-default-installations \
--root "${QT_INSTALL_PATH}"
}
## More information available at retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136
task_register build_bzlib
build_bzlib()
{
echo "build_bzlib()
################################################################################
################################################################################
################################################################################
"
B_dir="bzip2-${BZIP2_SOURCE_VERSION}"
rm -rf $B_dir
@ -279,14 +463,9 @@ build_bzlib()
}
## More information available at http://doc.qt.io/qt-5/opensslsupport.html
task_register build_openssl
build_openssl()
{
echo "build_openssl()
################################################################################
################################################################################
################################################################################
"
B_dir="openssl-${OPENSSL_SOURCE_VERSION}"
rm -rf $B_dir
@ -308,6 +487,10 @@ build_openssl()
--openssldir="${SYSROOT}/etc/ssl"
# sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile
# sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile
# Avoid documentation build which is unneded and time consuming
echo "exit 0; " > util/process_docs.pl
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libssl.so*
@ -315,14 +498,9 @@ build_openssl()
cd ..
}
task_register build_sqlite
build_sqlite()
{
echo "build_sqlite()
################################################################################
################################################################################
################################################################################
"
B_dir="sqlite-autoconf-${SQLITE_SOURCE_VERSION}"
rm -rf $B_dir
@ -335,18 +513,13 @@ build_sqlite()
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libsqlite3.so*
# ${CC} -shared -o libsqlite3.so -fPIC sqlite3.o -ldl
# cp libsqlite3.so "${SYSROOT}/usr/lib"
cd ..
}
task_register build_sqlcipher
build_sqlcipher()
{
echo "build_sqlcipher()
################################################################################
################################################################################
################################################################################
"
task_run build_sqlite
B_dir="sqlcipher-${SQLCIPHER_SOURCE_VERSION}"
rm -rf $B_dir
@ -358,14 +531,14 @@ build_sqlcipher()
tar -xf $T_file
cd $B_dir
case "${ANDROID_NDK_ARCH}" in
"arm64")
# SQLCipher config.sub is outdated and doesn't recognize newer architectures
rm config.sub
autoreconf --verbose --install --force
automake --add-missing --copy --force-missing
;;
esac
# case "${ANDROID_NDK_ARCH}" in
# "arm64")
# # SQLCipher config.sub is outdated and doesn't recognize newer architectures
# rm config.sub
# autoreconf --verbose --install --force
# automake --add-missing --copy --force-missing
# ;;
# esac
./configure --with-pic --build=$(sh ./config.guess) \
--host=${cArch}-linux \
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}" \
@ -377,14 +550,9 @@ build_sqlcipher()
cd ..
}
task_register build_libupnp
build_libupnp()
{
echo "build_libupnp()
################################################################################
################################################################################
################################################################################
"
B_dir="pupnp-release-${LIBUPNP_SOURCE_VERSION}"
B_ext=".tar.gz"
B_file="${B_dir}${B_ext}"
@ -408,14 +576,9 @@ build_libupnp()
cd ..
}
task_register build_rapidjson
build_rapidjson()
{
echo "build_rapidjson()
################################################################################
################################################################################
################################################################################
"
B_dir="rapidjson-${RAPIDJSON_SOURCE_VERSION}"
D_file="${B_dir}.tar.gz"
verified_download $D_file $RAPIDJSON_SOURCE_SHA256 \
@ -424,42 +587,25 @@ build_rapidjson()
cp -r "${B_dir}/include/rapidjson/" "${PREFIX}/include/rapidjson"
}
task_register build_restbed
build_restbed()
{
echo "build_restbed()
################################################################################
################################################################################
################################################################################
"
S_dir="restbed"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$RESTBED_SOURCE_REPO" "${RESTBED_SOURCE_VERSION}" \
"dependency/asio" "dependency/catch"
[ -d restbed ] || git clone --depth=2000 https://github.com/Corvusoft/restbed.git
cd restbed
# git fetch --tags
# git checkout tags/${RESTBED_SOURCE_VERSION}
git checkout ${RESTBED_SOURCE_VERSION}
git submodule update --init dependency/asio
git submodule update --init dependency/catch
git submodule update --init dependency/kashmir
cd ..
rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
rm -rf "$B_dir"; mkdir "$B_dir"
pushd "$B_dir"
andro_cmake -DBUILD_TESTS=OFF -DBUILD_SSL=OFF -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
cp "${PREFIX}/library/librestbed.a" "${PREFIX}/lib/"
cd ..
popd
}
task_register build_udp-discovery-cpp
build_udp-discovery-cpp()
{
echo "build_udp-discovery-cpp()
################################################################################
################################################################################
################################################################################
"
S_dir="udp-discovery-cpp"
[ -d $S_dir ] || git clone $UDP_DISCOVERY_CPP_SOURCE $S_dir
cd $S_dir
@ -468,23 +614,16 @@ build_udp-discovery-cpp()
B_dir="udp-discovery-cpp-build"
rm -rf ${B_dir}; mkdir ${B_dir}; cd ${B_dir}
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../udp-discovery-cpp
andro_cmake -B. -H../$S_dir
make -j${HOST_NUM_CPU}
cp libudp-discovery.a "${PREFIX}/lib/"
cp ../$S_dir/*.hpp "${PREFIX}/include/"
cd ..
}
task_register build_xapian
build_xapian()
{
echo "build_xapian()
################################################################################
################################################################################
################################################################################
"
B_dir="xapian-core-${XAPIAN_SOURCE_VERSION}"
D_file="$B_dir.tar.xz"
verified_download $D_file $XAPIAN_SOURCE_SHA256 \
@ -503,15 +642,12 @@ build_xapian()
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}"
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_miniupnpc
build_miniupnpc()
{
echo "build_miniupnpc()
################################################################################
################################################################################
################################################################################
"
S_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}"
B_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}-build"
D_file="$S_dir.tar.gz"
@ -521,29 +657,228 @@ build_miniupnpc()
tar -xf $D_file
mkdir $B_dir
cd $B_dir
cmake \
andro_cmake \
-DUPNPC_BUILD_STATIC=TRUE \
-DUPNPC_BUILD_SHARED=FALSE \
-DUPNPC_BUILD_TESTS=FALSE \
-DUPNPC_BUILD_SAMPLE=FALSE \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -S../$S_dir
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
build_toolchain
[ "${INSTALL_QT_ANDROID}X" != "trueX" ] || install_qt_android
build_bzlib
build_openssl
build_sqlite
build_sqlcipher
build_libupnp
build_rapidjson
build_restbed
build_udp-discovery-cpp
build_xapian
build_miniupnpc
delete_copied_includes
task_register build_zlib
build_zlib()
{
S_dir="zlib-${ZLIB_SOURCE_VERSION}"
B_dir="zlib-${ZLIB_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $ZLIB_SOURCE_SHA256 \
http://www.zlib.net/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
cd $B_dir
andro_cmake -B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libz.so*
cd ..
}
echo NATIVE_LIBS_TOOLCHAIN_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}
task_register build_libpng
build_libpng()
{
task_run build_zlib
S_dir="libpng-${LIBPNG_SOURCE_VERSION}"
B_dir="libpng-${LIBPNG_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $LIBPNG_SOURCE_SHA256 \
https://download.sourceforge.net/libpng/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
# libm is part of bionic An android
sed -i -e 's/find_library(M_LIBRARY m)/set(M_LIBRARY "")/' $S_dir/CMakeLists.txt
# Disable hardware acceleration as they are problematic for Android
# compilation and are not supported by all phones, it is necessary to fiddle
# with CMakeLists.txt as libpng 1.6.37 passing it as cmake options seems not
# working properly
# https://github.com/imagemin/optipng-bin/issues/97
# https://github.com/opencv/opencv/issues/7600
echo "add_definitions(-DPNG_ARM_NEON_OPT=0)" >> $S_dir/CMakeLists.txt
mkdir $B_dir
pushd $B_dir
HW_OPT="OFF"
# [ "$ANDROID_PLATFORM_VER" -ge "22" ] && HW_OPT="ON"
andro_cmake \
-DPNG_SHARED=OFF \
-DPNG_STATIC=ON \
-DPNG_TESTS=OFF \
-DPNG_HARDWARE_OPTIMIZATIONS=$HW_OPT \
-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
popd
}
task_register build_libjpeg
build_libjpeg()
{
S_dir="jpeg-${LIBJPEG_SOURCE_VERSION}"
D_file="jpegsrc.v${LIBJPEG_SOURCE_VERSION}.tar.gz"
verified_download $D_file $LIBJPEG_SOURCE_SHA256 \
https://www.ijg.org/files/$D_file
rm -rf $S_dir
tar -xf $D_file
cd $S_dir
./configure --with-pic --prefix="${PREFIX}" --host=${cArch}-linux
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libjpeg.so*
cd ..
}
task_register build_tiff
build_tiff()
{
S_dir="tiff-${TIFF_SOURCE_VERSION}"
B_dir="${S_dir}-build"
D_file="tiff-${TIFF_SOURCE_VERSION}.tar.gz"
verified_download $D_file $TIFF_SOURCE_SHA256 \
https://download.osgeo.org/libtiff/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
# Disable tools building, not needed for retroshare, and depending on some
# OpenGL headers not available on Android
echo "" > $S_dir/tools/CMakeLists.txt
# Disable tests building, not needed for retroshare, and causing linker
# errors
echo "" > $S_dir/test/CMakeLists.txt
# Disable extra tools building, not needed for retroshare, and causing
# linker errors
echo "" > $S_dir/contrib/CMakeLists.txt
# Disable more unneded stuff
echo "" > $S_dir/build/CMakeLists.txt
echo "" > $S_dir/html/CMakeLists.txt
echo "" > $S_dir/man/CMakeLists.txt
echo "" > $S_dir/port/CMakeLists.txt
# Change to static library build
sed -i 's\add_library(tiff\add_library(tiff STATIC\' \
$S_dir/libtiff/CMakeLists.txt
cd $B_dir
#TODO: build dependecies to support more formats
andro_cmake \
-Dlibdeflate=OFF -Djbig=OFF -Dlzma=OFF -Dzstd=OFF -Dwebp=OFF \
-Djpeg12=OFF \
-Dcxx=OFF \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_cimg
build_cimg()
{
task_run build_libpng
task_run build_libjpeg
task_run build_tiff
S_dir="CImg-${CIMG_SOURCE_VERSION}"
D_file="CImg_${CIMG_SOURCE_VERSION}.zip"
verified_download $D_file $CIMG_SOURCE_SHA256 \
https://cimg.eu/files/${D_file}
unzip -o $D_file
cp --archive --verbose "$S_dir/CImg.h" "$PREFIX/include/"
}
task_register build_phash
build_phash()
{
task_run build_cimg
S_dir="pHash"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$PHASH_SOURCE_REPO" "${PHASH_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -DPHASH_DYNAMIC=OFF -DPHASH_STATIC=ON -B. -H../pHash
make -j${HOST_NUM_CPU}
make install
popd
}
task_register build_mvptree
build_mvptree()
{
S_dir="mvptree"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$MVPTREE_SOURCE_REPO" "${MVPTREE_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
popd
}
task_register get_native_libs_toolchain_path
get_native_libs_toolchain_path()
{
echo ${NATIVE_LIBS_TOOLCHAIN_PATH}
}
task_register build_default_toolchain
build_default_toolchain()
{
task_run bootstrap_toolchain || return $?
task_run build_bzlib || return $?
task_run build_openssl || return $?
task_run build_sqlcipher || return $?
task_run build_rapidjson || return $?
task_run build_restbed || return $?
task_run build_udp-discovery-cpp || return $?
task_run build_xapian || return $?
task_run build_miniupnpc || return $?
task_run build_phash || return $?
task_run deduplicate_includes || return $?
task_run get_native_libs_toolchain_path || return $?
}
if [ "$1" == "" ]; then
rm -rf "$REPORT_DIR"
mkdir -p "$REPORT_DIR"
cat "$0" > "$REPORT_DIR/build_script"
env > "$REPORT_DIR/build_env"
build_default_toolchain
else
# do not delete report directory in this case so we can reuse material
# produced by previous run, like deduplicated includes
mkdir -p "$REPORT_DIR"
while [ "$1" != "" ] ; do
task_zap $1
task_run $1 || exit $?
shift
done
fi

46
build_scripts/GitlabCI/Android.Dockerfile Normal file
View File

@ -0,0 +1,46 @@
ARG ANDROID_NDK_ARCH=arm
FROM registry.gitlab.com/retroshare/retroshare:android_${ANDROID_NDK_ARCH}_base
RUN apt-get update -y && apt-get upgrade -y
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
WORKDIR /RetroShare
RUN git remote add testing $REPO_URL && \
git fetch --tags testing $REPO_BRANCH && \
git reset --hard testing/$REPO_BRANCH && \
git --no-pager log --max-count 1
RUN rm -rf /jsonapi-generator-build ; mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
# ARG declared before FROM get wiped after it, so we need declaring it again
ARG ANDROID_NDK_ARCH=arm
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
ARG ANDROID_PLATFORM_VER=16
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle

35
build_scripts/GitlabCI/base.Dockerfile
View File

@ -2,42 +2,43 @@
## force cloning a new
## To prepare an image suitable as base for Gitlab CI use
# docker build -t "${CI_REGISTRY_IMAGE}:base" --build-arg KEEP_SOURCE=true --build-arg REPO_DEPTH="" -f base.Dockerfile .
## Now you need to tag it so you can later push it
# docker tag ${ID_OF_THE_CREATED_IMAGE} registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
# export CI_REGISTRY_IMAGE="registry.gitlab.com/retroshare/retroshare:base"
# docker build -t "${CI_REGISTRY_IMAGE}" -f base.Dockerfile .
## To push it to gitlab CI registry you need first to login and the to push
# docker login registry.gitlab.com
# docker push registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
# docker push "${CI_REGISTRY_IMAGE}"
## To run the container
# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}:base" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
FROM ubuntu
ARG CACHEBUST=0
RUN \
apt-get update -y && apt-get upgrade -y && \
apt-get install -y build-essential libssl-dev libbz2-dev libsqlite3-dev \
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get upgrade -y -qq && \
apt-get install -y -qq build-essential cimg-dev libssl-dev libbz2-dev \
libsqlite3-dev \
libsqlcipher-dev libupnp-dev pkg-config libz-dev \
qt5-default libxapian-dev qttools5-dev doxygen rapidjson-dev \
git cmake curl
RUN git clone --depth 1 https://github.com/aetilius/pHash.git && \
rm -rf pHash-build && mkdir pHash-build && cd pHash-build && \
cmake -B. -H../pHash -DCMAKE_INSTALL_PREFIX=/usr && \
make -j$(nproc) && make install && cd .. && \
rm -rf pHash-build pHash
ARG FRESHCLONE=0
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
ARG REPO_DEPTH="--depth 2000"
ARG KEEP_SOURCE=false
RUN apt-get update -y && apt-get upgrade -y
RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
git fetch --tags && cd ..
RUN \
mkdir RetroShare-build && cd RetroShare-build && \
RUN mkdir RetroShare-build && cd RetroShare-build && \
qmake ../RetroShare \
CONFIG+=no_retroshare_plugins CONFIG+=ipv6 \
CONFIG+=no_retroshare_plugins \
CONFIG+=retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=rs_deep_search && \
(make -j$(nproc) || make -j$(nproc) || make) && make install && \
cd .. && rm -rf RetroShare-build && ($KEEP_SOURCE || rm -rf RetroShare)
cd .. && rm -rf RetroShare-build

9
build_scripts/GitlabCI/getRepoArgs.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
[ -n "$CI_MERGE_REQUEST_ID" ] &&
echo \
--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
echo \
--build-arg REPO_URL="$CI_REPOSITORY_URL" \
--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

2
build_scripts/OBS

@ -1 +1 @@
Subproject commit 6d6ee4aab171c23c603f7e813ba66d22192393ad
Subproject commit b260e58346b1eec782bdf88a7e8f3c9d36fd3ecb

2
build_scripts/Windows-msys2/build/pack.bat
View File

@ -147,8 +147,10 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
call :copy_dependencies "%%D" "%RsDeployPath%"
)
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

13
build_scripts/Windows-msys2/env/tools/prepare-msys2.bat vendored
View File

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
%cecho% info "Remove previous MSYS2 version"
call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
@ -41,12 +44,10 @@ if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download M
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows-msys2/env/tools/root/update-msys2.bat vendored
View File

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi
View File

@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${DEPLOYDIR}\qss\*.*"
File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

2
build_scripts/Windows/build/pack.bat
View File

@ -132,8 +132,10 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

17
build_scripts/Windows/env/tools/prepare-msys2.bat vendored
View File

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
set CMakeInstall=cmake-3.19.0-win32-x86.zip
set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.19.0/%CMakeInstall%
set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
@ -64,17 +67,11 @@ if "%FoundProfile%"=="0" (
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Update keyring"
"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows/env/tools/root/update-msys2.bat vendored
View File

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows/installer/retroshare-Qt5.nsi
View File

@ -334,7 +334,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

6
libretroshare/src/chat/distributedchat.cc
View File

@ -1513,7 +1513,7 @@ ChatLobbyVirtualPeerId DistributedChatService::makeVirtualPeerId(ChatLobbyId lob
}
void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
bool DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
{
RsStackMutex stack(mDistributedChatMtx); /********** STACK LOCKED MTX ******/
@ -1525,10 +1525,12 @@ void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
if(it == _lobby_invites_queue.end())
{
std::cerr << " (EE) lobby invite not in cache!!" << std::endl;
return ;
return false;
}
_lobby_invites_queue.erase(it) ;
return true;
}
bool DistributedChatService::joinVisibleChatLobby(const ChatLobbyId& lobby_id,const RsGxsId& gxs_id)

2
libretroshare/src/chat/distributedchat.h
View File

@ -61,7 +61,7 @@ class DistributedChatService
void getChatLobbyList(std::list<ChatLobbyId>& clids) ;
bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& clinfo) ;
bool acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& identity) ;
void denyLobbyInvite(const ChatLobbyId& id) ;
bool denyLobbyInvite(const ChatLobbyId& id) ;
void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
void invitePeerToLobby(const ChatLobbyId&, const RsPeerId& peer_id,bool connexion_challenge = false) ;
void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;

191
libretroshare/src/deep_search/commonutils.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,54 +18,49 @@
* *
*******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
#ifndef XAPIAN_AT_LEAST
/// Added in Xapian 1.4.2.
#define XAPIAN_AT_LEAST(A,B,C) \
(XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
try
{
#if XAPIAN_AT_LEAST(1,3,2)
std::unique_ptr<Xapian::Database> dbPtr(
new Xapian::Database(path, flags) );
#else
std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
if(flags)
{
RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
"library version: ", XAPIAN_VERSION, " < 1.3.2" );
}
#endif
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
catch(Xapian::DatabaseOpeningError& e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
catch(Xapian::DatabaseLockError&)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
return date;
}
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(rstime_t(50), maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
// strip also CSS inside <style></style>
oSize = retVal.size();
auto styleTagBegin(retVal.find("<style"));
if(styleTagBegin < oSize)
{
auto styleEnd(retVal.find("</style>", styleTagBegin));
if(styleEnd < oSize)
retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
}
std::string::size_type oPos;
std::string::size_type cPos;
int itCount = 0;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
// Avoid infinite loop with crafty input
if(itCount > 1000)
{
RS_WARN( "Breaking stripping loop due to max allowed iterations ",
"rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
break;
}
++itCount;
}
return retVal;
}
}

34
libretroshare/src/deep_search/commonutils.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h>
#include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
}

75
libretroshare/src/deep_search/filesindex.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,48 +18,48 @@
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
if(db)
{
oldDoc = db.get_document(*pIt);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
if( pIt != db->postlist_end(idTerm) )
{
oldDoc = db->get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
/* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
}
db.reset(); // Release DB read lock ASAP
}
Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
RS_DBG3(hash);
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
mWriteQueue.push([hash](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + hash.toStdString()); });
db->delete_document("Q" + hash.toStdString());
return true;
return std::error_condition();
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
std::error_condition DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}

23
libretroshare/src/deep_search/filesindex.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
@ -29,6 +26,9 @@
#include <map>
#include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
std::error_condition indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

208
libretroshare/src/deep_search/forumsindex.cpp Normal file
View File

@ -0,0 +1,208 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/forumsindex.hpp"
#include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "retroshare/rsgxsforums.h"
#include "util/rsdebuglevel4.h"
std::error_condition DeepForumsIndex::search(
const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
{
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepForumsSearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
{
RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
forumIndexId.setQueryKV(
RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
return forumIndexId.toString();
}
/*static*/ std::string DeepForumsIndex::postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
return postIndexId.toString();
}
std::error_condition DeepForumsIndex::indexForumGroup(
const RsGxsForumGroup& forum )
{
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
termgenerator.index_text(
DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
termgenerator.index_text(forum.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(forum.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(forum.mDescription);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink);
/* Store some fields for display purposes. Retrieved later to provide the
* matching snippet on search */
doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
/* We use the identifier to ensure each object ends up in the database only
* once no matter how many times we run the indexer.
* "Q" prefix is a Xapian convention for unique id term. */
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumFromIndex(
const RsGxsGroupId& grpId )
{
mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + forumIndexId(grpId)); });
return std::error_condition();
}
std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
{
RS_DBG4(post);
const auto& groupId = post.mMeta.mGroupId;
const auto& msgId = post.mMeta.mMsgId;
if(groupId.isNull() || msgId.isNull())
{
RS_ERR("Got post with invalid id ", post);
print_stacktrace();
return std::errc::invalid_argument;
}
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// Avoid indexing RetroShare-gui HTML tags
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
termgenerator.index_text(cleanMsg, 1, "XD" );
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(groupId, msgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

81
libretroshare/src/deep_search/forumsindex.hpp Normal file
View File

@ -0,0 +1,81 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <system_error>
#include <vector>
#include <xapian.h>
#include "util/rstime.h"
#include "retroshare/rsgxsforums.h"
#include "retroshare/rsevents.h"
#include "deep_search/commonutils.hpp"
struct DeepForumsSearchResult
{
std::string mUrl;
double mWeight;
std::string mSnippet;
};
struct DeepForumsIndex
{
explicit DeepForumsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
std::error_condition search( const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results,
uint32_t maxResults = 100 );
std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
std::error_condition indexForumPost(const RsGxsForumMsg& post);
std::error_condition removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
static std::string dbDefaultPath();
private:
static std::string forumIndexId(const RsGxsGroupId& grpId);
static std::string postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
};

146
libretroshare/src/file_sharing/dir_hierarchy.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,6 +31,7 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -70,7 +73,8 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
mTotalFiles = 0 ;
}
bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
bool InternalFileHierarchyStorage::getDirHashFromIndex(
const DirectoryStorage::EntryIndex& index, RsFileHash& hash ) const
{
if(!checkIndex(index,FileStorageNode::TYPE_DIR))
return false ;
@ -79,6 +83,7 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E
return true;
}
bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mDirHashes.find(hash) ;
@ -88,30 +93,34 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di
index = it->second;
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them.
if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast<DirEntry*>(mNodes[index])->dir_hash != hash)
/* make sure the hash actually points to some existing directory. If not,
* remove it. This is an opportunistic update of dir hashes: when we need
* them, we check them. */
if( !checkIndex(index, FileStorageNode::TYPE_DIR) ||
static_cast<DirEntry*>(mNodes[index])->dir_hash != hash )
{
std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl;
RS_INFO("removing non existing dir hash: ", hash, " from dir hash list");
mDirHashes.erase(it);
return false;
}
return true;
}
bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mFileHashes.find(hash) ;
if(it == mFileHashes.end())
return false;
bool InternalFileHierarchyStorage::getIndexFromFileHash(
const RsFileHash& hash, DirectoryStorage::EntryIndex& index )
{
auto it = std::as_const(mFileHashes).find(hash);
if(it == mFileHashes.end()) return false;
index = it->second;
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them.
if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast<FileEntry*>(mNodes[index])->file_hash != hash)
/* make sure the hash actually points to some existing file. If not, remove
* it. This is an opportunistic update of file hashes: when we need them,
* we check them. */
if( !checkIndex(it->second, FileStorageNode::TYPE_FILE) ||
static_cast<FileEntry*>(mNodes[index])->file_hash != hash )
{
std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl;
RS_INFO("removing non existing file hash: ", hash, " from file hash list");
mFileHashes.erase(it);
return false;
}
@ -155,7 +164,10 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
return e < mNodes.size() && mNodes[e] != NULL ;
}
bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::set<std::string>& subdirs, const RsFileHash& random_hash_seed)
bool InternalFileHierarchyStorage::updateSubDirectoryList(
const DirectoryStorage::EntryIndex& indx,
const std::set<std::string>& subdirs,
const RsFileHash& random_hash_seed )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
@ -284,10 +296,17 @@ bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,
return true;
}
bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
bool InternalFileHierarchyStorage::updateSubFilesList(
const DirectoryStorage::EntryIndex& indx,
const std::map<std::string,DirectoryStorage::FileTS>& subfiles,
std::map<std::string,DirectoryStorage::FileTS>& new_files )
{
if(!checkIndex(indx, FileStorageNode::TYPE_DIR))
{
RS_ERR("indx: ", indx, std::errc::not_a_directory);
print_stacktrace();
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
new_files = subfiles ;
@ -312,9 +331,11 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
continue;
}
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size
// file is newer and/or has different size
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)
{
f.file_hash.clear(); // hash needs recomputing
// hash needs recomputing
f.file_hash.clear();
f.file_modtime = it->second.modtime;
f.file_size = it->second.size;
@ -342,11 +363,14 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
}
return true;
}
bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
bool InternalFileHierarchyStorage::updateHash(
const DirectoryStorage::EntryIndex& file_index, const RsFileHash& hash )
{
if(!checkIndex(file_index, FileStorageNode::TYPE_FILE))
{
std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
RS_ERR( "Cannot update file at index ", file_index,
". Not a valid index, or not a file." );
print_stacktrace();
return false;
}
#ifdef DEBUG_DIRECTORY_STORAGE
@ -436,13 +460,19 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
return mNodes.size()-1 ;
}
bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,rstime_t most_recent_time,rstime_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
bool InternalFileHierarchyStorage::updateDirEntry(
const DirectoryStorage::EntryIndex& indx, const std::string& dir_name,
rstime_t most_recent_time, rstime_t dir_modtime,
const std::vector<RsFileHash>& subdirs_hash,
const std::vector<FileEntry>& subfiles_array )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
{
std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
RS_ERR( "cannot update dir at index ", indx, ". Not a valid index, or "
"not an existing dir." );
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx]));
#ifdef DEBUG_DIRECTORY_STORAGE
@ -703,14 +733,14 @@ const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorag
return NULL ;
}
const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::DirEntry*
InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return NULL ;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return nullptr;
return static_cast<DirEntry*>(mNodes[indx]);
}
const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::FileEntry*
InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
return NULL ;
@ -754,7 +784,10 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,DirectorySt
class DirectoryStorageExprFileEntry: public RsRegularExpression::ExpFileEntry
{
public:
DirectoryStorageExprFileEntry(const InternalFileHierarchyStorage::FileEntry& fe,const InternalFileHierarchyStorage::DirEntry& parent) : mFe(fe),mDe(parent) {}
DirectoryStorageExprFileEntry(
const InternalFileHierarchyStorage::FileEntry& fe,
const InternalFileHierarchyStorage::DirEntry& parent ) :
mFe(fe), mDe(parent) {}
inline virtual const std::string& file_name() const { return mFe.file_name ; }
inline virtual uint64_t file_size() const { return mFe.file_size ; }
@ -768,14 +801,18 @@ private:
const InternalFileHierarchyStorage::DirEntry& mDe ;
};
int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const
int InternalFileHierarchyStorage::searchBoolExp(
RsRegularExpression::Expression* exp,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL && exp->eval(
DirectoryStorageExprFileEntry(*static_cast<const FileEntry*>(mNodes[it->second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it->second]->parent_index])
for(auto& it: std::as_const(mFileHashes))
if(mNodes[it.second])
if(exp->eval(
DirectoryStorageExprFileEntry(
*static_cast<const FileEntry*>(mNodes[it.second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it.second]->parent_index])
) ))
results.push_back(it->second);
results.push_back(it.second);
return 0;
}
@ -784,26 +821,42 @@ int InternalFileHierarchyStorage::searchTerms(
const std::list<std::string>& terms,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
// most entries are likely to be files, so we could do a linear search over the entries tab.
// instead we go through the table of hashes.
/* most entries are likely to be files, so we could do a linear search over
* the entries tab. Instead we go through the table of hashes.*/
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL)
for(auto& it : std::as_const(mFileHashes))
{
const std::string &str1 = static_cast<FileEntry*>(mNodes[it->second])->file_name;
// node may be null for some hash waiting to be deleted
if(mNodes[it.second])
{
rs_view_ptr<FileEntry> tFileEntry =
static_cast<FileEntry*>(mNodes[it.second]);
for(std::list<std::string>::const_iterator iter(terms.begin()); iter != terms.end(); ++iter)
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
std::string tFilename = tFileEntry->file_name;
if(tFileEntry->file_name.find("/") != std::string::npos)
{
std::string _tParentDir;
RsDirUtil::splitDirFromFile(
tFileEntry->file_name, _tParentDir, tFilename );
}
for(auto& termIt : std::as_const(terms))
{
/* always ignore case */
const std::string &str2 = (*iter);
if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() ))
if(tFilename.end() != std::search(
tFilename.begin(), tFilename.end(),
termIt.begin(), termIt.end(),
RsRegularExpression::CompareCharIC() ))
{
results.push_back(it->second);
results.push_back(it.second);
break;
}
}
}
}
return 0;
}
@ -958,7 +1011,8 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
{
std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
RS_ERR(s);
print_stacktrace();
return false;
}

259
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,6 +32,7 @@
#include "directory_storage.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -67,8 +70,11 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
{ return mStorage->getSubFileIndex(mParentIndex, mFileTabIndex); }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
{ return mStorage->getSubDirIndex(mParentIndex, mDirTabIndex); }
DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
@ -139,7 +145,9 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx, const std:
mChanged = true ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
bool DirectoryStorage::updateSubFilesList(
const EntryIndex& indx, const std::map<std::string,FileTS>& subfiles,
std::map<std::string,FileTS>& new_files )
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
@ -348,7 +356,8 @@ int LocalDirectoryStorage::searchHash(const RsFileHash& hash, RsFileHash& real_h
return false ;
}
void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
void LocalDirectoryStorage::setSharedDirectoryList(
const std::list<SharedDirInfo>& lst )
{
std::set<std::string> dirs_with_new_virtualname ;
bool dirs_with_changed_flags = false ;
@ -379,7 +388,9 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo
virtual_names.insert(candidate_virtual_name) ;
}
// now for each member of the processed list, check if it is an existing shared directory that has been changed. If so, we need to update the dir TS of that directory
/* now for each member of the processed list, check if it is an existing
* shared directory that has been changed. If so, we need to update the
* dir TS of that directory */
std::map<std::string,SharedDirInfo> new_dirs ;
@ -538,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX
@ -617,46 +628,76 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil
return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
}
bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
bool LocalDirectoryStorage::locked_getFileSharingPermissions(
const EntryIndex& indx, FileStorageFlags& flags,
std::list<RsNodeGroupId>& parent_groups )
{
flags.clear();
parent_groups.clear();
std::string base_dir;
/* We got a request for root directory no need to do anything more after
* clearing outputs */
if(!indx) return true;
const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
using FileStorageNode = InternalFileHierarchyStorage::FileStorageNode;
using EntryIndex = DirectoryStorage::EntryIndex;
if(n == NULL)
rs_view_ptr<const FileStorageNode> n = mFileHierarchy->getNode(indx);
if(!n)
{
RS_ERR("Node for index: ", indx, "not found");
print_stacktrace();
return false;
for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
{
const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
if(e == NULL)
break ;
if(e->parent_index == 0)
{
base_dir = e->dir_name ;
break ;
}
i = e->parent_index ;
}
if(!base_dir.empty())
// Climb down node tree up to root + 1
EntryIndex curIndex = indx;
while (n->parent_index)
{
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
curIndex = n->parent_index;
n = mFileHierarchy->getNode(curIndex);
}
// Retrieve base name
std::string tBaseName;
switch (n->type())
{
// Handle single file shared case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE:
tBaseName = mFileHierarchy->getFileEntry(curIndex)->file_name;
break;
// Handle shared directory case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR:
tBaseName = mFileHierarchy->getDirEntry(curIndex)->dir_name;
break;
default:
RS_ERR("Got unhandled node type: ", n->type());
print_stacktrace();
return false;
}
// Use base name to retrieve sharing permissions
if(!tBaseName.empty())
{
auto it = std::as_const(mLocalDirs).find(tBaseName);
if(it == mLocalDirs.end())
{
std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
RS_ERR( "base name \"", tBaseName,
"\" for index: ", indx, " not found in shared dir list." );
print_stacktrace();
return false;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
else
{
RS_ERR("base name for indx: ", indx, " is empty");
print_stacktrace();
return false;
}
return true;
}
@ -705,18 +746,22 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
return it->second.virtualname + "/" + res;
}
bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
bool LocalDirectoryStorage::serialiseDirEntry(
const EntryIndex& indx, RsTlvBinaryData& bindata,
const RsPeerId& client_id )
{
RS_STACK_MUTEX(mDirStorageMtx);
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
const InternalFileHierarchyStorage::DirEntry* dir =
mFileHierarchy->getDirEntry(indx);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl;
#endif
if(dir == NULL)
if(!dir)
{
std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
RS_ERR("Cannot find entry ", indx);
return false;
}
@ -725,18 +770,28 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
FileStorageFlags node_flags;
std::list<RsNodeGroupId> node_groups;
// for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
/* for each subdir, compute the node flags and groups, then ask rsPeers to
* compute the mask that result from these flags for the particular peer
* supplied in parameter */
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
if(indx != 0 || (
locked_getFileSharingPermissions(
dir->subdirs[i], node_flags, node_groups ) &&
( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
{
RsFileHash hash;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
RS_ERR( "Cannot get hash from subdir index: ",
dir->subdirs[i], ". Weird bug." );
print_stacktrace();
return false;
}
allowed_subdirs.push_back(hash);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl;
#endif
@ -746,50 +801,77 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
std::cerr << " not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl;
#endif
// now count the files that do not have a null hash (meaning the hash has indeed been computed)
/* now count the files that do not have a null hash (meaning the hash has
* indeed been computed), also in case files are shared singularly (without
* a shared directory) so they are child of root check browsability
* permission */
uint32_t allowed_subfiles = 0;
for(uint32_t i=0; i<dir->subfiles.size(); ++i)
{
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file != NULL && !file->file_hash.isNull())
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
if(file != nullptr && !file->file_hash.isNull()
&& ( indx !=0 || (
locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ) &&
rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
allowed_subfiles++;
}
unsigned char *section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
if(!section_data)
return false ;
unsigned char* section_data = (unsigned char *)
rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!section_data) return false;
uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
uint32_t section_offset = 0;
// we need to send:
// - the name of the directory, its TS
// - the index entry for each subdir (the updte TS are exchanged at a higher level)
// - the file info for each subfile
//
/* we need to send:
* - the name of the directory, its TS
* - the index entry for each subdir (the updte TS are exchanged at a
* higher level)
* - the file info for each subfile */
std::string virtual_dir_name = locked_getVirtualDirName(indx);
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) { free(section_data); return false ;}
/* Manual serialization AGAIN! This is terrible and should be ported to the
* new serialization system ASAP! */
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_DIR_NAME, virtual_dir_name ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RECURS_MODIF_TS,
(uint32_t)dir->dir_most_recent_time ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)dir->dir_modtime ))
{ free(section_data); return false;}
// serialise number of subdirs and number of subfiles
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles )) { free(section_data); return false ;}
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subdirs.size() ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subfiles ))
{ free(section_data); return false; }
// serialise subdirs entry indexes
for(uint32_t i=0; i<allowed_subdirs.size(); ++i)
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i] )) { free(section_data); return false ;}
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_ENTRY_INDEX, allowed_subdirs[i] ))
{ free(section_data); return false; }
// serialise directory subfiles, with info for each of them
unsigned char *file_section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
unsigned char* file_section_data =
(unsigned char *) rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!file_section_data)
{
free(section_data);
@ -802,22 +884,60 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
{
uint32_t file_section_offset = 0;
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
if(file == NULL || file->file_hash.isNull())
if(file == nullptr || file->file_hash.isNull())
{
std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
RS_INFO( "skipping unhashed or Null file entry ",
dir->subfiles[i], " to get/send file info." );
continue;
}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) { free(section_data);free(file_section_data);return false ;}
if(indx == 0)
{
if(!locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ))
{
RS_ERR( "Failure getting sharing permission for single file: ",
dir->subfiles[i] );
print_stacktrace();
continue;
}
if(!( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ))
{
RS_INFO( "Skipping single file shared without browse "
"permission" );
continue;
}
}
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_NAME, file->file_name ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SIZE, file->file_size ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SHA1_HASH, file->file_hash ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)file->file_modtime ))
{ free(section_data); free(file_section_data); return false; }
// now write the whole string into a single section in the file
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) { free(section_data); free(file_section_data);return false ;}
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,
file_section_data, file_section_offset ))
{ free(section_data); free(file_section_data); return false; }
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl;
@ -829,7 +949,8 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary
std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
#endif
bindata.bin_data = realloc(section_data,section_offset) ; // This discards the possibly unused trailing bytes in the end of section_data
// Discards the possibly unused trailing bytes in the end of section_data
bindata.bin_data = realloc(section_data,section_offset);
bindata.bin_len = section_offset;
return true;

232
libretroshare/src/file_sharing/directory_updater.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,13 +21,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
******************************************************************************/
#include "util/cxx17retrocompat.h"
#include "util/folderiterator.h"
#include "util/rstime.h"
#include "rsserver/p3face.h"
#include "directory_storage.h"
#include "directory_updater.h"
#include "file_sharing_defaults.h"
#include "util/rsdebuglevel3.h"
//#define DEBUG_LOCAL_DIR_UPDATER 1
@ -123,51 +127,99 @@ bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
{
if(mHashSalt.isNull())
{
std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
RS_ERR("no salt value in LocalDirectoryUpdater");
print_stacktrace();
return false;
}
mIsChecking = true;
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
/* recursive update algorithm works that way:
* - the external loop starts on the shared directory list and goes through
* sub-directories
* - at the same time, it updates the local list of shared directories.
* A single sweep is performed over the whole directory structure.
* - the information that is costly to compute (the hash) is stored
* externally into a separate structure.
* - doing so, changing directory names or moving files between directories
* does not cause a re-hash of the content. */
std::list<SharedDirInfo> shared_directory_list;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
std::set<std::string> sub_dir_list;
// We re-check that each dir actually exists. It might have been removed from the disk.
/* Support also single files sharing as it make much more sense on some
* platforms like Android */
std::map<std::string, DirectoryStorage::FileTS> singleFilesMap;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
if(RsDirUtil::checkDirectory( (*real_dir_it).filename ) )
sub_dir_list.insert( (*real_dir_it).filename ) ;
/* We re-check that each dir actually exists. It might have been removed
* from the disk. Accept also single files not just directories. */
for(auto& realDir: std::as_const(shared_directory_list))
{
const auto& fPath = realDir.filename;
if(RsDirUtil::checkDirectory(fPath))
sub_dir_list.insert(fPath);
else if (RsDirUtil::fileExists(fPath))
{
rstime_t lastWrite= RsDirUtil::lastWriteTime(fPath);
if(time(nullptr) >= lastWrite + MIN_TIME_AFTER_LAST_MODIFICATION)
{
uint64_t fSize = 0;
RsDirUtil::checkFile(fPath,fSize);
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
singleFilesMap[fPath].modtime = lastWrite;
singleFilesMap[fPath].size = fSize;
}
else
{
some_files_not_ready = true;
RS_INFO( "file: \"", fPath, "\" is "
"probably being written to. Keep it for later");
}
}
else RS_WARN( "Got non existent file \"", fPath,
"\" in shared directories list. Ignored." );
}
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
{
const auto tRoot = mSharedDirectories->root();
std::map<std::string, DirectoryStorage::FileTS> needsUpdate;
mSharedDirectories->updateSubFilesList(
tRoot, singleFilesMap, needsUpdate);
for( DirectoryStorage::FileIterator storedSingleFilesIt(
mSharedDirectories, mSharedDirectories->root() );
storedSingleFilesIt; ++storedSingleFilesIt )
{
const auto& it = storedSingleFilesIt;
RsFileHash hash;
if( mHashCache->requestHash(
it.name(), it.size(), it.modtime(), hash,
this, *it ) )
mSharedDirectories->updateHash(*it, hash, it.hash() != hash);
}
}
/* make sure that entries in stored_dir_it are the same than paths in
* real_dir_it, and in the same order. */
mSharedDirectories->updateSubDirectoryList(
mSharedDirectories->root(), sub_dir_list, mHashSalt );
// now for each of them, go recursively and match both files and dirs
std::set<std::string> existing_dirs;
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
for( DirectoryStorage::DirIterator stored_dir_it(
mSharedDirectories, mSharedDirectories->root() );
stored_dir_it; ++stored_dir_it )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
RS_DBG4("recursing into \"", stored_dir_it.name());
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
recursUpdateSharedDir(
stored_dir_it.name(), *stored_dir_it,
existing_dirs, 1, some_files_not_ready );
/* here we need to use the list that was stored, instead of the shared
* dir list, because the two are not necessarily in the same order. */
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
@ -176,130 +228,135 @@ bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
return true;
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
void LocalDirectoryUpdater::recursUpdateSharedDir(
const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,
std::set<std::string>& existing_directories, uint32_t current_depth,
bool& some_files_not_ready )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
RS_DBG4("parsing directory \"", cumulated_path, "\" index: ", indx);
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
/* make sure list of subdirs is the same
* make sure list of subfiles is the same
* request all hashes to the hashcache */
librs::util::FolderIterator dirIt(cumulated_path,mFollowSymLinks,false); // disallow symbolic links and files from the future.
// disallow symbolic links and files from the future.
librs::util::FolderIterator dirIt(cumulated_path, mFollowSymLinks, false);
rstime_t dir_local_mod_time;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
RS_ERR("Cannot get local mod time for dir index: ", indx);
print_stacktrace();
return;
}
rstime_t now = time(NULL) ;
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match.
// we only want to detect when the directory has changed on the disk
rstime_t now = time(nullptr);
/* the > is because we may have changed the virtual name, and therefore the
* TS wont match. We only want to detect when the directory has changed on
* the disk */
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)
{
// collect subdirs and subfiles
std::map<std::string, DirectoryStorage::FileTS> subfiles;
std::set<std::string> subdirs;
for( ; dirIt.isValid(); dirIt.next() )
if(filterFile(dirIt.file_name()))
{
switch(dirIt.file_type())
const auto fType = dirIt.file_type();
switch(fType)
{
case librs::util::FolderIterator::TYPE_FILE:
if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
if(now >= dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime();
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
RS_DBG4("adding sub-file \"", dirIt.file_name(), "\"");
}
else
{
some_files_not_ready = true;
std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being written to. Keeping it for later." << std::endl;
RS_INFO( "file: \"", dirIt.file_fullpath(), "\" is "
"probably being written to. Keep it for later");
}
break;
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true;
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64)) // 64 is here as a safe limit, to make loops impossible.
/* 64 is here as a safe limit, to make infinite loops
* impossible.
* TODO: Make it a visible constexpr in the header */
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth)
|| (mMaxShareDepth == 0 && current_depth >= 64) )
dir_is_accepted = false;
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
std::string real_path = RsDirUtil::removeSymLinks(
cumulated_path + "/" + dirIt.file_name() );
if(existing_directories.end() != existing_directories.find(real_path))
if( existing_directories.end() !=
existing_directories.find(real_path) )
{
std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
RS_WARN( "Directory: \"", cumulated_path,
"\" has real path: \"", real_path,
"\" which already belongs to another "
"shared directory. Ignoring" );
dir_is_accepted = false;
}
else
existing_directories.insert(real_path) ;
else existing_directories.insert(real_path);
}
if(dir_is_accepted)
subdirs.insert(dirIt.file_name());
if(dir_is_accepted) subdirs.insert(dirIt.file_name());
RS_DBG4("adding sub-dir \"", dirIt.file_name(), "\"");
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
break;
}
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
RS_ERR( "Got Dir entry of unknown type:", fType,
"with path \"", cumulated_path, "/",
dirIt.file_name(), "\"" );
print_stacktrace();
break;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
/* update folder modificatoin time, which is the only way to detect
* e.g. removed or renamed files. */
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime());
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt);
std::map<std::string, DirectoryStorage::FileTS> new_files;
mSharedDirectories->updateSubFilesList(indx, subfiles, new_files);
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
for( DirectoryStorage::FileIterator dit(mSharedDirectories,indx);
dit; ++dit )
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
/* ask about the hash. If not present, ask HashCache.
* If not present, or different, the callback will update it. */
RsFileHash hash;
// mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed.
/* mSharedDirectories does two things: store H(F), and
* compute H(H(F)), which is used in FT.
* The later is always needed. */
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit))
if( mHashCache->requestHash(
cumulated_path + "/" + dit.name(),
dit.size(), dit.modtime(), hash, this, *dit ) )
mSharedDirectories->updateHash(*dit, hash, hash != dit.hash());
}
}
#ifdef DEBUG_LOCAL_DIR_UPDATER
else
std::cerr << " directory is unchanged. Keeping existing files and subdirs list." << std::endl;
#endif
// go through the list of sub-dirs and recursively update
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
}
for( DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories, indx);
stored_dir_it; ++stored_dir_it )
recursUpdateSharedDir( cumulated_path + "/" + stored_dir_it.name(),
*stored_dir_it, existing_directories,
current_depth+1, some_files_not_ready );
}
bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@ -347,7 +404,8 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
{
return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
return mSharedDirectories->getEntryType(
DirectoryStorage::EntryIndex(client_param) ) == DIR_TYPE_FILE;
}
void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)

17
libretroshare/src/file_sharing/hash_cache.cc
View File

@ -187,9 +187,10 @@ void HashStorage::threadTick()
else
rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
//RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
if(rsEvents)
{
/* Emit deprecated event only for retrocompatibility
* TODO: create a proper event with structured data instead of a
* formatted string */
auto ev = std::make_shared<RsSharedDirectoriesEvent>();
ev->mEventCode = RsSharedDirectoriesEventCode::HASHING_FILE;
ev->mMessage = tmpout;
@ -218,8 +219,7 @@ void HashStorage::threadTick()
mChanged = true ;
mTotalHashedSize += size ;
}
else
std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
else RS_ERR("Failure hashing file: ", job.full_path);
mHashingTime += rstime::RsScopeTimer::currentTime() - seconds_origin ;
mHashedBytes += size ;
@ -234,10 +234,17 @@ void HashStorage::threadTick()
++mHashCounter ;
}
}
// call the client
// call the client
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
/* Notify we completed hashing a file */
auto ev = std::make_shared<RsFileHashingCompletedEvent>();
ev->mFilePath = job.full_path;
ev->mHashingSpeed = mCurrentHashingSpeed;
ev->mFileHash = hash;
rsEvents->postEvent(ev);
}
bool HashStorage::requestHash(const std::string& full_path,uint64_t size,rstime_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)

196
libretroshare/src/file_sharing/p3filelists.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,7 +32,7 @@
#include "retroshare/rsids.h"
#include "retroshare/rspeers.h"
#include "retroshare/rsinit.h"
#include "util/cxx17retrocompat.h"
#include "rsserver/p3face.h"
#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : "
@ -1044,31 +1046,36 @@ void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIn
}
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
int p3FileDatabase::RequestDirDetails(
void* ref, DirDetails& d, FileSearchFlags flags ) const
{
RS_STACK_MUTEX(mFLSMtx);
d.children.clear();
// Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
// or at the top of our own list of shared directories, depending on the flags.
/* Case where the pointer is NULL, which means we're at the top of the list
* of shared directories for all friends (including us) or at the top of our
* own list of shared directories, depending on the flags.
*
* Friend index is used as follows:
* 0 : own id
* 1...n : other friends
*
* entry_index: starts at 0.
*
* The point is: we cannot use (0,0) because it encodes to NULL. No existing
* combination should encode to NULL.
* So we need to properly convert the friend index into 0 or into a friend
* tab index in mRemoteDirectories.
*
* We should also check the consistency between flags and the content of ref.
*/
// Friend index is used as follows:
// 0 : own id
// 1...n : other friends
//
// entry_index: starts at 0.
//
// The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
// So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
//
// We should also check the consistency between flags and the content of ref.
if (ref == NULL)
if (ref == nullptr)
{
d.ref = NULL ;
d.ref = nullptr;
d.type = DIR_TYPE_ROOT;
d.parent = NULL;
d.parent = nullptr;
d.prow = -1;
d.name = "root";
d.hash.clear() ;
@ -1079,10 +1086,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(flags & RS_FILE_HINTS_LOCAL)
{
void *p;
void *p = nullptr;
{
convertEntryIndexToPointer<sizeof(void*)>(0,0,p); // root of own directories
// root of own directories
convertEntryIndexToPointer<sizeof(void*)>(0, 0, p);
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = mServCtrl->getOwnId().toStdString();
@ -1092,9 +1100,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(mExtraFiles->size() > 0)
{
convertEntryIndexToPointer<sizeof(void*)>(0,1,p); // local shared files from extra list
// local shared files from extra list
convertEntryIndexToPointer<sizeof(void*)>(0, 1, p);
DirStub stub;
stub.type = DIR_TYPE_PERSON; // not totally exact, but used as a trick.
// not totally exact, but used as a trick.
stub.type = DIR_TYPE_PERSON;
stub.name = "[Extra List]";
stub.ref = p;
@ -1132,9 +1142,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||
(fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
{
P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
RS_ERR("Remote request on local index or local request on remote index");
return false;
}
@ -1150,25 +1161,28 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
}
DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
DirectoryStorage* storage =
(flags & RS_FILE_HINTS_LOCAL) ?
((DirectoryStorage*) mLocalSharedDirs) :
((DirectoryStorage*) mRemoteDirectories[fi-1]);
// Case where the index is the top of a single person. Can be us, or a friend.
if(storage==NULL || !storage->extractData(e,d))
/* Case where the index is the top of a single person.
* Can be us, or a friend. */
if(!storage || !storage->extractData(e,d))
{
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
#endif
RS_WARN( "request on index; ", e, ", for directory ID:",
( (!storage)? ("[NULL]") : (storage->peerId().toStdString()) ),
" failed" );
return false;
}
// update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
// a pointer into an int.
/* update indexes. This is a bit hacky, but does the job. The cast to
* intptr_t is the proper way to convert a pointer into an int. */
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref);
for(uint32_t i=0; i<d.children.size(); ++i)
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
convertEntryIndexToPointer<sizeof(void*)>(
(intptr_t) d.children[i].ref, fi, d.children[i].ref );
if(e == 0) // root
{
@ -1307,7 +1321,9 @@ uint32_t p3FileDatabase::watchPeriod()
return mLocalDirWatcher->fileWatchPeriod();
}
int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
int p3FileDatabase::SearchKeywords(
const std::list<std::string>& keywords, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& client_peer_id )
{
if(flags & RS_FILE_HINTS_LOCAL)
{
@ -1319,10 +1335,10 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
mLocalSharedDirs->searchTerms(keywords,firesults) ;
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
for(auto& it: std::as_const(firesults))
{
void *p=NULL;
convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
void *p = nullptr;
convertEntryIndexToPointer<sizeof(void*)>(it, 0, p);
pointers.push_back(p);
}
}
@ -1476,7 +1492,9 @@ bool p3FileDatabase::search(
return false;
}
int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
int p3FileDatabase::filterResults(
const std::list<void*>& firesults, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& peer_id ) const
{
results.clear();
@ -1484,22 +1502,26 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
/* translate/filter results */
for(std::list<void*>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
for(void* rit: std::as_const(firesults))
{
DirDetails cdetails;
if(!RequestDirDetails (*rit,cdetails,RS_FILE_HINTS_LOCAL))
if(!RequestDirDetails(rit, cdetails, RS_FILE_HINTS_LOCAL))
{
P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << *rit << std::endl;
RS_ERR("Cannot retrieve dir details for entry: ", rit);
print_stacktrace();
continue ;
}
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Filtering candidate " << *rit << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
#endif
RS_DBG( "Filtering candidate: ", rit,
", name: ", cdetails.name, ", path: ", cdetails.path,
", flags: ", cdetails.flags, ", peer: ", peer_id );
if(!peer_id.isNull())
{
FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
FileSearchFlags permission_flags =
rsPeers->computePeerPermissionFlags(
peer_id, cdetails.flags, cdetails.parent_groups );
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
@ -1518,6 +1540,18 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
results.push_back(cdetails);
}
for(auto& res: results)
{
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
if(res.name.find("/") != std::string::npos )
{
std::string _tDirPath;
RsDirUtil::splitDirFromFile(res.name, _tDirPath, res.name);
}
}
return !results.empty();
}
@ -1598,17 +1632,16 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
{
RS_STACK_MUTEX(mFLSMtx);
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
#endif
RS_DBG( "Received directory sync request from peer ", item->PeerId(),
". hash=", item->entry_hash, ", flags=", item->flags,
", request id: ", item->request_id, ", last known TS: ",
item->last_known_recurs_modf_TS );
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#endif
RS_DBG("Cannot find entry index for hash ", item->entry_hash,
" cannot respond to sync request." );
return;
}
@ -1622,47 +1655,60 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
if(entry_type != DIR_TYPE_DIR)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
RS_DBG( "Directory does not exist anymore, or is not a directory, "
"or permission denied. Answering with proper flags." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
else if( entry_index != 0 &&
(!mLocalSharedDirs->getFileSharingPermissions(
entry_index, node_flags,node_groups ) ||
!(rsPeers->computePeerPermissionFlags(
item->PeerId(), node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE)) )
{
P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
RS_ERR("cannot get file permissions for entry index: ", entry_index,
", or permission denied." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else
{
rstime_t local_recurs_max_time;
mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
mLocalSharedDirs->getDirectoryRecursModTime(
entry_index, local_recurs_max_time );
if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
/* normally, should be "<", but since we provided the TS it should
* be equal, so != is more robust. */
if(item->last_known_recurs_modf_TS != local_recurs_max_time)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
#endif
RS_DBG( "Directory is more recent than what the friend knows. "
"Sending full dir content as response.");
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
// We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
/* We supply the peer id, in order to possibly remove some
* subdirs, if entries are not allowed to be seen by this peer.
*/
mLocalSharedDirs->serialiseDirEntry(
entry_index, ritem->directory_content_data,
item->PeerId() );
}
else
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
#endif
RS_DBG3( "Directory is up to date w.r.t. what the friend knows."
" Sending ACK." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
}
}
}
// sends the response.
splitAndSendItem(ritem);
}

8
libretroshare/src/ft/ftcontroller.cc
View File

@ -735,8 +735,7 @@ bool ftController::completeFile(const RsFileHash& hash)
RsDirUtil::splitDirFromFile(fc->mDestination,dst_dir,dst_file) ;
// We use this intermediate file in case the destination directory is not available, so as to not keep the partial file name.
std::string intermediate_file_name = src_dir+'/'+dst_file ;
std::string intermediate_file_name = RsDirUtil::makePath(src_dir, dst_file);
// I don't know how the size can be zero, but believe me, this happens,
// and it causes an error on linux because then the file may not even exist.
@ -1542,6 +1541,11 @@ std::string ftController::getPartialsDirectory()
return mPartialsPath;
}
bool ftController::FileServerCancel(const RsFileHash& hash)
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
return mDataplex->deleteServer(hash);
}
bool ftController::setDestinationDirectory(const RsFileHash& hash,const std::string& dest_dir)
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/

1
libretroshare/src/ft/ftcontroller.h
View File

@ -157,6 +157,7 @@ public:
bool getFileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info);
bool setDestinationName(const RsFileHash& hash,const std::string& dest_name) ;
bool setDestinationDirectory(const RsFileHash& hash,const std::string& dest_name) ;
bool FileServerCancel(const RsFileHash& hash);
// Download speed
bool getPriority(const RsFileHash& hash,DwlSpeed& p);

17
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -1039,6 +1039,23 @@ void ftDataMultiplex::handlePendingCrcRequests()
}
}
bool ftDataMultiplex::deleteServer(const RsFileHash& hash)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
auto sit = mServers.find(hash);
if(sit == mServers.end())
return false;
// We don't delete servers that are clients at the same time !
if(dynamic_cast<ftFileCreator*>(sit->second) == NULL)
delete sit->second;
mServers.erase(sit);
return true;
}
void ftDataMultiplex::deleteUnusedServers()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

2
libretroshare/src/ft/ftdatamultiplex.h
View File

@ -113,9 +113,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool FileDetails(const RsFileHash &hash, FileSearchFlags hintsflag, FileInfo &info);
void deleteUnusedServers() ;
bool deleteServer(const RsFileHash& hash); // deletes FtServers for the given hash. Used when removing an extra file from shares.
void handlePendingCrcRequests() ;
/*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */

15
libretroshare/src/ft/ftserver.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -817,7 +819,10 @@ bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t
}
bool ftServer::ExtraFileRemove(const RsFileHash& hash)
{ return mFileDatabase->removeExtraFile(hash); }
{
mFtController->FileServerCancel(hash);
return mFileDatabase->removeExtraFile(hash);
}
bool ftServer::ExtraFileHash( std::string localpath, rstime_t period, TransferRequestFlags flags )
{
@ -873,7 +878,9 @@ int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetai
{
return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
}
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
int ftServer::SearchKeywords(
std::list<std::string> keywords, std::list<DirDetails> &results,
FileSearchFlags flags, const RsPeerId& peer_id )
{
return mFileDatabase->SearchKeywords(keywords, results, flags, peer_id);
}
@ -2030,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{
RsFileSearchResultItem resIt;

63
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60)
{
mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
}
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@ -362,12 +322,12 @@ void RsGenExchange::tick()
{
mIntegrityCheck = new RsGxsIntegrityCheck( mDataStore, this,
*mSerialiser, mGixs);
mIntegrityCheck->start("gxs integrity");
mChecking = true;
std::string thisName = typeid(*this).name();
mChecking = mIntegrityCheck->start("gxs IC4 "+thisName);
}
}
if(mIntegrityCheck->isDone())
if(mIntegrityCheck->isDone() || !mChecking)
{
std::vector<RsGxsGroupId> grpIds;
GxsMsgReq msgIds;
@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
msgDeleted.push_back(note.msgIds);
}
for(const auto& msgreq:msgDeleted)
for(const auto& msgit:msgreq)
for(const auto& msg:msgit.second)
mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
/* Three nested for looks like a performance bomb, but as Cyril says here
* https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
* this should actually not explode at all because it is just one message at
* time that get notified */
for(const auto& msd : mMsgDeletePublish)
for(auto& msgMap : msd.mMsgs)
for(auto& msgId : msgMap.second)
mNotifications.push_back(
new RsGxsMsgDeletedChange(msgMap.first, msgId) );
mMsgDeletePublish.clear();
}

124
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{
@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{
std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_CHANNEL_INDEX
#ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important
* to test that the group meta is both non null and actually corresponds to the group id we seek. */
* mDataStore may in some situations allocate an empty group meta
* data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId);
@ -5332,16 +5330,17 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId);
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
RS_WARN("Received search results for unknown request: ", req);
return;
}
RsGxsGroupId grpId = it->second;
@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId);
}
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
if(!mGxsNetTunnel)
{
free(searchData);
return std::errc::function_not_supported;
}
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos )
{
group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,7 +5477,6 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;

62
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGXSNETSERVICE_H
#define RSGXSNETSERVICE_H
#pragma once
#include <list>
#include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
/// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -629,5 +665,3 @@ private:
bool mUseMetaCache;
};
#endif // RSGXSNETSERVICE_H

545
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
mLastDump = time(NULL) + (RSRandom::random_u32()%20);
/* adds some variance in order to avoid doing all this tasks at once across
* services */
auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
}
//===========================================================================================================================================//
//============================================================================//
// Transport Items //
//===========================================================================================================================================//
//============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem
struct RsGxsNetTunnelItem: RsItem
{
public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{
// no priority. All items are encapsulated into generic Turtle items anyway.
}
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
};
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
};
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{
public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
}
};
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{
public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer
{
public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
return NULL ;
RS_ERR( "received item with wrong service ID ", service);
print_stacktrace();
return nullptr;
}
switch(item_subtype)
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
return new RsGxsNetTunnelVirtualPeerItem;
case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
return new RsGxsNetTunnelKeepAliveItem;
case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
return new RsGxsNetTunnelRandomBiasItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
return NULL ;
RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return nullptr;
}
}
};
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ;
}
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ;
uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL)
return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
uint32_t size = tSerializer.size(&searchItem);
uint8_t* buf = rs_malloc<uint8_t>(size);
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ;
RsNetworkExchangeService *service = nullptr;
requestId = mTurtle->turtleSearch(buf, size, this);
if(!requestId) return std::errc::result_out_of_range;
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
return std::error_condition();
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RS_DBG3("");
RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{
RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
print_stacktrace();
return false;
}
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{
if(!search_result_data)
{
RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
"results!" );
print_stacktrace();
break;
}
auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
tSerializer.serialise(
&search_result_item, search_result_data,
&search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
unsigned char *encrypted_group_data = NULL ;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
delete item;
return false;
}
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
RS_DBG3(request_id);
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl;
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
if(result_gs != NULL)
auto castFailedWarn = [](const uint8_t subtype)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
#ifdef DEBUG_RSGXSNETTUNNEL
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
return it->second;
};
delete item;
return ;
}
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
if(result_gd != NULL)
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
result_gd->encrypted_group_data = NULL ; // prevents deletion
delete item;
return ;
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{
groups = mGroups ;
virtual_peers = mVirtualPeers ;

52
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once
#include <map>
#include <system_error>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService:
@ -109,7 +112,7 @@ class RsGxsNetTunnelService:
{
public:
RsGxsNetTunnelService();
virtual ~RsGxsNetTunnelService() ;
~RsGxsNetTunnelService() override;
/*!
* \brief registerSearchableService
@ -182,23 +185,37 @@ public:
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
void connectToTurtleRouter(p3turtle *tr) override;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/** Gxs services (channels, forums...) are supposed to use this to request
* searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
///@see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* search_request_data,
uint32_t search_request_data_len,
unsigned char*& search_result_data,
uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread
///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected:
// interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

10
libretroshare/src/gxs/rsgxsnotify.h
View File

@ -97,3 +97,13 @@ private:
bool mMetaChange;
};
struct RsGxsMsgDeletedChange : RsGxsNotify
{
RsGxsMsgDeletedChange(
const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
RsGxsNotify(gid), messageId(msgId) {}
NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
const RsGxsMessageId messageId;
};

129
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
@ -198,8 +192,7 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
, RsGixs* gixs )
RsSerialType&, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true;
}
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
bool RsGxsSinglePassIntegrityCheck::check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
{
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,54 +380,13 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
}
else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete grp;
@ -471,13 +417,7 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
}
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete msg;

30
libretroshare/src/gxs/rsgxsutil.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -162,15 +163,6 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
RsGixs* gixs );
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
static bool check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
};
class GroupUpdate

46
libretroshare/src/gxs/rsnxs.h
View File

@ -3,8 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker *
* Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGNP_H
#define RSGNP_H
#pragma once
#include <set>
#include <string>
#include "util/rstime.h"
#include <stdlib.h>
#include <list>
#include <map>
#include "util/rstime.h"
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService
{
public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() = default;
virtual uint16_t serviceType() const =0;
/*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*!
* \brief turtleGroupRequest
@ -121,7 +135,11 @@ public:
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*!
* \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
@ -306,5 +326,3 @@ public:
}
}
};
#endif // RSGNP_H

24
libretroshare/src/util/rserrno.h → libretroshare/src/gxs/rsnxsobserver.cpp
View File

@ -1,9 +1,8 @@
/*******************************************************************************
* libretroshare/src/util: rserrno.h *
* RetroShare General eXchange System *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,6 +18,19 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
const char* rsErrnoName(int err);
#include "gxs/rsnxsobserver.h"
const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
{
switch(static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return std::errc::operation_not_supported;
default:
return std::error_condition(ev, *this);
}
}

106
libretroshare/src/gxs/rsnxsobserver.h
View File

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher *
* Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSOBSERVER_H
#define RSNXSOBSERVER_H
#pragma once
#include <set>
#include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver
{
public:
RsNxsObserver() {}
public:
/*!
@ -56,6 +99,46 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data storage for a pointer to search
* result reply data or nullptr if no mathing results where found
* @param[out] resultSize storage for results data size or 0 if no matching
* results where found
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
/* Avoid unused paramethers warning this way so doxygen can still parse
* paramethers documentation */
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*!
* @param grpId group id
*/
@ -70,6 +153,7 @@ public:
* @param grpId group id
*/
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H
RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

62
libretroshare/src/libretroshare.pro
View File

@ -398,7 +398,6 @@ HEADERS += pqi/authssl.h \
pqi/pqissl.h \
pqi/pqissllistener.h \
pqi/pqisslpersongrp.h \
pqi/pqissli2pbob.h \
pqi/pqisslproxy.h \
pqi/pqistore.h \
pqi/pqistreamer.h \
@ -459,7 +458,7 @@ HEADERS += rsitems/rsitem.h \
rsitems/rsgxsupdateitems.h \
rsitems/rsserviceinfoitems.h \
HEADERS += services/autoproxy/p3i2pbob.h \
HEADERS += \
services/rseventsservice.h \
services/autoproxy/rsautoproxymonitor.h \
services/p3msgservice.h \
@ -513,8 +512,7 @@ HEADERS += util/folderiterator.h \
util/cxx11retrocompat.h \
util/cxx14retrocompat.h \
util/cxx17retrocompat.h \
util/rsurl.h \
util/rserrno.h
util/rsurl.h
SOURCES += ft/ftchunkmap.cc \
ft/ftcontroller.cc \
@ -563,7 +561,6 @@ SOURCES += pqi/authgpg.cc \
pqi/pqissl.cc \
pqi/pqissllistener.cc \
pqi/pqisslpersongrp.cc \
pqi/pqissli2pbob.cpp \
pqi/pqisslproxy.cc \
pqi/pqistore.cc \
pqi/pqistreamer.cc \
@ -621,7 +618,6 @@ SOURCES += serialiser/rsbaseserial.cc \
SOURCES += services/autoproxy/rsautoproxymonitor.cc \
services/rseventsservice.cc \
services/autoproxy/p3i2pbob.cc \
services/p3msgservice.cc \
services/p3service.cc \
services/p3statusservice.cc \
@ -641,10 +637,10 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \
util/rsnet.cc \
util/rsnet_ss.cc \
util/rsdnsutils.cc \
util/extaddrfinder.cc \
util/dnsresolver.cc \
util/rsprint.cc \
@ -655,8 +651,7 @@ SOURCES += util/folderiterator.cc \
util/rsrecogn.cc \
util/rstime.cc \
util/rsurl.cc \
util/rsbase64.cc \
util/rserrno.cc
util/rsbase64.cc
equals(RS_UPNP_LIB, miniupnpc) {
HEADERS += rs_upnp/upnputil.h rs_upnp/upnphandler_miniupnp.h
@ -720,7 +715,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc
gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# Tor
HEADERS += retroshare/rstor.h
@ -986,6 +982,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
@ -1059,6 +1063,44 @@ rs_broadcast_discovery {
}
}
rs_sam3 {
SOURCES += \
services/autoproxy/p3i2psam3.cpp \
pqi/pqissli2psam3.cpp \
HEADERS += \
services/autoproxy/p3i2psam3.h \
pqi/pqissli2psam3.h \
}
rs_sam3_libsam3 {
DUMMYQMAKECOMPILERINPUT = FORCE
libsam3.name = Generating libsam3.
libsam3.input = DUMMYQMAKECOMPILERINPUT
libsam3.output = $$clean_path($${LIBSAM3_BUILD_PATH}/libsam3.a)
libsam3.CONFIG += target_predeps combine
libsam3.variable_out = PRE_TARGETDEPS
win32-g++:isEmpty(QMAKE_SH) {
LIBSAM3_MAKE_PARAMS = CC=gcc
libsam3.commands = \
cd /D $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/libsam3 || cd . $$escape_expand(\\n\\t) \
$(CHK_DIR_EXISTS) $$shell_path($$LIBSAM3_BUILD_PATH) $(MKDIR) $$shell_path($${LIBSAM3_BUILD_PATH}) $$escape_expand(\\n\\t) \
$(COPY_DIR) $$shell_path($${LIBSAM3_SRC_PATH}) $$shell_path($${LIBSAM3_BUILD_PATH}) || cd . $$escape_expand(\\n\\t)
} else {
LIBSAM3_MAKE_PARAMS =
libsam3.commands = \
cd $${RS_SRC_PATH} && ( \
git submodule update --init supportlibs/libsam3 || \
true ) && \
mkdir -p $${LIBSAM3_BUILD_PATH} && \
(cp -r $${LIBSAM3_SRC_PATH}/* $${LIBSAM3_BUILD_PATH} || true) &&
}
libsam3.commands += \
cd $$shell_path($${LIBSAM3_BUILD_PATH}) && \
$(MAKE) build $${LIBSAM3_MAKE_PARAMS}
QMAKE_EXTRA_COMPILERS += libsam3
}
###########################################################################################################
# OLD CONFIG OPTIONS.
# Not used much - but might be useful one day.

228
libretroshare/src/pqi/p3netmgr.cc
View File

@ -47,14 +47,15 @@ struct RsLog::logInfo p3netmgrzoneInfo = {RsLog::Default, "p3netmgr"};
/* Network setup States */
const uint32_t RS_NET_NEEDS_RESET = 0x0000;
const uint32_t RS_NET_UNKNOWN = 0x0001;
const uint32_t RS_NET_UPNP_INIT = 0x0002;
const uint32_t RS_NET_UPNP_SETUP = 0x0003;
const uint32_t RS_NET_EXT_SETUP = 0x0004;
const uint32_t RS_NET_DONE = 0x0005;
const uint32_t RS_NET_LOOPBACK = 0x0006;
//const uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_NEEDS_RESET = 0x0000;
constexpr uint32_t RS_NET_UNKNOWN = 0x0001;
constexpr uint32_t RS_NET_UPNP_INIT = 0x0002;
constexpr uint32_t RS_NET_UPNP_SETUP = 0x0003;
constexpr uint32_t RS_NET_EXT_SETUP = 0x0004;
constexpr uint32_t RS_NET_DONE = 0x0005;
constexpr uint32_t RS_NET_LOOPBACK = 0x0006;
//constexpr uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_SHUTDOWN = 0x00FF; //Highest value to not restart UPnP nor ExtAddrFinder
/* Stun modes (TODO) */
//const uint32_t RS_STUN_DHT = 0x0001;
@ -68,6 +69,8 @@ const uint32_t MAX_UPNP_COMPLETE = 600; /* 10 min... seems to take a while */
//const uint32_t MIN_TIME_BETWEEN_NET_RESET = 5;
const uint32_t MIN_TIME_EXT_FINDER_UPDATE = 300; /* 5min to check if external IP is changed */
/****
* #define NETMGR_DEBUG 1
* #define NETMGR_DEBUG_RESET 1
@ -107,19 +110,17 @@ void pqiNetStatus::print(std::ostream &out)
}
p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
mDoNotNetCheckUntilTs(0)
p3NetMgrIMPL::p3NetMgrIMPL()
: mPeerMgr(nullptr), mLinkMgr(nullptr)
, mNetMtx("p3NetMgr"), mNetMode(RS_NET_MODE_UDP), mVsDisc(RS_VS_DISC_FULL), mVsDht(RS_VS_DHT_FULL)// default to full.
, mNetInitTS(0), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
, mUseExtAddrFinder(true), mNetExtAddrFinderTs(0), mDoNotNetCheckUntilTs(0)
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetMode = RS_NET_MODE_UDP;
mUseExtAddrFinder = true;
mExtAddrFinder = new ExtAddrFinder();
mNetInitTS = 0;
mNetFlags = pqiNetStatus();
mOldNetFlags = pqiNetStatus();
@ -133,10 +134,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mLocalAddr.ss_family = AF_INET;
mExtAddr.ss_family = AF_INET;
// default to full.
mVsDisc = RS_VS_DISC_FULL;
mVsDht = RS_VS_DHT_FULL;
}
#ifdef NETMGR_DEBUG
@ -235,6 +232,10 @@ void p3NetMgrIMPL::netReset()
rslog(RSL_ALERT, p3netmgrzone, "p3NetMgr::netReset() Called");
shutdown(); /* blocking shutdown call */
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
}
// Will initiate a new call for determining the external ip.
if (mUseExtAddrFinder)
@ -242,7 +243,7 @@ void p3NetMgrIMPL::netReset()
#ifdef NETMGR_DEBUG_RESET
std::cerr << "p3NetMgrIMPL::netReset() restarting AddrFinder" << std::endl;
#endif
mExtAddrFinder->reset() ;
mExtAddrFinder->reset(true) ;
}
else
{
@ -265,7 +266,7 @@ void p3NetMgrIMPL::netReset()
* as it calls back to p3ConnMgr.
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
struct sockaddr_storage iaddr = mLocalAddr;
@ -283,8 +284,7 @@ void p3NetMgrIMPL::netReset()
}
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
netStatusReset_locked();
}
@ -311,8 +311,8 @@ bool p3NetMgrIMPL::shutdown() /* blocking shutdown call */
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_SHUTDOWN;
mNetInitTS = time(NULL);
netStatusReset_locked();
}
@ -347,7 +347,7 @@ void p3NetMgrIMPL::netStartup()
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetInitTS = time(NULL);
netStatusReset_locked();
@ -460,35 +460,51 @@ void p3NetMgrIMPL::netStatusTick()
uint32_t netStatus = 0;
rstime_t age = 0;
bool needExtFinderUpdate = false;
{
RsStackMutex stack(mNetMtx); /************** LOCK MUTEX ***************/
RS_STACK_MUTEX(mNetMtx); /************** LOCK MUTEX ***************/
netStatus = mNetStatus;
age = time(NULL) - mNetInitTS;
needExtFinderUpdate = netStatus == RS_NET_DONE;
needExtFinderUpdate &= mNetExtAddrFinderTs < time(nullptr);
if(needExtFinderUpdate)
mNetExtAddrFinderTs = time(nullptr) + MIN_TIME_EXT_FINDER_UPDATE;
}
if(netStatus <= RS_NET_UPNP_SETUP && mUseExtAddrFinder)
if( mUseExtAddrFinder
&& ( netStatus <= RS_NET_UPNP_SETUP
|| needExtFinderUpdate) )
{
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "Asking ExtAddrFinder for IP. Initializing port with " << sockaddr_storage_port(tmpip) << std::endl;
RS_DBG("Asking ExtAddrFinder for IP. Initializing port with ", sockaddr_storage_port(tmpip));
#endif
if(mExtAddrFinder->hasValidIP(tmpip) && sockaddr_storage_ipv6_to_ipv4(tmpip) && !sockaddr_storage_same(tmpip,mExtAddr))
if(mExtAddrFinder->hasValidIPV4(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
#endif
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
else if(mExtAddrFinder->hasValidIPV6(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
//Only if no IPv4 else, reset connections on setExtAddress()
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
}
switch(netStatus)
{
@ -578,7 +594,7 @@ void p3NetMgrIMPL::netDhtInit()
uint32_t vs = 0;
{
RsStackMutex stack(mNetMtx); /*********** LOCKED MUTEX ************/
RS_STACK_MUTEX(mNetMtx); /*********** LOCKED MUTEX ************/
vs = mVsDht;
}
@ -745,25 +761,16 @@ void p3NetMgrIMPL::netExtCheck()
if (mUseExtAddrFinder)
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() checking ExtAddrFinder" << std::endl;
RS_DBG("checking ExtAddrFinder");
#endif
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
if (extFinderOk && sockaddr_storage_ipv6_to_ipv4(tmpip))
// Test for IPv4 first to be compatible with older versions.
if (mExtAddrFinder->hasValidIPV4(tmpip))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
#endif
sockaddr_storage_setport(tmpip, guessNewExtPort());
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
#endif
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
@ -773,10 +780,22 @@ void p3NetMgrIMPL::netExtCheck()
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
std::cerr << __PRETTY_FUNCTION__ << " ExtAddrFinder "
<< " reported external address "
<< sockaddr_storage_iptostring(tmpip)
<< std::endl;
RS_DBG("Reported external IPv4 address ", sockaddr_storage_iptostring(tmpip));
}
else if (mExtAddrFinder->hasValidIPV6(tmpip))
{
sockaddr_storage_setport(tmpip, guessNewExtPort());
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
/* XXX HACK TO FIX drbob: ALLOWING
* ExtAddrFinder -> ExtAddrStableOk = true
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
RS_DBG("Reported external IPv6 address ", sockaddr_storage_iptostring(tmpip));
}
}
}
@ -967,7 +986,7 @@ void p3NetMgrIMPL::netExtCheck()
if (netSetupDone)
{
std::cerr << "p3NetMgrIMPL::netExtCheck() netSetupDone" << std::endl;
RS_DBG("netSetupDone");
/* Setup NetStateBox with this info */
updateNetStateBox_startup();
@ -978,12 +997,14 @@ void p3NetMgrIMPL::netExtCheck()
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
/* inform DHT about our external address */
/* inform DHT about our external IPV4 address, it doesn't support IPv6 for now.*/
if(sockaddr_storage_ipv6_to_ipv4(mExtAddr))
{
RsPeerId fakeId;
netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
}
std::cerr << __PRETTY_FUNCTION__ << " Network Setup Complete"
<< std::endl;
RS_INFO("Network Setup Complete");
}
}
@ -995,6 +1016,7 @@ bool p3NetMgrIMPL::checkNetAddress()
{
bool addrChanged = false;
bool validAddr = false;
bool needOwnAddrUpdate = false;
sockaddr_storage prefAddr;
sockaddr_storage oldAddr;
@ -1067,8 +1089,7 @@ bool p3NetMgrIMPL::checkNetAddress()
if (!validAddr)
{
RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
<<" found. Report to developers." << std::endl;
RS_ERR("no valid local network address found. Report to developers.");
print_stacktrace();
return false;
@ -1109,6 +1130,7 @@ bool p3NetMgrIMPL::checkNetAddress()
sockaddr_storage_setport(mExtAddr, port); // this accounts for when the port was updated
addrChanged = true;
}
} // RS_STACK_MUTEX(mNetMtx);
if (addrChanged)
@ -1123,11 +1145,24 @@ bool p3NetMgrIMPL::checkNetAddress()
rsEvents->postEvent(ev);
}
if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
needOwnAddrUpdate = true;
netReset();
}
if (mPeerMgr)
{
// Retrieve last known IP, if none, update own addresse to get current.
peerState ps;
mPeerMgr->getOwnNetStatus(ps);
needOwnAddrUpdate |= ps.ipAddrs.mLocal.mAddrs.empty();
needOwnAddrUpdate |= ps.ipAddrs.mExt.mAddrs.empty();
if (needOwnAddrUpdate)
{
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
}
return true;
}
@ -1139,7 +1174,7 @@ bool p3NetMgrIMPL::checkNetAddress()
/* to allow resets of network stuff */
void p3NetMgrIMPL::addNetListener(pqiNetListener *listener)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetListeners.push_back(listener);
}
@ -1149,7 +1184,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mLocalAddr, addr))
{
changed = true;
@ -1171,7 +1206,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
}
bool p3NetMgrIMPL::getExtAddress(struct sockaddr_storage& addr)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if(mNetFlags.mExtAddrOk)
{
@ -1186,7 +1221,7 @@ bool p3NetMgrIMPL::setExtAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mExtAddr, addr))
{
changed = true;
@ -1220,7 +1255,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
{
uint32_t oldNetMode;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* only change TRY flags */
oldNetMode = mNetMode;
@ -1266,7 +1301,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
bool p3NetMgrIMPL::setVisState(uint16_t vs_disc, uint16_t vs_dht)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mVsDisc = vs_disc;
mVsDht = vs_dht;
@ -1628,7 +1663,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
uint32_t netsize = 0, rsnetsize = 0;
netAssistConnectStats(netsize, rsnetsize);
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* quick update of the stuff that can change! */
mNetFlags.mDhtOk = dhtOk;
@ -1652,7 +1687,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
bool p3NetMgrIMPL::getIPServersEnabled()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mUseExtAddrFinder;
}
@ -1661,15 +1696,32 @@ void p3NetMgrIMPL::getIPServersList(std::list<std::string>& ip_servers)
mExtAddrFinder->getIPServersList(ip_servers);
}
void p3NetMgrIMPL::getCurrentExtIPList(std::list<std::string>& ip_list)
{
ip_list.clear();
sockaddr_storage addr;
if(mExtAddrFinder->hasValidIPV4(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
if(mExtAddrFinder->hasValidIPV6(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
}
void p3NetMgrIMPL::setIPServersEnabled(bool b)
{
if (mUseExtAddrFinder != b)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mExtAddrFinder->reset(true);
if (b)
mExtAddrFinder->start_request();
}
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mUseExtAddrFinder = b;
}
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgr: setIPServers to " << b << std::endl ;
RS_DBG("set mUseExtAddrFinder to ", b);
#endif
}
@ -1682,31 +1734,31 @@ void p3NetMgrIMPL::setIPServersEnabled(bool b)
RsNetState p3NetMgrIMPL::getNetStateMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetStateMode();
}
RsNetworkMode p3NetMgrIMPL::getNetworkMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetworkMode();
}
RsNatTypeMode p3NetMgrIMPL::getNatTypeMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatTypeMode();
}
RsNatHoleMode p3NetMgrIMPL::getNatHoleMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatHoleMode();
}
RsConnectModes p3NetMgrIMPL::getConnectModes()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getConnectModes();
}
@ -1730,7 +1782,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunDht(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1749,7 +1801,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mProxyStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunProxy(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1768,7 +1820,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
bool dhtOn = netAssistConnectEnabled();
bool dhtActive = netAssistConnectActive();
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setDhtState(dhtOn, dhtActive);
}
@ -1777,7 +1829,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
#ifdef NETMGR_DEBUG_STATEBOX
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
auto netstate = mNetStateBox.getNetStateMode();
auto netMode = mNetStateBox.getNetworkMode();
@ -1823,7 +1875,7 @@ void p3NetMgrIMPL::updateNatSetting()
RsNatTypeMode natType = RsNatTypeMode::UNKNOWN;
RsNatHoleMode natHole = RsNatHoleMode::UNKNOWN;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
natType = mNetStateBox.getNatTypeMode();
natHole = mNetStateBox.getNatHoleMode();
@ -1925,7 +1977,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* fill in the data */
struct sockaddr_storage tmpip;
@ -1974,7 +2026,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
if (mUseExtAddrFinder)
{
tmpip = mLocalAddr;
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
bool extFinderOk = mExtAddrFinder->hasValidIPV4(tmpip);
if (extFinderOk)
{
@ -2020,7 +2072,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
void p3NetMgrIMPL::updateNetStateBox_reset()
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.reset();

3
libretroshare/src/pqi/p3netmgr.h
View File

@ -120,6 +120,7 @@ virtual bool shutdown() = 0; /* blocking shutdown call */
virtual bool getIPServersEnabled() = 0;
virtual void setIPServersEnabled(bool b) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status) = 0;
@ -171,6 +172,7 @@ virtual bool shutdown(); /* blocking shutdown call */
virtual bool getIPServersEnabled();
virtual void setIPServersEnabled(bool b);
virtual void getIPServersList(std::list<std::string>& ip_servers);
virtual void getCurrentExtIPList(std::list<std::string>& ip_list);
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status);
@ -313,6 +315,7 @@ void netStatusReset_locked();
bool mStatusChanged;
bool mUseExtAddrFinder;
rstime_t mNetExtAddrFinderTs;
/* network status flags (read by rsiface) */
pqiNetStatus mNetFlags;

7
libretroshare/src/pqi/pqissl.cc
View File

@ -1742,6 +1742,13 @@ bool pqissl::moretoread(uint32_t usec)
{
rslog(RSL_ALERT, pqisslzone,
"pqissl::moretoread() Select ERROR!");
RS_WARN(strerror(errno));
if (errno == EBADF) {
// happens when SAM is shut down
rslog(RSL_ALERT, pqisslzone, "pqissl::moretoread() -> calling reset()");
reset_locked();
}
return 0;
}

52
libretroshare/src/pqi/pqissli2pbob.cpp
View File

@ -1,52 +0,0 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "pqissli2pbob.h"
bool pqissli2pbob::connect_parameter(uint32_t type, const std::string &value)
{
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_STACK_MUTEX(mSslMtx);
// a new line must be appended!
mI2pAddr = value + '\n';
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2pbob::Basic_Connection_Complete()
{
int ret;
if ((ret = pqissl::Basic_Connection_Complete()) != 1)
{
// basic connection not complete.
return ret;
}
// send addr. (new line is already appended)
ret = send(sockfd, mI2pAddr.c_str(), mI2pAddr.length(), 0);
if (ret != (int)mI2pAddr.length())
return -1;
return 1;
}

52
libretroshare/src/pqi/pqissli2pbob.h
View File

@ -1,52 +0,0 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef PQISSLI2PBOB_H
#define PQISSLI2PBOB_H
#include "pqi/pqissl.h"
/*
* This class is a minimal varied version of pqissl to work with I2P BOB tunnels.
* The only difference is that the [.b32].i2p addresses must be sent first.
*
* Everything else is untouched.
*/
class pqissli2pbob : public pqissl
{
public:
pqissli2pbob(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm) {}
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Basic_Connection_Complete();
private:
std::string mI2pAddr;
};
#endif // PQISSLI2PBOB_H

261
libretroshare/src/pqi/pqissli2psam3.cpp Normal file
View File

@ -0,0 +1,261 @@
#include "pqissli2psam3.h"
#include <libsam3.h>
RS_SET_CONTEXT_DEBUG_LEVEL(2)
static constexpr int pqiDone = 1;
static constexpr int pqiWait = 0;
static constexpr int pqiError = -1;
pqissli2psam3::pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm), mState(pqisslSam3State::NONE), mI2pAddrB32(), mI2pAddrLong()
{
RS_DBG4();
mConn = nullptr;
}
bool pqissli2psam3::connect_parameter(uint32_t type, const std::string &value)
{
RS_DBG4();
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_DBG1("got addr:", value);
RS_STACK_MUTEX(mSslMtx);
mI2pAddrB32 = value;
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2psam3::Initiate_Connection()
{
RS_DBG4();
if(waiting != WAITING_DELAY)
{
RS_ERR("Already Attempt in Progress!");
return pqiError;
}
switch (mState) {
case(pqisslSam3State::NONE):
RS_DBG2("NONE");
{
if(mConn) {
// how did we end up here?
RS_ERR("state is NONE but a connection is existing?!");
}
mConn = 0;
// get SAM session
mConn = 0;
samSettings ss;
ss.session = nullptr;
rsAutoProxyMonitor::taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, static_cast<void*>(&ss));
if (!!ss.session) {
RS_DBG3("NONE->DO_LOOKUP");
mState = pqisslSam3State::DO_LOOKUP;
} else {
RS_DBG3("NONE->DO_LOOKUP NOPE", ss.session);
}
}
break;
case(pqisslSam3State::DO_LOOKUP):
RS_DBG1("DO_LOOKUP");
if (!mI2pAddrLong.empty()) {
// skip lookup, it is highly unlikely/impossible for a public key to change (isn't it?)
mState = pqisslSam3State::WAIT_LOOKUP;
break;
}
{
i2p::address *addr = new i2p::address;
addr->clear();
addr->base32 = mI2pAddrB32;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::lookupKey, this, static_cast<void*>(addr));
}
mState = pqisslSam3State::WAIT_LOOKUP;
break;
case(pqisslSam3State::DO_CONNECT):
RS_DBG2("DO_CONNECT");
{
auto wrapper = new samEstablishConnectionWrapper();
wrapper->address.clear();
wrapper->address.publicKey = mI2pAddrLong;
wrapper->connection = nullptr;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::establishConnection, this, static_cast<void*>(wrapper));
}
mState = pqisslSam3State::WAIT_CONNECT;
break;
case(pqisslSam3State::DONE):
RS_DBG2("DONE");
if (setupSocket())
return pqiDone;
return pqiError;
/* waiting */
case(pqisslSam3State::WAIT_LOOKUP):
RS_DBG3("WAIT_LOOKUP");
break;
case(pqisslSam3State::WAIT_CONNECT):
RS_DBG3("WAIT_CONNECT");
break;
}
return pqiWait;
}
int pqissli2psam3::net_internal_close(int fd)
{
RS_DBG4();
// sanity check
if (mConn && fd != mConn->fd) {
// this should never happen!
RS_ERR("fd != mConn");
// sam3CloseConnection(mConn);
}
// now to the actuall closing
int ret = pqissl::net_internal_close(fd);
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::closeConnection, this, mConn);
// finally cleanup
mConn = 0;
mState = pqisslSam3State::NONE;
return ret;
}
void pqissli2psam3::taskFinished(taskTicket *&ticket)
{
RS_DBG4();
switch (ticket->task) {
case autoProxyTask::lookupKey:
{
auto addr = static_cast<i2p::address*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mI2pAddrLong = addr->publicKey;
mState = pqisslSam3State::DO_CONNECT;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete addr;
ticket->data = nullptr;
addr = nullptr;
}
break;
case autoProxyTask::establishConnection:
{
auto wrapper = static_cast<struct samEstablishConnectionWrapper*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mConn = wrapper->connection;
mState = pqisslSam3State::DONE;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete wrapper;
ticket->data = nullptr;
wrapper = nullptr;
}
break;
case autoProxyTask::closeConnection:
// nothing to do here
break;
default:
RS_WARN("unkown task", ticket->task);
}
// clean up!
delete ticket;
ticket = nullptr;
}
bool pqissli2psam3::setupSocket()
{
/*
* This function contains the generis part from pqissl::Initiate_Connection()
*/
int err;
int osock = mConn->fd;
err = unix_fcntl_nonblock(osock);
if (err < 0)
{
RS_ERR("Cannot make socket NON-Blocking:", err);
waiting = WAITING_FAIL_INTERFACE;
net_internal_close(osock);
return false;
}
#ifdef WINDOWS_SYS
/* Set TCP buffer size for Windows systems */
int sockbufsize = 0;
int size = sizeof(int);
err = getsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP receive buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP receive buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = 0;
err = getsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP send buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP send buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = WINDOWS_TCP_BUFFER_SIZE;
err = setsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP receive buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP receive buffer size. Error " << err << std::endl;
}
#endif
err = setsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP send buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP send buffer size. Error " << err << std::endl;
}
#endif
#endif // WINDOWS_SYS
mTimeoutTS = time(NULL) + mConnectTimeout;
//std::cerr << "Setting Connect Timeout " << mConnectTimeout << " Seconds into Future " << std::endl;
waiting = WAITING_SOCK_CONNECT;
sockfd = osock;
return true;
}

47
libretroshare/src/pqi/pqissli2psam3.h Normal file
View File

@ -0,0 +1,47 @@
#ifndef PQISSLI2PSAM3_H
#define PQISSLI2PSAM3_H
#include "pqi/pqissl.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "services/autoproxy/p3i2psam3.h"
// Use a state machine as the whole pqi code is designed around them and some operation (like lookup) might be blocking
enum class pqisslSam3State : uint8_t {
NONE = 0,
DO_LOOKUP,
WAIT_LOOKUP,
DO_CONNECT,
WAIT_CONNECT,
DONE
};
class pqissli2psam3 : public pqissl, public autoProxyCallback
{
public:
pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm);
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Initiate_Connection();
int net_internal_close(int fd);
// autoProxyCallback interface
public:
void taskFinished(taskTicket *&ticket);
private:
bool setupSocket();
private:
pqisslSam3State mState;
std::string mI2pAddrB32;
std::string mI2pAddrLong;
Sam3Connection *mConn;
};
#endif // PQISSLI2PSAM3_H

4
libretroshare/src/pqi/pqissllistener.h
View File

@ -186,8 +186,8 @@ public:
virtual int finaliseConnection(int fd, SSL *ssl, const RsPeerId& peerId,
const sockaddr_storage &raddr);
RS_SET_CONTEXT_DEBUG_LEVEL(2)
private:
std::map<RsPeerId, pqissl*> listenaddr;
RS_SET_CONTEXT_DEBUG_LEVEL(2)
};

22
libretroshare/src/pqi/pqisslpersongrp.cc
View File

@ -46,7 +46,7 @@ static struct RsLog::logInfo pqipersongrpzoneInfo = {RsLog::Default, "pqipersong
#endif
#include "pqi/pqisslproxy.h"
#include "pqi/pqissli2pbob.h"
#include "pqi/pqissli2psam3.h"
pqilistener * pqisslpersongrp::locked_createListener(const struct sockaddr_storage &laddr)
{
@ -74,24 +74,26 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
std::cerr << std::endl;
#endif
// Use pqicI2PBOB for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2PBOB;
// Use pqicI2P for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2P;
{
pqisslproxy *pqis = new pqisslproxy((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicSOCKSProxy = new pqiconnect(pqip, rss, pqis);
}
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PBOB))
#ifdef RS_USE_I2P_SAM3
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PSAM3))
{
pqissli2pbob *pqis = new pqissli2pbob((pqissllistener *) listener, pqip, mLinkMgr);
pqissli2psam3 *pqis = new pqissli2psam3((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicI2PBOB = new pqiconnect(pqip, rss, pqis);
} else {
pqicI2PBOB = pqicSOCKSProxy;
pqicI2P = new pqiconnect(pqip, rss, pqis);
}
else
#endif // def RS_USE_I2P_SAM3
pqicI2P = pqicSOCKSProxy;
/* first select type based on peer */
@ -101,7 +103,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_TOR_TCP, pqicSOCKSProxy);
break;
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* peer is not a hidden one but we are */
@ -109,7 +111,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
uint32_t typeOwn = mPeerMgr->getHiddenType(AuthSSL::getAuthSSL()->OwnId());
switch (typeOwn) {
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* this case shouldn't happen! */

5
libretroshare/src/retroshare/rsevents.h
View File

@ -91,7 +91,7 @@ enum class RsEventType : uint32_t
/// @see RsGxsPostedEvent
GXS_IDENTITY = 12,
/// @see RsFiles
/// @see RsFiles @deprecated
SHARED_DIRECTORIES = 13,
/// @see RsFiles
@ -103,6 +103,9 @@ enum class RsEventType : uint32_t
/// @see rspeers.h
NETWORK = 16,
/** Emitted to update library clients about file hashing being completed */
FILE_HASHING_COMPLETED = 20,
/// @see rspeers.h
TOR_MANAGER = 17,

28
libretroshare/src/retroshare/rsfiles.h
View File

@ -194,7 +194,8 @@ enum class RsFileTransferEventCode: uint8_t {
COMPLETED_FILES_REMOVED = 0x02, //
};
struct RsSharedDirectoriesEvent: RsEvent
struct RS_DEPRECATED_FOR("Packing arbitrary data into an std::string is bad idea")
RsSharedDirectoriesEvent: RsEvent
{
RsSharedDirectoriesEvent() : RsEvent(RsEventType::SHARED_DIRECTORIES), mEventCode(RsSharedDirectoriesEventCode::UNKNOWN) {}
~RsSharedDirectoriesEvent() override = default;
@ -212,6 +213,31 @@ struct RsSharedDirectoriesEvent: RsEvent
std::string mMessage;
};
struct RsFileHashingCompletedEvent: RsEvent
{
RsFileHashingCompletedEvent():
RsEvent(RsEventType::FILE_HASHING_COMPLETED), mHashingSpeed(0) {}
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mFilePath);
RS_SERIAL_PROCESS(mFileHash);
RS_SERIAL_PROCESS(mHashingSpeed);
}
/// Complete path of the file being hashed
std::string mFilePath;
/// File hash, null if error occurred
RsFileHash mFileHash;
/// Hashing speed in MB/s
double mHashingSpeed;
};
struct RsFileTransferEvent: RsEvent
{
RsFileTransferEvent() : RsEvent(RsEventType::FILE_TRANSFER), mFileTransferEventCode(RsFileTransferEventCode::UNKNOWN) {}

5
libretroshare/src/retroshare/rsgxscircles.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
{
UNKNOWN = 0, /// Used to detect uninizialized values.
PUBLIC = 1, /// Public distribution, based on GxsIds
PUBLIC = 1, /// Public distribution
EXTERNAL = 2, /// Restricted to an external circle, based on GxsIds
NODES_GROUP = 3, /// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

76
libretroshare/src/retroshare/rsgxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -118,6 +118,10 @@ enum class RsForumEventCode: uint8_t
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
DELETED_POST = 0x0d, /// Post deleted (usually by cleaning)
/// Distant search result received
DISTANT_SEARCH_RESULT = 0x0e
};
struct RsGxsForumEvent: RsEvent
@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mForumGroupId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mModeratorsAdded);
RS_SERIAL_PROCESS(mModeratorsRemoved);
}
@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
~RsGxsForumEvent() override;
};
/** This event is fired once distant search results are received */
struct RsGxsForumsDistantSearchEvent: RsEvent
{
RsGxsForumsDistantSearchEvent():
RsEvent(RsEventType::GXS_FORUMS),
mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
RsForumEventCode mForumEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsForums: public RsGxsIfaceHelper
{
public:
@ -385,6 +411,50 @@ public:
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) = 0;
/**
* @brief Get forum content summaries
* @jsonapi{development}
* @param[in] forumId id of the forum of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries
* @return success or error details if something failed
*/
virtual std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/**
* @brief Search the whole reachable network for matching forums and
* posts
* @jsonapi{development}
* An @see RsGxsForumsDistantSearchEvent is emitted when matching results
* arrives from the network
* @param[in] matchString string to search into the forum and posts
* @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/
virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/**
* @brief Search the local index for matching forums and posts
* @jsonapi{development}
* @param[in] matchString string to search into the index
* @param[out] searchResults storage for searchr esults
* @return success or error details
*/
virtual std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) = 0;
////////////////////////////////////////////////////////////////////////////
/* Following functions are deprecated and should not be considered a stable
* to use API */
/**
* @brief Create forum. Blocking API.
* @jsonapi{development}

62
libretroshare/src/retroshare/rsgxsiface.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*!
* This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSummary : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{
RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSearchResults : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default;
};
/*!
* Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes.

83
libretroshare/src/retroshare/rsgxsifacehelper.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -208,6 +209,58 @@ public:
return mGxs.getGroupStatistic(token, stats);
}
/*!
* @brief Get default maximum storage time for GXS messages
* @jsonapi{development}
* @return storage time in seconds
*/
uint32_t getDefaultStoragePeriod()
{ return mGxs.getDefaultStoragePeriod(); }
/*!
* @brief Get maximum storage time of GXS messages for the given group
* @jsonapi{development}
* @param[in] groupId Id of the group
* @return storage time in seconds
*/
uint32_t getStoragePeriod(const RsGxsGroupId& groupId)
{ return mGxs.getStoragePeriod(groupId); }
/*!
* @brief Set GXS messages maximum storage time for the given group
* @jsonapi{development}
* @param[in] groupId Id of the group
* @param[in] storageSecs storage time in seconds
*/
void setStoragePeriod(const RsGxsGroupId& groupId, uint32_t storageSecs)
{ mGxs.setStoragePeriod(groupId, storageSecs); }
/*!
* @brief Get default maximum syncronization age for GXS messages
* @jsonapi{development}
* @return age in seconds
*/
uint32_t getDefaultSyncPeriod()
{ return mGxs.getDefaultSyncPeriod(); }
/*!
* @brief Get maximum syncronization age of GXS messages for the given group
* @jsonapi{development}
* @param[in] groupId Id of the group
* @return age in seconds
*/
uint32_t getSyncPeriod(const RsGxsGroupId& groupId)
{ return mGxs.getSyncPeriod(groupId); }
/*!
* @brief Set GXS messages maximum syncronization age for the given group
* @jsonapi{development}
* @param[in] groupId Id of the group
* @param[in] syncAge age in seconds
*/
void setSyncPeriod(const RsGxsGroupId& groupId, uint32_t syncAge)
{ mGxs.setSyncPeriod(groupId, syncAge); }
/*!
* This determines the reputation threshold messages need to surpass in order
* for it to be accepted by local user from remote source
@ -221,34 +274,6 @@ public:
return mGxs.setGroupReputationCutOff(token, grpId, CutOff);
}
/*!
* @return storage/sync time of messages in secs
*/
uint32_t getDefaultStoragePeriod()
{
return mGxs.getDefaultStoragePeriod();
}
uint32_t getStoragePeriod(const RsGxsGroupId& grpId)
{
return mGxs.getStoragePeriod(grpId);
}
void setStoragePeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs)
{
mGxs.setStoragePeriod(grpId,age_in_secs);
}
uint32_t getDefaultSyncPeriod()
{
return mGxs.getDefaultSyncPeriod();
}
uint32_t getSyncPeriod(const RsGxsGroupId& grpId)
{
return mGxs.getSyncPeriod(grpId);
}
void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs)
{
mGxs.setSyncPeriod(grpId,age_in_secs);
}
RsReputationLevel minReputationForForwardingMessages(
uint32_t group_sign_flags, uint32_t identity_flags )
{

2
libretroshare/src/retroshare/rsgxsifacetypes.h
View File

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {}
virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

2
libretroshare/src/retroshare/rsinit.h
View File

@ -197,7 +197,7 @@ public:
/*
* Setup Hidden Location;
*/
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob);
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p);
static bool LoadPassword(const std::string& passwd) ;

3
libretroshare/src/retroshare/rsmsgs.h
View File

@ -906,8 +906,9 @@ virtual void getOwnAvatarData(unsigned char *& data,int& size) = 0 ;
* @brief denyLobbyInvite deny a chat lobby invite
* @jsonapi{development}
* @param[in] id chat lobby id you were invited into
* @return true on success
*/
virtual void denyLobbyInvite(const ChatLobbyId &id) = 0 ;
virtual bool denyLobbyInvite(const ChatLobbyId &id) = 0 ;
/**
* @brief getPendingChatLobbyInvites get a list of all pending chat lobby invites

1
libretroshare/src/retroshare/rspeers.h
View File

@ -788,6 +788,7 @@ public:
virtual bool setProxyServer(const uint32_t type, const std::string &addr, const uint16_t port) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
virtual void allowServerIPDetermination(bool) = 0;
virtual bool resetOwnExternalAddressList() = 0;
virtual bool getAllowServerIPDetermination() = 0 ;

51
libretroshare/src/retroshare/rstypes.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -135,17 +137,10 @@ struct PeerBandwidthLimits : RsSerializable
}
};
//class SearchRequest // unused stuff.
//{
// public:
// int searchId;
// RsCertId toId; /* all zeros for everyone! */
// std::list<Condition> tests;
//};
/********************** For FileCache Interface *****************/
/* G10h4ck: Having this kind of stuff on public headers is pretty dangerous for
* name space pollution, a C++11 enum class should be used instad ASAP */
#define DIR_TYPE_UNKNOWN 0x00
#define DIR_TYPE_ROOT 0x01
#define DIR_TYPE_PERSON 0x02
@ -258,6 +253,9 @@ struct DirStub : RsSerializable
uint8_t type;
std::string name;
/* G10h4ck do we still need to keep this as void* instead of uint64_t for
* retroshare-gui sake? */
void* ref;
/// @see RsSerializable
@ -267,29 +265,12 @@ struct DirStub : RsSerializable
RS_SERIAL_PROCESS(type);
RS_SERIAL_PROCESS(name);
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // defined(__GNUC__) && !defined(__clang__)
// (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
// preferable to force type serialization to the correct size rather than letting the compiler choose for us.
// /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
if(sizeof(ref) == 4)
{
std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
}
else if(sizeof(ref) == 8)
{
std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
RS_SERIAL_PROCESS(handle);
}
else
std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic pop
#endif // defined(__GNUC__) && !defined(__clang__)
ref = reinterpret_cast<void*>(handle);
}
};
@ -330,9 +311,9 @@ struct DirDetails : RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx) override
{
/* Enforce serialization as uint64_t because void* changes size
* depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
*/
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
ref = reinterpret_cast<void*>(handle);

62
libretroshare/src/rsitems/rsgxsforumitems.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_GXS_FORUM_ITEMS_H
#define RS_GXS_FORUM_ITEMS_H
#pragma once
#include <map>
@ -31,7 +32,18 @@
#include "retroshare/rsgxsforums.h"
enum class RsGxsForumsItems : uint8_t
{
GROUP_ITEM = 0x02,
MESSAGE_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
class RsGxsForumGroupItem : public RsGxsGrpItem
@ -61,6 +73,48 @@ public:
RsGxsForumMsg mMsg;
};
struct RsGxsForumsSearchRequest : RsSerializable
{
RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsForumsSearchRequest() override = default;
};
struct RsGxsForumsSearchReply : RsSerializable
{
RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsForumsSearchReply() override = default;
};
class RsGxsForumSerialiser : public RsServiceSerializer
{
public:
@ -69,5 +123,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
};
#endif /* RS_GXS_FORUM_ITEMS_H */

5
libretroshare/src/rsitems/rsitem.h
View File

@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0;
{
std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
<< "new serialization classes, but not derived! Class is "
<< typeid(*this).name() << std::endl;
RS_ERR( "called by an item using new serialization system without "
"overriding Class is: ", typeid(*this).name() );
print_stacktrace();
}

14
libretroshare/src/rsitems/rsserviceids.h
View File

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t
{
NONE = 0, /// To detect non-initialized reads
/// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012,
MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028,
BANLIST = 0x0101,
STATUS = 0x0102,
NXS = 0x0200,
@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216,
CHANNELS = 0x0217,
GXSCIRCLE = 0x0218,
/// not gxs, but used with identities.
REPUTATION = 0x0219,
GXS_RECOGN = 0x0220,
@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB,

6
libretroshare/src/rsserver/p3face.h
View File

@ -43,7 +43,7 @@
class p3heartbeat;
class p3discovery2;
class p3I2pBob;
class p3I2pSam3;
/* GXS Classes - just declare the classes.
so we don't have to totally recompile to switch */
@ -161,8 +161,8 @@ public:
p3ChatService *chatSrv;
p3StatusService *mStatusSrv;
p3GxsTunnelService *mGxsTunnels;
#ifdef RS_USE_I2P_BOB
p3I2pBob *mI2pBob;
#ifdef RS_USE_I2P_SAM3
p3I2pSam3 *mI2pSam3;
#endif
// This list contains all threaded services. It will be used to shut them down properly.

4
libretroshare/src/rsserver/p3msgs.cc
View File

@ -530,9 +530,9 @@ bool p3Msgs::acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& gxs_id)
{
return mChatSrv->acceptLobbyInvite(id,gxs_id) ;
}
void p3Msgs::denyLobbyInvite(const ChatLobbyId& id)
bool p3Msgs::denyLobbyInvite(const ChatLobbyId& id)
{
mChatSrv->denyLobbyInvite(id) ;
return mChatSrv->denyLobbyInvite(id) ;
}
void p3Msgs::getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites)
{

110
libretroshare/src/rsserver/p3msgs.h
View File

@ -64,66 +64,66 @@ public:
/*!
* @param msgList ref to list summarising client's msgs
*/
virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList);
virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg);
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox);
virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList)override ;
virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg)override ;
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox)override ;
RS_DEPRECATED_FOR(sendMail)
virtual bool MessageSend(Rs::Msgs::MessageInfo &info);
virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag);
virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId);
virtual bool MessageToTrash(const std::string &mid, bool bTrash);
virtual bool MessageDelete(const std::string &mid);
virtual bool MessageRead(const std::string &mid, bool unreadByUser);
virtual bool MessageReplied(const std::string &mid, bool replied);
virtual bool MessageForwarded(const std::string &mid, bool forwarded);
virtual bool MessageStar(const std::string &mid, bool star);
virtual bool MessageJunk(const std::string &mid, bool junk);
virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load);
virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId);
virtual bool MessageSend(Rs::Msgs::MessageInfo &info)override ;
virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag)override ;
virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId)override ;
virtual bool MessageToTrash(const std::string &mid, bool bTrash)override ;
virtual bool MessageDelete(const std::string &mid)override ;
virtual bool MessageRead(const std::string &mid, bool unreadByUser)override ;
virtual bool MessageReplied(const std::string &mid, bool replied)override ;
virtual bool MessageForwarded(const std::string &mid, bool forwarded)override ;
virtual bool MessageStar(const std::string &mid, bool star)override ;
virtual bool MessageJunk(const std::string &mid, bool junk)override ;
virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load)override ;
virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId)override ;
virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags);
virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color);
virtual bool removeMessageTagType(uint32_t tagId);
virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags)override ;
virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color)override ;
virtual bool removeMessageTagType(uint32_t tagId)override ;
virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info);
virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info)override ;
/* set == false && tagId == 0 --> remove all */
virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set);
virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set)override ;
virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags);
virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags)override ;
virtual uint32_t getDistantMessagingPermissionFlags() ;
virtual void setDistantMessagingPermissionFlags(uint32_t flags) ;
virtual uint32_t getDistantMessagingPermissionFlags() override ;
virtual void setDistantMessagingPermissionFlags(uint32_t flags) override ;
/*!
* gets avatar from peer, image data in jpeg format
*/
virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size);
virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size)override ;
/*!
* sets clients avatar, image data should be in jpeg format
*/
virtual void setOwnAvatarData(const unsigned char *data,int size);
virtual void setOwnAvatarData(const unsigned char *data,int size)override ;
/*!
* retrieve clients avatar, image data in jpeg format
*/
virtual void getOwnAvatarData(unsigned char *& data,int& size);
virtual void getOwnAvatarData(unsigned char *& data,int& size)override ;
/*!
* sets clients custom status (e.g. "i'm tired")
*/
virtual void setCustomStateString(const std::string& status_string) ;
virtual void setCustomStateString(const std::string& status_string) override ;
/*!
* retrieves client's custom status
*/
virtual std::string getCustomStateString() ;
virtual std::string getCustomStateString() override ;
/*!
* retrieves peer's custom status
*/
virtual std::string getCustomStateString(const RsPeerId& peer_id) ;
virtual std::string getCustomStateString(const RsPeerId& peer_id) override ;
/*!
@ -132,56 +132,56 @@ public:
* @param msg the message
* @see ChatId
*/
virtual bool sendChat(ChatId destination, std::string msg) ;
virtual bool sendChat(ChatId destination, std::string msg) override ;
/*!
* Return the max message size for security forwarding
*/
virtual uint32_t getMaxMessageSecuritySize(int type);
virtual uint32_t getMaxMessageSecuritySize(int type)override ;
/*!
* sends immediate status string to a specific peer, e.g. in a private chat
* @param chat_id chat id to send status string to
* @param status_string immediate status to send
*/
virtual void sendStatusString(const ChatId& id, const std::string& status_string) ;
virtual void sendStatusString(const ChatId& id, const std::string& status_string) override ;
/**
* @brief clearChatLobby: Signal chat was cleared by GUI.
* @param id: Chat id cleared.
*/
virtual void clearChatLobby(const ChatId &id);
virtual void clearChatLobby(const ChatId &id)override ;
/****************************************/
virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) ;
virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) ;
virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) ;
virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) ;
virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) ;
virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) ;
virtual void denyLobbyInvite(const ChatLobbyId& id) ;
virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;
virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id);
virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) ;
virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) ;
virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) ;
virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) ;
virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe);
virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id);
virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) ;
virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) override ;
virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) override ;
virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) override ;
virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) override ;
virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) override ;
virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) override ;
virtual bool denyLobbyInvite(const ChatLobbyId& id) override ;
virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) override ;
virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) override ;
virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id)override ;
virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) override ;
virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) override ;
virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) override ;
virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) override ;
virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe)override ;
virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id)override ;
virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) override ;
virtual bool initiateDistantChatConnexion(
const RsGxsId& to_gxs_id, const RsGxsId& from_gxs_id,
DistantChatPeerId &pid, uint32_t& error_code,
bool notify = true );
bool notify = true )override ;
virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info);
virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) ;
virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info)override ;
virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) override ;
virtual uint32_t getDistantChatPermissionFlags() ;
virtual bool setDistantChatPermissionFlags(uint32_t flags) ;
virtual uint32_t getDistantChatPermissionFlags() override ;
virtual bool setDistantChatPermissionFlags(uint32_t flags) override ;
private:

53
libretroshare/src/rsserver/p3peers.cc
View File

@ -373,7 +373,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec loc", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
for(it = ps.ipAddrs.mExt.mAddrs.begin(); it != ps.ipAddrs.mExt.mAddrs.end(); ++it)
@ -381,7 +381,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec ext", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
}
@ -859,8 +859,14 @@ void p3Peers::getIPServersList(std::list<std::string>& ip_servers)
{
mNetMgr->getIPServersList(ip_servers) ;
}
void p3Peers::getCurrentExtIPList(std::list<std::string>& ip_list)
{
mNetMgr->getCurrentExtIPList(ip_list) ;
}
bool p3Peers::resetOwnExternalAddressList()
{
//TODO Phenom 2021-10-30: Need to call something like mNetMgr->netReset();
// to update this addresslist.
return mPeerMgr->resetOwnExternalAddressList();
}
void p3Peers::allowServerIPDetermination(bool b)
@ -1287,7 +1293,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
#else
sockaddr_storage tLocal;
if(sockaddr_storage_inet_pton(tLocal, tDetails.localAddr) && sockaddr_storage_isValidNet(tLocal) && sockaddr_storage_ipv6_to_ipv4(tLocal) && tDetails.localPort )
bool validLoc = sockaddr_storage_inet_pton(tLocal, tDetails.localAddr)
&& sockaddr_storage_isValidNet(tLocal)
&& tDetails.localPort;
bool isLocIpv4 = sockaddr_storage_ipv6_to_ipv4(tLocal);
if(validLoc && isLocIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tLocal).sin_addr.s_addr;
@ -1305,7 +1315,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
sockaddr_storage tExt;
if(sockaddr_storage_inet_pton(tExt, tDetails.extAddr) && sockaddr_storage_isValidNet(tExt) && sockaddr_storage_ipv6_to_ipv4(tExt) && tDetails.extPort )
bool validExt = sockaddr_storage_inet_pton(tExt, tDetails.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& tDetails.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if(validExt && isExtIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tExt).sin_addr.s_addr;
@ -1321,6 +1335,17 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
offset += 4+2;
}
else if(validExt && !isExtIpv4)
{
// External address is IPv6, save it on LOCATOR
sockaddr_storage_setport(tExt,tDetails.extPort);
std::string tLocator = sockaddr_storage_tostring(tExt);
addPacketHeader(RsShortInviteFieldType::LOCATOR, tLocator.size(),buf,offset,buf_size);
memcpy(&buf[offset],tLocator.c_str(),tLocator.size());
offset += tLocator.size();
}
#endif
}
@ -1595,9 +1620,27 @@ std::string p3Peers::GetRetroshareInvite( const RsPeerId& sslId, RetroshareInvit
if (getPeerDetails(ssl_id, detail))
{
if(!(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY) || detail.isHiddenNode)
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
|| detail.isHiddenNode)
detail.ipAddressList.clear();
//Check if external address is IPv6, then move it to ipAddressList as RsCertificate only allow 4 numbers.
sockaddr_storage tExt;
bool validExt = sockaddr_storage_inet_pton(tExt, detail.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& detail.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
&& !detail.isHiddenNode
&& validExt && !isExtIpv4)
{
sockaddr_storage_setport(tExt,detail.extPort);
detail.ipAddressList.push_front(sockaddr_storage_tostring(tExt) + " "); // Space needed to later parse.
detail.extAddr = ""; //Clear it to not trigg error.
detail.extPort = 0;
}
unsigned char *mem_block = nullptr;
size_t mem_block_size = 0;

1
libretroshare/src/rsserver/p3peers.h
View File

@ -132,6 +132,7 @@ public:
virtual bool isProxyAddress(const uint32_t type, const sockaddr_storage &addr);
virtual void getIPServersList(std::list<std::string>& ip_servers) override;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) override;
virtual void allowServerIPDetermination(bool) override;
virtual bool getAllowServerIPDetermination() override;
virtual bool resetOwnExternalAddressList() override;

82
libretroshare/src/rsserver/rsinit.cc
View File

@ -2,7 +2,8 @@
* libretroshare/src/retroshare: rsinit.cc *
* *
* Copyright (C) 2004-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -173,7 +174,7 @@ struct RsInitConfig
std::string hiddenNodeAddress;
uint16_t hiddenNodePort;
bool hiddenNodeI2PBOB;
bool hiddenNodeI2P;
/* Logging */
bool haveLogFile;
@ -667,13 +668,13 @@ void RsInit::setAutoLogin(bool autoLogin){
}
/* Setup Hidden Location; */
void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob)
void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p)
{
/* parse the bugger (todo) */
rsInitConfig->hiddenNodeSet = true;
rsInitConfig->hiddenNodeAddress = hiddenaddress;
rsInitConfig->hiddenNodePort = port;
rsInitConfig->hiddenNodeI2PBOB = useBob;
rsInitConfig->hiddenNodeI2P = useI2p;
}
@ -720,7 +721,7 @@ RsGRouter *rsGRouter = NULL ;
# include "rs_upnp/upnphandler_miniupnp.h"
#endif // def RS_USE_LIBUPNP
#include "services/autoproxy/p3i2pbob.h"
#include "services/autoproxy/p3i2psam3.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "services/p3gxsreputation.h"
@ -926,9 +927,9 @@ int RsServer::StartupRetroShare()
mNetMgr->setManagers(mPeerMgr, mLinkMgr);
rsAutoProxyMonitor *autoProxy = rsAutoProxyMonitor::instance();
#ifdef RS_USE_I2P_BOB
mI2pBob = new p3I2pBob(mPeerMgr);
autoProxy->addProxy(autoProxyType::I2PBOB, mI2pBob);
#ifdef RS_USE_I2P_SAM3
mI2pSam3 = new p3I2pSam3(mPeerMgr);
autoProxy->addProxy(autoProxyType::I2PSAM3, mI2pSam3);
#endif
//load all the SSL certs as friends
@ -1339,23 +1340,27 @@ int RsServer::StartupRetroShare()
mWiki->setNetworkExchangeService(wiki_ns) ;
#endif
/**** Forum GXS service ****/
/************************* Forum GXS service ******************************/
RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
RsGeneralDataService* gxsforums_ds = new RsDataService(
currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
nullptr, rsInitConfig->gxs_passwd );
p3GxsForums* mGxsForums = new p3GxsForums(
gxsforums_ds, nullptr, mGxsIdService );
p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
#ifdef RS_DEEP_FORUMS_INDEX
gxsForumsTunnelService = mGxsNetTunnel;
#endif
// create GXS photo service
RsGxsNetService* gxsforums_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
mGxsForums, mGxsForums->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
mGxsForums->setNetworkExchangeService(gxsforums_ns);
/**** Channel GXS service ****/
RsGeneralDataService* gxschannels_ds = new RsDataService(currGxsDir + "/", "gxschannels_db",
@ -1602,6 +1607,9 @@ int RsServer::StartupRetroShare()
// Turtle search for GXS services
mGxsNetTunnel->registerSearchableService(gxschannels_ns);
#ifdef RS_DEEP_FORUMS_INDEX
mGxsNetTunnel->registerSearchableService(gxsforums_ns);
#endif
/**************************************************************************/
@ -1658,8 +1666,9 @@ int RsServer::StartupRetroShare()
mConfigMgr->addConfiguration("wire.cfg", wire_ns);
#endif
#endif //RS_ENABLE_GXS
#ifdef RS_USE_I2P_BOB
mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pBob);
#ifdef RS_USE_I2P_SAM3
// to make migration easiert, SAM will use BOBs configuration, as they are compatible / the same.
mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pSam3);
#endif
mPluginsManager->addConfigurations(mConfigMgr) ;
@ -1712,34 +1721,33 @@ int RsServer::StartupRetroShare()
{
std::cout << "RsServer::StartupRetroShare setting up hidden locations" << std::endl;
if (rsInitConfig->hiddenNodeI2PBOB) {
std::cout << "RsServer::StartupRetroShare setting up BOB" << std::endl;
if (rsInitConfig->hiddenNodeI2P) {
std::cout << "RsServer::StartupRetroShare setting up SAMv3" << std::endl;
// we need a local port!
mNetMgr->checkNetAddress();
// add i2p proxy
// bob will use this address
sockaddr_storage i2pInstance;
sockaddr_storage_ipv4_aton(i2pInstance, rsInitConfig->hiddenNodeAddress.c_str());
mPeerMgr->setProxyServerAddress(RS_HIDDEN_TYPE_I2P, i2pInstance);
std::string addr; // will be set by auto proxy service
uint16_t port = rsInitConfig->hiddenNodePort; // unused by bob
uint16_t port; // unused by SAM
bool r = autoProxy->initialSetup(autoProxyType::I2PBOB, addr, port);
bool r = autoProxy->initialSetup(autoProxyType::I2PSAM3, addr, port);
if (r && !addr.empty()) {
mPeerMgr->setupHiddenNode(addr, port);
// now enable bob
bobSettings bs;
autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::getSettings, &bs);
bs.enable = true;
autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &bs);
// now enable SAM
samSettings ss;
autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, &ss);
ss.enable = true;
autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::setSettings, &ss);
} else {
std::cerr << "RsServer::StartupRetroShare failed to receive keys" << std::endl;
/// TODO add notify for failed bob setup
/// TODO add notify for failed i2p setup
}
} else {
mPeerMgr->setupHiddenNode(rsInitConfig->hiddenNodeAddress, rsInitConfig->hiddenNodePort);
@ -1761,19 +1769,17 @@ int RsServer::StartupRetroShare()
if (rsInitConfig->hiddenNodeSet) {
// newly created location
// mNetMgr->checkNetAddress() will setup ports for us
#if 0 // this was used for BOB but is not requires for SAMv3
// trigger updates for auto proxy services
std::vector<autoProxyType::autoProxyType_enum> types;
// i2p bob need to rebuild its command map
types.push_back(autoProxyType::I2PBOB);
rsAutoProxyMonitor::taskSync(types, autoProxyTask::reloadConfig);
#endif
}
/**************************************************************************/
/* startup (stuff dependent on Ids/peers is after this point) */
/**************************************************************************/
autoProxy->startAll();
pqih->init_listener();
@ -1806,8 +1812,8 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
// auto proxy threads
#ifdef RS_USE_I2P_BOB
startServiceThread(mI2pBob, "I2P-BOB");
#ifdef RS_USE_I2P_SAM3
startServiceThread(mI2pSam3, "I2P-SAM3");
#endif
#ifdef RS_ENABLE_GXS

1117
libretroshare/src/services/autoproxy/p3i2pbob.cc
View File

File diff suppressed because it is too large Load Diff

256
libretroshare/src/services/autoproxy/p3i2pbob.h
View File

@ -1,256 +0,0 @@
/*******************************************************************************
* libretroshare/src/services/autoproxy: p3i2pbob.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef P3I2PBOB_H
#define P3I2PBOB_H
#include <map>
#include <queue>
#include <sys/types.h>
#include "util/rstime.h"
#ifndef WINDOWS_SYS
#include <sys/socket.h>
#endif
#include "pqi/p3cfgmgr.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "util/rsthreads.h"
#include "util/i2pcommon.h"
/*
* This class implements I2P BOB (BASIC OPEN BRIDGE) communication to allow RS
* to automatically remote control I2P to setup the needed tunnel.
* BOB is a simple text-based interface: https://geti2p.net/en/docs/api/bob
*
* Note 1:
* One tunnel is enough even for hidden locations since it can be used
* bidirectional. (In contrast to what RS I2P users had to set up manually.)
*
* Note 2:
* BOB tunnels are no SOCKS tunnel. Therefore pqissli2pbob implements a simplified
* proxy specially for BOB tunnels.
*
* Note 3:
* BOB needs a unique name as an ID for each tunnel.
* We use 'RetroShare-' + 8 random base32 characters.
*
* Design:
* The service uses three state machines to manage its task:
* int stateMachineBOB();
* mBOBState
* int stateMachineController();
* mState
* mTask
*
* stateMachineBOB:
* This state machine manages the low level communication with BOB. It basically has a linked
* list (currently a implemented as a std::map) that contains a command and the next
* state.
* Each high level operation (start up / shut down / get keys) is represented by a
* chain of states. E.g. the chain to retrieve new keys:
* mCommands[bobState::setnickN] = {setnick, bobState::newkeysN};
* mCommands[bobState::newkeysN] = {newkeys, bobState::getkeys};
* mCommands[bobState::getkeys] = {getkeys, bobState::clear};
* mCommands[bobState::clear] = {clear, bobState::quit};
* mCommands[bobState::quit] = {quit, bobState::cleared};
*
* stateMachineController:
* This state machine manages the high level tasks.
* It is controlled by mState and mTask.
*
* mTast:
* Tracks the high level operation (like start up).
* It will keep its value even when a task is done to track
* the requested BOB state.
* When other operations are performed like a conection check
* the last task gets backed up and is later restored again
*
* mState:
* This state lives only for one operation an manages the communication
* with the BOB instance. This is basically connecting, starting BOB
* protocol and disconnecting
*
* How a task looks like:
* 1) RS sets task using the ticket system
* 2) stateMachineController connects to BOBs control port, sets mBobState to a lists head
* 3) stateMachineBOB processes command chain
* 4) stateMachineBOB is done and sets mBobState to cleared signaling that the connection
* is cleared and can be closed
* 5) stateMachineController disconnects from BOBs control port and updates mState
*/
///
/// \brief The controllerState enum
/// States for the controller to keep track of what he is currently doing
enum controllerState {
csIdel,
csDoConnect,
csConnected,
csWaitForBob,
csDoDisconnect,
csDisconnected,
csError
};
///
/// \brief The controllerTask enum
/// This state tracks the controllers tast (e.g. setup a BOB tunnel or shut down
/// an existing one).
enum controllerTask {
ctIdle,
ctRunSetUp,
ctRunShutDown,
ctRunGetKeys,
ctRunCheck
};
///
/// \brief The bobState enum
/// One state for each message
///
enum bobState {
bsCleared,
bsSetnickC, // chain head for only client tunnel
bsSetnickN, // chain head for getting new (server) keys
bsSetnickS, // chain head for client and server tunnel
bsGetnick,
bsNewkeysC, // part of chain for only client tunnel
bsNewkeysN, // part of chain for getting new (server) keys
bsGetkeys,
bsSetkeys,
bsInhost,
bsOuthost,
bsInport,
bsOutport,
bsInlength,
bsOutlength,
bsInvariance,
bsOutvariance,
bsInquantity,
bsOutquantity,
bsQuiet,
bsStart,
bsStop,
bsClear,
bsList, // chain head for 'list' command
bsQuit
};
///
/// \brief The bobStateInfo struct
/// State machine with commands
/// \todo This could be replaced by a linked list instead of a map
struct bobStateInfo {
std::string command;
bobState nextState;
};
struct bobSettings : i2p::settings {};
///
/// \brief The bobStates struct
/// This container struct is used to pass all states.
/// Additionally, the tunnel name is included to to show it in the GUI.
/// The advantage of a struct is that it can be forward declared.
struct bobStates {
bobState bs;
controllerState cs;
controllerTask ct;
std::string tunnelName;
};
class p3PeerMgr;
class p3I2pBob : public RsTickingThread, public p3Config, public autoProxyService
{
public:
explicit p3I2pBob(p3PeerMgr *peerMgr);
// autoProxyService interface
public:
bool isEnabled();
bool initialSetup(std::string &addr, uint16_t &);
void processTaskAsync(taskTicket *ticket);
void processTaskSync(taskTicket *ticket);
void threadTick() override; /// @see RsTickingThread
private:
int stateMachineBOB();
int stateMachineBOB_locked_failure(const std::string &answer, const bobStateInfo &currentState);
int stateMachineController();
int stateMachineController_locked_idle();
int stateMachineController_locked_connected();
int stateMachineController_locked_disconnected();
int stateMachineController_locked_error();
// p3Config interface
protected:
RsSerialiser *setupSerialiser();
bool saveList(bool &cleanup, std::list<RsItem *> &lst);
bool loadList(std::list<RsItem *> &load);
private:
// helpers
void getBOBSettings(bobSettings *settings);
void setBOBSettings(const bobSettings *settings);
void getStates(bobStates *bs);
std::string executeCommand(const std::string &command);
bool connectI2P();
bool disconnectI2P();
void finalizeSettings_locked();
void updateSettings_locked();
std::string recv();
// states for state machines
controllerState mState;
controllerTask mTask;
// used to store old state when in error state
// mStateOld is also used as a flag when an error occured in BOB protocol
controllerState mStateOld;
// mTaskOld is used to keep the previous task (start up / shut down) when requesting keys or checking the connection
controllerTask mTaskOld;
bobSettings mSetting;
bobState mBOBState;
// used variables
p3PeerMgr *mPeerMgr;
bool mConfigLoaded;
int mSocket;
rstime_t mLastProxyCheck;
sockaddr_storage mI2PProxyAddr;
std::map<bobState, bobStateInfo> mCommands;
std::string mErrorMsg;
std::string mTunnelName;
std::queue<taskTicket *> mPending;
taskTicket *mProcessing;
// mutex
RsMutex mLock;
};
#endif // P3I2PBOB_H

798
libretroshare/src/services/autoproxy/p3i2psam3.cpp Normal file
View File

@ -0,0 +1,798 @@
#include "p3i2psam3.h"
#include <libsam3.h>
#include "pqi/p3peermgr.h"
#include "rsitems/rsconfigitems.h"
static const std::string kConfigKeySAM3Enable = "SAM3_ENABLE";
static const std::string kConfigKeyDestPriv = "DEST_PRIV";
static const std::string kConfigKeyInLength = "IN_LENGTH";
static const std::string kConfigKeyInQuantity = "IN_QUANTITY";
static const std::string kConfigKeyInVariance = "IN_VARIANCE";
static const std::string kConfigKeyInBackupQuantity = "IN_BACKUPQUANTITY";
static const std::string kConfigKeyOutLength = "OUT_LENGTH";
static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY";
static const std::string kConfigKeyOutVariance = "OUT_VARIANCE";
static const std::string kConfigKeyOutBackupQuantity = "OUT_BACKUPQUANTITY";
#ifdef RS_I2P_SAM3_BOB_COMPAT
// used for migration from BOB to SAM
static const std::string kConfigKeyBOBEnable = "BOB_ENABLE";
static const std::string kConfigKeyBOBKey = "BOB_KEY";
static const std::string kConfigKeyBOBAddr = "BOB_ADDR";
#endif
static constexpr bool kDefaultSAM3Enable = false;
RS_SET_CONTEXT_DEBUG_LEVEL(2)
static void inline doSleep(std::chrono::duration<long, std::ratio<1,1000>> timeToSleepMS) {
std::this_thread::sleep_for(timeToSleepMS);
}
p3I2pSam3::p3I2pSam3(p3PeerMgr *peerMgr) :
mConfigLoaded(false), mPeerMgr(peerMgr), mPending(), mLock("p3i2p-sam3")
#ifdef RS_USE_I2P_SAM3_LIBSAM3
, mLockSam3Access("p3i2p-sam3-access")
#endif
{
RS_DBG4();
// set defaults
mSetting.initDefault();
mSetting.enable = kDefaultSAM3Enable;
mSetting.session = nullptr;
libsam3_debug = 0;
}
bool p3I2pSam3::isEnabled()
{
RS_STACK_MUTEX(mLock);
return mSetting.enable;
}
bool p3I2pSam3::initialSetup(std::string &addr, uint16_t &/*port*/)
{
RS_DBG4();
RS_STACK_MUTEX(mLock);
if (!mSetting.address.publicKey.empty() || !mSetting.address.privateKey.empty())
RS_WARN("overwriting keys!");
bool success = generateKey(mSetting.address.publicKey, mSetting.address.privateKey);
if (!success) {
RS_WARN("failed to retrieve keys");
return false;
} else {
std::string s, c;
i2p::getKeyTypes(mSetting.address.publicKey, s, c);
RS_INFO("received key ", s, " ", c);
RS_INFO("public key: ", mSetting.address.publicKey);
RS_INFO("private key: ", mSetting.address.privateKey);
RS_INFO("address: ", i2p::keyToBase32Addr(mSetting.address.publicKey));
// sanity check
auto pub = i2p::publicKeyFromPrivate(mSetting.address.privateKey);
RS_INFO("pub key derived: ", pub);
RS_INFO("address: ", i2p::keyToBase32Addr(pub));
if (pub != mSetting.address.publicKey) {
RS_WARN("public key does not match private key! fixing ...");
mSetting.address.publicKey = pub;
}
mSetting.address.base32 = i2p::keyToBase32Addr(mSetting.address.publicKey);
IndicateConfigChanged();
}
addr = mSetting.address.base32;
return true;
}
void p3I2pSam3::processTaskAsync(taskTicket *ticket)
{
RS_DBG4();
switch (ticket->task) {
case autoProxyTask::stop: [[fallthrough]];
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::proxyStatusCheck: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
{
RS_STACK_MUTEX(mLock);
mPending.push(ticket);
}
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig:
// These are supposed to be sync!
RS_DBG("unknown task or sync one!");
rsAutoProxyMonitor::taskError(ticket);
break;
}
}
void p3I2pSam3::processTaskSync(taskTicket *ticket)
{
// RS_DBG4();
const bool data = !!ticket->data;
switch (ticket->task) {
case autoProxyTask::status:
{
samStatus *ss = static_cast<struct samStatus *>(ticket->data);
RS_STACK_MUTEX(mLock);
ss->state = mState;
if (mSetting.session)
ss->sessionName = mSetting.session->channel;
else
ss->sessionName = "none";
}
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::getSettings:
// check if everything needed is set
if (!data) {
RS_DBG("autoProxyTask::getSettings data is missing");
rsAutoProxyMonitor::taskError(ticket);
break;
}
// get settings
{
RS_STACK_MUTEX(mLock);
*static_cast<struct samSettings *>(ticket->data) = mSetting;
}
// finish task
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::setSettings:
// check if everything needed is set
if (!data) {
RS_DBG("autoProxyTask::setSettings data is missing");
rsAutoProxyMonitor::taskError(ticket);
break;
}
// set settings
{
RS_STACK_MUTEX(mLock);
mSetting = *static_cast<struct samSettings *>(ticket->data);
updateSettings_locked();
}
// finish task
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::getErrorInfo:
*static_cast<std::string *>(ticket->data) = mSetting.session->error;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::reloadConfig:
{
RS_STACK_MUTEX(mLock);
updateSettings_locked();
}
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::stop:
#if 0 // doesn't seem to work, socket stays "CLOSE_WAIT"
// there can be a case where libsam3 will block forever because for some reason it does not detect that the socket it has is dead
// as a workaroung kill it from here
if (mState == samStatus::samState::connectSession || mState == samStatus::samState::connectForward) {
// lock should be held by the main thread
if (!mTmpSession) {
// now it's getting weird
RS_WARN("session is nullptr but mState says it is connecting.");
// no break! just ignore for now ...
} else {
// just close it from here, libsam3 is not thread safe.
// a bit of a hack but should do the trick
// sam3CloseSession(mSetting.session);
sam3tcpDisconnect(mTmpSession->fd);
// no break! continue as usual to keep everything in line
}
}
#endif
[[fallthrough]];
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::proxyStatusCheck: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
// These are supposed to be async!
RS_WARN("unknown task or async one!");
rsAutoProxyMonitor::taskError(ticket);
break;
}
}
void p3I2pSam3::threadTick()
{
// {
// RS_STACK_MUTEX(mLock);
// Dbg4() << __PRETTY_FUNCTION__ << " mPending: " << mPending.size() << std::endl;
// }
if(mPending.empty()) {
// sleep outisde of lock!
doSleep(std::chrono::milliseconds(250));
return;
}
// get task
taskTicket *tt = nullptr;
{
RS_STACK_MUTEX(mLock);
tt = mPending.front();
mPending.pop();
}
switch (tt->task) {
case autoProxyTask::stop:
mState = samStatus::samState::offline;
stopForwarding();
stopSession();
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::offline);
break;
case autoProxyTask::start:
{
if (!mSetting.enable) {
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::disabled);
break;
}
// create main session
mState = samStatus::samState::connectSession;
bool ret = startSession();
if (!ret) {
mState = samStatus::samState::offline;
rsAutoProxyMonitor::taskError(tt);
break;
}
// start forwarding
mState = samStatus::samState::connectForward;
ret = startForwarding();
// finish ticket
if (ret) {
mState = samStatus::samState::online;
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::online);
} else {
mState = samStatus::samState::offline;
rsAutoProxyMonitor::taskError(tt);
}
}
break;
case autoProxyTask::receiveKey:
{
i2p::address *addr = static_cast<i2p::address *>(tt->data);
if (generateKey(addr->publicKey, addr->privateKey)) {
addr->base32 = i2p::keyToBase32Addr(addr->publicKey);
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::ok);
} else {
rsAutoProxyMonitor::taskError(tt);
}
}
break;
case autoProxyTask::lookupKey:
lookupKey(tt);
break;
case autoProxyTask::proxyStatusCheck:
{
// TODO better detection of status
bool ok;
ok = !!mSetting.session->fd;
ok &= !!mSetting.session->fwd_fd;
*static_cast<bool*>(tt->data) = ok;
rsAutoProxyMonitor::taskDone(tt, ok ? autoProxyStatus::ok : autoProxyStatus::error);
}
break;
case autoProxyTask::establishConnection:
establishConnection(tt);
break;
case autoProxyTask::closeConnection:
closeConnection(tt);
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig:
RS_ERR("unable to handle! This is a bug! task:", tt->task);
rsAutoProxyMonitor::taskError(tt);
break;
}
tt = nullptr;
// give i2p backend some time
doSleep(std::chrono::milliseconds(100));
}
RsSerialiser *p3I2pSam3::setupSerialiser()
{
RsSerialiser* rsSerialiser = new RsSerialiser();
rsSerialiser->addSerialType(new RsGeneralConfigSerialiser());
return rsSerialiser;
}
#define addKVS(_key, _value) \
kv.key = _key;\
kv.value = _value;\
vitem->tlvkvs.pairs.push_back(kv);
#define addKVSInt(_key, _value) \
kv.key = _key;\
rs_sprintf(kv.value, "%d", _value);\
vitem->tlvkvs.pairs.push_back(kv);
bool p3I2pSam3::saveList(bool &cleanup, std::list<RsItem *> &lst)
{
RS_DBG4();
cleanup = true;
RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet;
RsTlvKeyValue kv;
RS_STACK_MUTEX(mLock);
addKVS(kConfigKeySAM3Enable, mSetting.enable ? "TRUE" : "FALSE")
addKVS(kConfigKeyDestPriv, mSetting.address.privateKey);
addKVSInt(kConfigKeyInLength, mSetting.inLength)
addKVSInt(kConfigKeyInQuantity, mSetting.inQuantity)
addKVSInt(kConfigKeyInVariance, mSetting.inVariance)
addKVSInt(kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
addKVSInt(kConfigKeyOutLength, mSetting.outLength)
addKVSInt(kConfigKeyOutQuantity, mSetting.outQuantity)
addKVSInt(kConfigKeyOutVariance, mSetting.outVariance)
addKVSInt(kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
#ifdef RS_I2P_SAM3_BOB_COMPAT
// these allow SAMv3 users to switch back to BOB
// remove after some time
addKVS(kConfigKeyBOBEnable, mSetting.enable ? "TRUE" : "FALSE")
addKVS(kConfigKeyBOBKey, mSetting.address.privateKey)
addKVS(kConfigKeyBOBAddr, mSetting.address.base32)
#endif
lst.push_back(vitem);
return true;
}
#undef addKVS
#undef addKVSUInt
#define getKVSUInt(_kit, _key, _value) \
else if (_kit->key == _key) {\
std::istringstream is(_kit->value);\
int tmp;\
is >> tmp;\
_value = (int8_t)tmp;\
}
bool p3I2pSam3::loadList(std::list<RsItem *> &load)
{
RS_DBG4();
std::string priv;
priv.clear();
for(std::list<RsItem*>::const_iterator it = load.begin(); it!=load.end(); ++it) {
RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it);
if(vitem != NULL) {
RS_STACK_MUTEX(mLock);
for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) {
if (kit->key == kConfigKeySAM3Enable)
mSetting.enable = kit->value == "TRUE";
else if (kit->key == kConfigKeyDestPriv)
priv = kit->value;
getKVSUInt(kit, kConfigKeyInLength, mSetting.inLength)
getKVSUInt(kit, kConfigKeyInQuantity, mSetting.inQuantity)
getKVSUInt(kit, kConfigKeyInVariance, mSetting.inVariance)
getKVSUInt(kit, kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
getKVSUInt(kit, kConfigKeyOutLength, mSetting.outLength)
getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity)
getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance)
getKVSUInt(kit, kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
#ifdef RS_I2P_SAM3_BOB_COMPAT
// import BOB settings
else if (kit->key == kConfigKeyBOBEnable)
mSetting.enable = kit->value == "TRUE";
else if (kit->key == kConfigKeyBOBKey) {
// don't overwirte, just import when not set already!
if (priv.empty())
priv = kit->value;
}
#endif
else
RS_INFO("unknown key:", kit->key);
}
}
delete vitem;
}
// get the pub key
std::string pub = i2p::publicKeyFromPrivate(priv);
if (pub.empty() || priv.empty())
RS_DBG("no destination to load");
else {
RS_STACK_MUTEX(mLock);
mSetting.address.publicKey = pub;
mSetting.address.privateKey = priv;
mSetting.address.base32 = i2p::keyToBase32Addr(pub);
}
RS_STACK_MUTEX(mLock);
mConfigLoaded = true;
return true;
}
#undef getKVSUInt
bool p3I2pSam3::startSession()
{
RS_DBG4();
constexpr size_t len = 8;
const std::string location = RsRandom::alphaNumeric(len);
const std::string nick = "RetroShare-" + location;
std::vector<std::string> params;
{
RS_STACK_MUTEX(mLock);
// length
params.push_back(i2p::makeOption("inbound.length", mSetting.inLength));
params.push_back(i2p::makeOption("outbound.length", mSetting.outLength));
// variance
params.push_back(i2p::makeOption("inbound.lengthVariance", + mSetting.inVariance));
params.push_back(i2p::makeOption("outbound.lengthVariance", + mSetting.outVariance));
// quantity
params.push_back(i2p::makeOption("inbound.quantity", + mSetting.inQuantity));
params.push_back(i2p::makeOption("outbound.quantity", + mSetting.outQuantity));
// backup quantity
params.push_back(i2p::makeOption("inbound.backupQuantity", + mSetting.inBackupQuantity));
params.push_back(i2p::makeOption("outbound.backupQuantity", + mSetting.outBackupQuantity));
}
std::string paramsStr;
for (auto &&p : params)
paramsStr.append(p + " ");
// keep trailing space for easier extending when necessary
int ret;
if (mSetting.session) {
stopSession();
}
auto session = (Sam3Session*)rs_malloc(sizeof (Sam3Session));
// add nick
paramsStr.append("inbound.nickname=" + nick); // leading space is already there
{
RS_STACK_MUTEX(mLockSam3Access);
if(!mSetting.address.privateKey.empty()) {
RS_DBG3("with destination");
ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, mSetting.address.privateKey.c_str(), Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
} else {
RS_DBG("without destination");
ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, SAM3_DESTINATION_TRANSIENT, Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
}
}
if (ret != 0) {
delete session;
session = nullptr;
return false;
}
#if 0 // this check is useless. For non i2p hidden locations the public key is temporal anyway and for i2p hidden ones, it is part of the (fixed) private key.
if (!mSetting.address.publicKey.empty() && mSetting.address.publicKey != session->pubkey)
// This should be ok for non hidden locations. This should be a problem for hidden i2p locations...
RS_DBG("public key changed! Yet unsure if this is ok or a problem. Should be fine for non i2p hidden locations or clear net.");
#endif
/*
* Note: sam3CreateSession will issue a name looup of "ME" to receive its public key, thus it is always correct.
* No need to use i2p::publicKeyFromPrivate()
*/
RS_STACK_MUTEX(mLock);
mSetting.session = session;
mSetting.address.publicKey = session->pubkey;
mSetting.address.base32 = i2p::keyToBase32Addr(session->pubkey);
// do not overwrite the private key, if any!!
RS_DBG1("nick: ", nick, " address: ", mSetting.address.base32);
RS_DBG2(" myDestination.pub ", mSetting.address.publicKey);
RS_DBG2(" myDestination.priv ", mSetting.address.privateKey);
return true;
}
bool p3I2pSam3::startForwarding()
{
RS_DBG4();
if(mSetting.address.privateKey.empty()) {
RS_DBG3("no private key set");
// IMPORANT: return true here!
// since there is no forward session for non hidden nodes, this funtion is successfull by doing nothing
return true;
}
if (!mSetting.session) {
RS_WARN("no session found!");
return false;
}
peerState ps;
mPeerMgr->getOwnNetStatus(ps);
RS_STACK_MUTEX(mLockSam3Access);
mSetting.session->silent = true;
int ret = sam3StreamForward(mSetting.session, sockaddr_storage_iptostring(ps.localaddr).c_str(), sockaddr_storage_port(ps.localaddr));
if (ret < 0) {
RS_DBG("forward failed, due to", mSetting.session->error);
return false;
}
return true;
}
void p3I2pSam3::stopSession()
{
RS_DBG4();
{
RS_STACK_MUTEX(mLock);
if (!mSetting.session)
return;
// swap connections
mInvalidConnections = mValidConnections;
mValidConnections.clear();
RS_STACK_MUTEX(mLockSam3Access);
sam3CloseSession(mSetting.session);
free(mSetting.session);
mSetting.session = nullptr;
mState = samStatus::samState::offline;
}
// At least i2pd doesn't like to instantaniously stop and (re)start a session, wait here just a little bit.
// Not ideal but does the trick.
// (This happens when using the "restart" button in the settings.)
doSleep(std::chrono::seconds(10));
}
void p3I2pSam3::stopForwarding()
{
// nothing to do here, forwarding is stop when closing the seassion
}
bool p3I2pSam3::generateKey(std::string &pub, std::string &priv)
{
RS_DBG4();
pub.clear();
priv.clear();
// The session is only usef for transporting the data
Sam3Session ss;
if (0 > sam3GenerateKeys(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, Sam3SigType::EdDSA_SHA512_Ed25519)) {
RS_DBG("got error: ", ss.error);
return false;
}
pub = std::string(ss.pubkey);
priv = std::string(ss.privkey);
// sanity check
auto p = i2p::publicKeyFromPrivate(priv);
if (p != pub) {
RS_WARN("public key does not match private key! fixing ...");
pub = p;
}
RS_DBG2("publuc key / address ", pub);
RS_DBG2("private key ", priv);
return true;
}
void p3I2pSam3::lookupKey(taskTicket *ticket)
{
// this can be called independend of the main SAM session!
auto addr = static_cast<i2p::address*>(ticket->data);
if (addr->base32.empty()) {
RS_ERR("lookupKey: called with empty address");
rsAutoProxyMonitor::taskError(ticket);
return;
}
RsThread::async([ticket]()
{
auto addr = static_cast<i2p::address*>(ticket->data);
// The session is only usef for transporting the data
Sam3Session ss;
int ret = sam3NameLookup(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, addr->base32.c_str());
if (ret < 0) {
// get error
RS_DBG("key: ", addr->base32);
RS_DBG("got error: ", ss.error);
rsAutoProxyMonitor::taskError(ticket);
} else {
addr->publicKey = ss.destkey;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
RS_DBG1("success");
}
});
}
void p3I2pSam3::establishConnection(taskTicket *ticket)
{
if (mState != samStatus::samState::online || !mSetting.session) {
RS_WARN("no session found!");
rsAutoProxyMonitor::taskError(ticket);
return;
}
samEstablishConnectionWrapper *wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
if (wrapper->address.publicKey.empty()) {
RS_ERR("no public key given");
rsAutoProxyMonitor::taskError(ticket);
return;
}
RsThread::async([ticket, this]() {
auto wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
struct Sam3Connection *connection;
{
auto l = this->mLockSam3Access;
RS_STACK_MUTEX(l);
mSetting.session->silent = false;
connection = sam3StreamConnect(this->mSetting.session, wrapper->address.publicKey.c_str());
}
if (!connection) {
// get error
RS_DBG("got error:", this->mSetting.session->error);
rsAutoProxyMonitor::taskError(ticket);
} else {
wrapper->connection = connection;
{
auto l = this->mLockSam3Access;
RS_STACK_MUTEX(l);
this->mValidConnections.push_back(connection);
}
RS_DBG1("success");
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
}
});
}
void p3I2pSam3::closeConnection(taskTicket *ticket)
{
Sam3Connection *conn = static_cast<Sam3Connection*>(ticket->data);
if (mState == samStatus::samState::offline || !mSetting.session) {
// no session found, sam was likely stopped
RS_DBG2("no session found");
auto it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
if (it != mInvalidConnections.end()) {
// this is the expected case
mInvalidConnections.erase(it);
} else {
// this is unexpected but not a big deal, just warn
RS_WARN("cannot find connection in mInvalidConnections");
it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
if (it != mValidConnections.end()) {
mValidConnections.erase(it);
// now it is getting even weirder, still not a big deal, just warn
RS_WARN("found connection in mValidConnections");
}
}
// when libsam3 has already handled closing of the connection - which should be the case here - the memory has been freed already (-> pointer is invalid)
conn = nullptr;
} else {
RS_STACK_MUTEX(mLock);
bool callClose = true;
// search in current connections
auto it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
if (it != mValidConnections.end()) {
RS_DBG2("found valid connection");
mValidConnections.erase(it);
} else {
// search in old connections
it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
if (it != mInvalidConnections.end()) {
// old connection, just ignore. *should* be freed already
mInvalidConnections.erase(it);
RS_DBG2("found old (invalid) connection");
callClose = false;
conn = nullptr;
} else {
// weird
RS_WARN("could'n find connection!");
// best thing we can do here
callClose = false;
conn = nullptr;
}
}
if (callClose) {
RS_DBG2("closing connection");
RS_STACK_MUTEX(mLockSam3Access);
sam3CloseConnection(conn);
conn = nullptr; // freed by above call
}
}
if (conn) {
free(conn);
conn = nullptr;
}
ticket->data = nullptr;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
return;
}
void p3I2pSam3::updateSettings_locked()
{
RS_DBG4();
IndicateConfigChanged();
#if 0 // TODO recreat session when active, can we just recreat it?
if (mSs) {
stopSession();
startSession();
}
#endif
}

110
libretroshare/src/services/autoproxy/p3i2psam3.h Normal file
View File

@ -0,0 +1,110 @@
#ifndef P3I2PSAM3_H
#define P3I2PSAM3_H
#include <queue>
#include <list>
#include "services/autoproxy/rsautoproxymonitor.h"
#include "pqi/p3cfgmgr.h"
#include "util/i2pcommon.h"
#include "util/rsthreads.h"
/*
* This class implements I2P SAMv3 (Simple Anonymous Messaging) to allow RS
* to automatically setup tunnel to and from I2P.
* SAMv3 is a simple text-based interface: https://geti2p.net/de/docs/api/samv3
*
* For the actual SAM commands / low level stuff libsam3 (https://github.com/i2p/libsam3)
* is used with some minor adjustments, for exmaple, the FORWARD session is always silent.
*
* SAM in a nutshell works like this:
* 1) setup main/control session which configures everything (destination ID, tunnel number, hops number, and so on)
* 2) setup a forward session, so that I2P will establish a connection to RS for each incoming connection to our i2p destination
* 3a) query/lookup the destination (public key) for a given i2p address
* 3b) connect to the given destination
*
* An established connection (both incoming or outgoing) are then handed over to RS.
* The lifetime of a session (and its subordinates connections) is bound to their tcp socket. When the socket closes, the session is closed, too.
*
*/
class p3PeerMgr;
class Sam3Session;
class Sam3Connection;
typedef Sam3Session samSession;
struct samSettings : i2p::settings {
samSession *session;
};
struct samEstablishConnectionWrapper {
i2p::address address;
Sam3Connection *connection;
};
struct samStatus {
std::string sessionName;
enum samState {
offline,
connectSession,
connectForward,
online
} state; // the name is kinda redundant ...
};
class p3I2pSam3 : public RsTickingThread, public p3Config, public autoProxyService
{
public:
p3I2pSam3(p3PeerMgr *peerMgr);
// autoProxyService interface
public:
bool isEnabled();
bool initialSetup(std::string &addr, uint16_t &port);
void processTaskAsync(taskTicket *ticket);
void processTaskSync(taskTicket *ticket);
// RsTickingThread interface
public:
void threadTick(); /// @see RsTickingThread
// p3Config interface
protected:
RsSerialiser *setupSerialiser();
bool saveList(bool &cleanup, std::list<RsItem *> &);
bool loadList(std::list<RsItem *> &load);
private:
bool startSession();
bool startForwarding();
void stopSession();
void stopForwarding();
bool generateKey(std::string &pub, std::string &priv);
void lookupKey(taskTicket *ticket);
void establishConnection(taskTicket *ticket);
void closeConnection(taskTicket *ticket);
void updateSettings_locked();
bool mConfigLoaded;
samSettings mSetting;
p3PeerMgr *mPeerMgr;
std::queue<taskTicket *> mPending;
// Used to report the state to the gui
// (Since the create session call/will can block and there is no easy way from outside the main thread to see
// what is going on, it is easier to store the current state in an extra variable independen from the main thread)
samStatus::samState mState;
// used to keep track of connections, libsam3 does it internally but it can be unreliable since pointers are shared
std::list<Sam3Connection *> mValidConnections, mInvalidConnections;
// mutex
RsMutex mLock;
RsMutex mLockSam3Access; // libsam3 is not thread safe! (except for key lookup)
};
#endif // P3I2PSAM3_H

20
libretroshare/src/services/autoproxy/rsautoproxymonitor.cc
View File

@ -329,14 +329,22 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType
bool rsAutoProxyMonitor::isAsyncTask(autoProxyTask::autoProxyTask_enum t)
{
// Explicit list all values, so that missing ones will be detected by the compiler.
switch (t) {
case autoProxyTask::start:
case autoProxyTask::stop:
case autoProxyTask::receiveKey:
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::stop: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
return true;
break;
default:
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig: [[fallthrough]];
case autoProxyTask::proxyStatusCheck:
return false;
}
return false;
}

6
libretroshare/src/services/autoproxy/rsautoproxymonitor.h
View File

@ -31,7 +31,8 @@ class autoProxyCallback;
namespace autoProxyType {
enum autoProxyType_enum {
I2PBOB
// I2PBOB,
I2PSAM3
};
}
@ -41,7 +42,10 @@ namespace autoProxyTask {
start, ///< start up proxy
stop, ///< shut down proxy
receiveKey, ///< renew proxy key (if any)
lookupKey, ///< look up a base32 addr
proxyStatusCheck, ///< use to check if the proxy is still running
establishConnection, ///< create a connection to a given public key or base32 address
closeConnection, ///< closes a connection
/* sync tasks */
status, ///< get status from auto proxy
getSettings, ///< get setting from auto proxy

524
libretroshare/src/services/p3gxsforums.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
mGenActive(false), mGenCount(0),
mKnownForumsMutex("GXS forums known forums timestamp cache")
#ifdef RS_DEEP_FORUMS_INDEX
, mDeepIndex(DeepForumsIndex::dbDefaultPath())
#endif
{
// Test Data disabled in Repo.
//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@ -192,158 +196,197 @@ RsSerialiser* p3GxsForums::setupSerialiser()
void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
{
#ifdef GXSFORUMS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
RS_DBG2(changes.size(), " changes to notify");
for(RsGxsNotify* gxsChange: changes)
{
// Let the compiler delete the change for us
std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
switch(gxsChange->getType())
{
case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
case RsGxsNotify::TYPE_PUBLISHED:
{
auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
if(msgChange) /* Message received */
{
uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(msgSubtype))
{
case RsGxsForumsItems::MESSAGE_ITEM:
{
auto newForumMessageItem =
dynamic_cast<RsGxsForumMsgItem*>(
msgChange->mNewMsgItem );
if(!newForumMessageItem)
{
RS_ERR("Received message change with mNewMsgItem type "
"mismatching or null");
print_stacktrace();
return;
}
#ifdef RS_DEEP_FORUMS_INDEX
RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
tmpPost.mMeta = newForumMessageItem->meta;
mDeepIndex.indexForumPost(tmpPost);
#endif
std::vector<RsGxsNotify *>::iterator it;
for(it = changes.begin(); it != changes.end(); ++it)
{
RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
if (msgChange)
{
if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
if (rsEvents)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumMsgId = msgChange->mMsgId;
ev->mForumGroupId = msgChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
rsEvents->postEvent(ev);
}
#ifdef NOT_USED_YET
if (!msgChange->metaChange())
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
std::cerr << std::endl;
#endif
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
std::cerr << std::endl;
#endif
bool enabled = false;
if (autoDownloadEnabled(mit->first, enabled) && enabled)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
std::cerr << std::endl;
#endif
/* problem is most of these will be comments and votes,
* should make it occasional - every 5mins / 10minutes TODO */
unprocessedGroups.push_back(mit->first);
}
}
}
#endif
}
else
{
if (rsEvents)
{
RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
if (grpChange)
{
switch (grpChange->getType())
{
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
}
default:
RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
break;
}
}
case RsGxsNotify::TYPE_PUBLISHED:
case RsGxsNotify::TYPE_RECEIVED_NEW:
auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
if(groupChange) /* Group received */
{
/* group received */
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
mKnownForums[grpChange->mGroupId] = time(nullptr);
unknown = ( mKnownForums.find(gxsChange->mGroupId)
== mKnownForums.end() );
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
rsEvents->postEvent(ev);
}
else
RsInfo() << __PRETTY_FUNCTION__
<< " Not notifying already known forum "
<< grpChange->mGroupId << std::endl;
#ifdef RS_DEEP_FORUMS_INDEX
uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(itemType))
{
case RsGxsForumsItems::GROUP_ITEM:
{
auto newForumGroupItem =
static_cast<RsGxsForumGroupItem*>(
groupChange->mNewGroupItem );
mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
break;
}
default:
RS_WARN("Got unknown gxs group subtype: ", itemType);
break;
}
#endif // def RS_DEEP_FORUMS_INDEX
}
break;
case RsGxsNotify::TYPE_GROUP_DELETED:
}
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_MESSAGE_DELETED:
{
auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
if(!delChange)
{
RS_ERR( "Got mismatching notification type: ",
gxsChange->getType() );
print_stacktrace();
break;
}
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumPostFromIndex(
delChange->mGroupId, delChange->messageId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumEventCode = RsForumEventCode::DELETED_POST;
ev->mForumGroupId = delChange->mGroupId;
ev->mForumMsgId = delChange->messageId;
break;
}
case RsGxsNotify::TYPE_GROUP_DELETED:
{
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
}
break;
}
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[grpChange->mGroupId] = time(nullptr);
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
break;
}
case RsGxsNotify::TYPE_UPDATED:
{
// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
/* Happens when the group data has changed. In this case we need to
* analyse the old and new group in order to detect possible
* notifications for clients */
RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
RsGxsForumGroupItem* old_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem* new_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
{
RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
delete grpChange;
continue;
RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
"mNewGroup not of type RsGxsForumGroupItem or NULL. "
"This is inconsistent!");
print_stacktrace();
break;
}
// First of all, we check if there is a difference between the old and new list of moderators
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
#endif
/* First of all, we check if there is a difference between the old
* and new list of moderators */
std::list<RsGxsId> added_mods, removed_mods;
for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== old_forum_grp_item->mGroup.mAdminList.ids.end() )
added_mods.push_back(gxs_id);
for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== new_forum_grp_item->mGroup.mAdminList.ids.end() )
removed_mods.push_back(gxs_id);
if(!added_mods.empty() || !removed_mods.empty())
@ -359,24 +402,23 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
}
// check the list of pinned posts
std::list<RsGxsMessageId> added_pins, removed_pins;
for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
added_pins.push_back(msg_id);
for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
removed_pins.push_back(msg_id);
if(!added_pins.empty() || !removed_pins.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
rsEvents->postEvent(ev);
}
@ -384,31 +426,24 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
|| old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName
|| old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
|| old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId
)
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId )
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
rsEvents->postEvent(ev);
}
}
break;
break;
}
default:
RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
" Currently not handled." );
break;
}
}
}
}
/* shouldn't need to worry about groups - as they need to be subscribed to */
delete *it;
}
}
void p3GxsForums::service_tick()
{
@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
| GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
/* This flag have always this value even for circle restricted forums due to
* how GXS distribute/verify groups */
forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
forum.mMeta.mCircleId.clear();
@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
id == mMeta.mAuthorId;
}
std::error_condition p3GxsForums::getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries )
{
uint32_t token;
RsTokReqOptions opts;
opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
GxsMsgReq msgReq;
msgReq[forumId] = contentIds;
if(!requestMsgInfo(token, opts, msgReq))
{
RS_ERR("requestMsgInfo failed");
return std::errc::invalid_argument;
}
switch(waitToken(token, std::chrono::seconds(5)))
{
case RsTokenService::COMPLETE:
{
GxsMsgMetaMap metaMap;
if(!RsGenExchange::getMsgMeta(token, metaMap))
return std::errc::result_out_of_range;
summaries = metaMap[forumId];
return std::error_condition();
}
case RsTokenService::PARTIAL: // [[fallthrough]];
case RsTokenService::PENDING:
return std::errc::timed_out;
default:
return std::errc::not_supported;
}
}
#ifdef RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG1("");
RsGxsForumsSearchRequest request;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(request);
}
if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(request.mType) );
return std::errc::bad_message;
}
RsGxsForumsSearchReply reply;
auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
if(mErr || reply.mResults.empty()) return mErr;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(reply);
resultSize = ctx.mOffset;
}
resultData = rs_malloc<uint8_t>(resultSize);
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(reply);
return std::error_condition();
}
std::error_condition p3GxsForums::distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId )
{
RsGxsForumsSearchRequest request;
request.mQuery = matchString;
uint32_t requestSize;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(request);
requestSize = ctx.mOffset;
}
std::error_condition ec;
auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
if(!requestData) return ec;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(request);
}
return netService()->distantSearchRequest(
requestData, requestSize,
static_cast<RsServiceType>(serviceType()), searchId );
}
std::error_condition p3GxsForums::localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults )
{ return prepareSearchResults(matchString, false, searchResults); }
std::error_condition p3GxsForums::prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults )
{
std::vector<DeepForumsSearchResult> results;
auto mErr = mDeepIndex.search(matchString, results);
if(mErr) return mErr;
searchResults.clear();
for(auto uRes: results)
{
RsUrl resUrl(uRes.mUrl);
const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
if(!forumIdStr)
{
RS_ERR( "Forum URL retrieved from deep index miss ID. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::address_not_available;
}
std::vector<RsGxsForumGroup> forumsInfo;
RsGxsGroupId forumId(*forumIdStr);
if(forumId.isNull())
{
RS_ERR( "Forum ID retrieved from deep index is invalid. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
forumsInfo.empty() )
{
RS_ERR( "Forum just parsed from deep index link not found. "
"Should never happen! ", forumId, " ", uRes.mUrl );
print_stacktrace();
return std::errc::identifier_removed;
}
RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
// Avoid leaking sensitive information to unkown peers
if( publicOnly &&
( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
RsGxsCircleType::PUBLIC ) ) continue;
RsGxsSearchResult res;
res.mGroupId = forumId;
res.mGroupName = fMeta.mGroupName;
res.mAuthorId = fMeta.mAuthorId;
res.mPublishTs = fMeta.mPublishTs;
res.mSearchContext = uRes.mSnippet;
auto postIdStr =
resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
if(postIdStr)
{
RsGxsMessageId msgId(*postIdStr);
if(msgId.isNull())
{
RS_ERR( "Post just parsed from deep index link is invalid. "
"Should never happen! ", postIdStr, " ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
std::vector<RsMsgMetaData> msgSummaries;
auto errc = getContentSummaries(
forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
if(errc) return errc;
if(msgSummaries.size() != 1)
{
RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
"should never happen!" );
return std::errc::result_out_of_range;
}
RsMsgMetaData& msgMeta(msgSummaries[0]);
res.mMsgId = msgMeta.mMsgId;
res.mMsgName = msgMeta.mMsgName;
res.mAuthorId = msgMeta.mAuthorId;
}
RS_DBG4(res);
searchResults.push_back(res);
}
return std::error_condition();
}
std::error_condition p3GxsForums::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RsGxsForumsSearchReply reply;
{
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(reply);
}
free(resultData);
if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(reply.mType) );
return std::errc::bad_message;
}
auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
event->mSearchId = requestId;
event->mSearchResults = reply.mResults;
rsEvents->postEvent(event);
return std::error_condition();
}
#else // def RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::distantSearchRequest(
const std::string&, TurtleRequestId& )
{ return std::errc::function_not_supported; }
std::error_condition p3GxsForums::localSearch(
const std::string&,
std::vector<RsGxsSearchResult>& )
{ return std::errc::function_not_supported; }
#endif // def RS_DEEP_FORUMS_INDEX
/*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
"retroshare:///forums";
/*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

50
libretroshare/src/services/p3gxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -32,6 +32,10 @@
#include "util/rstickevent.h"
#include "util/rsdebug.h"
#ifdef RS_DEEP_FORUMS_INDEX
#include "deep_search/forumsindex.hpp"
#endif
class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
public RsTickEvent /* only needed for testing - remove after */
@ -144,6 +148,33 @@ public:
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) override;
/// @see RsGxsForums
std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override;
/// @see RsGxsForums
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
/// @see RsGxsForums
std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) override;
#ifdef RS_DEEP_FORUMS_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNxsObserver
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// implementation of rsGxsGorums
///
bool getGroupData(const uint32_t &token, std::vector<RsGxsForumGroup> &groups) override;
@ -155,6 +186,17 @@ public:
bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
protected:
#ifdef RS_DEEP_FORUMS_INDEX
/** Internal usage
* @param[in] publicOnly if true is passed only results pertaining to
* publicly shared forums are returned
*/
std::error_condition prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults );
#endif //def RS_DEEP_FORUMS_INDEX
private:
static uint32_t forumsAuthenPolicy();
@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
std::map<RsGxsGroupId,rstime_t> mKnownForums ;
RsMutex mKnownForumsMutex;
#ifdef RS_DEEP_FORUMS_INDEX
DeepForumsIndex mDeepIndex;
#endif
};

33
libretroshare/src/turtle/p3turtle.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2009-2018 by Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2009-2018 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -26,6 +28,10 @@
#include <stdexcept>
#include <stdlib.h>
#include <assert.h>
#include <iostream>
#include <cerrno>
#include <cmath>
#include <cstdio>
#include "rsserver/p3face.h"
#include "crypto/rscrypto.h"
@ -39,13 +45,7 @@
#include "ft/ftcontroller.h"
#include "p3turtle.h"
#include <iostream>
#include <errno.h>
#include <cmath>
#include <stdio.h>
#include "util/cxx17retrocompat.h"
#include "util/rsdebug.h"
#include "util/rsprint.h"
#include "util/rsrandom.h"
@ -1975,7 +1975,8 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
//
void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
void RsTurtleStringSearchRequestItem::search(
std::list<TurtleFileInfo>& result ) const
{
/* call to core */
std::list<DirDetails> initialResults;
@ -1988,17 +1989,19 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
std::cerr << "Performing rsFiles->search()" << std::endl ;
#endif
// now, search!
rsFiles->SearchKeywords(words, initialResults,RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE,PeerId());
rsFiles->SearchKeywords(
words, initialResults,
RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE, PeerId() );
#ifdef P3TURTLE_DEBUG
std::cerr << initialResults.size() << " matches found." << std::endl ;
#endif
result.clear() ;
for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
for(auto& it: std::as_const(initialResults))
{
// retain only file type
if (it->type == DIR_TYPE_DIR)
if (it.type == DIR_TYPE_DIR)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Skipping directory " << it->name << std::endl ;
@ -2007,9 +2010,9 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
}
TurtleFileInfo i;
i.hash = it->hash ;
i.size = it->size ;
i.name = it->name ;
i.hash = it.hash;
i.size = it.size;
i.name = it.name;
result.push_back(i);
}

53
libretroshare/src/turtle/turtleclientservice.h
View File

@ -19,23 +19,25 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
// This class is the parent class for any service that will use the turtle router to distribute its packets.
// Typical representative clients include:
//
// p3ChatService: opens tunnels to distant peers for chatting
// ftServer: searches and open tunnels to distant sources for file transfer
//
#pragma once
#include <string>
#include <stdlib.h>
#include <serialiser/rsserial.h>
#include <turtle/rsturtleitem.h>
#include "serialiser/rsserial.h"
#include "turtle/rsturtleitem.h"
#include "util/rsdebug.h"
struct RsItem;
class p3turtle ;
/** This class is the parent class for any service that will use the turtle
* router to distribute its packets.
* Typical representative clients include:
* p3ChatService: opens tunnels to distant peers for chatting
* ftServer: searches and open tunnels to distant sources for file
* transfer
*/
class RsTurtleClientService
{
public:
@ -88,27 +90,32 @@ class RsTurtleClientService
}
/*!
* \brief receiveSearchRequest
* This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
* result contains the serialised generic result returned by the client.
*
* This method is called by the turtle router to notify the client of a
* search request in the form generic data.
* The returned result contains the serialised generic result returned by the
* client service.
* The turtle router keeps the memory ownership over search_request_data
*
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and forwarded
*
* \return true if the search is successful.
* \param max_allowed_hits max number of hits allowed to be sent back and
* forwarded
* \return true if matching results are available, false otherwise.
*/
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
uint32_t /*search_request_data_len*/,
unsigned char *& /*search_result_data*/,
uint32_t& /*search_result_data_len*/,
uint32_t& /* max_allows_hits */)
virtual bool receiveSearchRequest(
unsigned char *search_request_data, uint32_t search_request_data_len,
unsigned char *& search_result_data, uint32_t& search_result_data_len,
uint32_t& max_allows_hits )
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
/* Suppress unused warning this way and not commenting the param names
* so doxygen match documentation against params */
(void) search_request_data; (void) search_request_data_len;
(void) search_result_data; (void) search_result_data_len;
(void) max_allows_hits;
RS_WARN( "Received search request from turtle router, but the client "
"is not handling it!" );
return false;
}

9
libretroshare/src/use_libretroshare.pri
View File

@ -90,6 +90,15 @@ rs_broadcast_discovery {
win32-g++|win32-clang-g++:dLibs *= wsock32
}
rs_sam3_libsam3 {
LIBSAM3_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/libsam3/)
LIBSAM3_BUILD_PATH=$$clean_path($${RS_BUILD_PATH}/supportlibs/libsam3/)
INCLUDEPATH *= $$clean_path($${LIBSAM3_SRC_PATH}/src/libsam3/)
DEPENDPATH *= $$clean_path($${LIBSAM3_BUILD_PATH})
QMAKE_LIBDIR *= $$clean_path($${LIBSAM3_BUILD_PATH})
LIBS *= -L$$clean_path($${LIBSAM3_BUILD_PATH}) -lsam3
}
static {
sLibs *= $$mLibs
} else {

2
libretroshare/src/util/dnsresolver.cc
View File

@ -41,8 +41,6 @@
const rstime_t MAX_TIME_BEFORE_RETRY = 300 ; /* seconds before retrying an ip address */
const rstime_t MAX_KEEP_DNS_ENTRY = 3600 ; /* seconds during which a DNS entry is considered valid */
static const std::string ADDR_AGENT = "Mozilla/5.0";
void *solveDNSEntries(void *p)
{
bool more_to_go = true ;

355
libretroshare/src/util/extaddrfinder.cc
View File

@ -19,9 +19,13 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
//#define EXTADDRSEARCH_DEBUG
#include "extaddrfinder.h"
#include "pqi/pqinetwork.h"
#include "rsdebug.h"
#include "util/rsstring.h"
#include "util/rsmemory.h"
@ -38,286 +42,237 @@
#include <stdio.h>
#include "util/rstime.h"
#include <map>
const uint32_t MAX_IP_STORE = 300; /* seconds ip address timeout */
//#define EXTADDRSEARCH_DEBUG
static const std::string ADDR_AGENT = "Mozilla/5.0";
static std::string scan_ip(const std::string& text)
class ZeroInt
{
std::set<unsigned char> digits ;
digits.insert('0') ; digits.insert('3') ; digits.insert('6') ;
digits.insert('1') ; digits.insert('4') ; digits.insert('7') ;
digits.insert('2') ; digits.insert('5') ; digits.insert('8') ;
digits.insert('9') ;
public:
ZeroInt() : n(0) {}
uint32_t n ;
};
for(int i=0;i<(int)text.size();++i)
{
while(i < (int)text.size() && digits.find(text[i])==digits.end()) ++i ;
if(i>=(int)text.size())
return "" ;
unsigned int a,b,c,d ;
if(sscanf(text.c_str()+i,"%u.%u.%u.%u",&a,&b,&c,&d) != 4)
continue ;
if(a < 256 && b<256 && c<256 && d<256)
{
std::string s ;
rs_sprintf(s, "%u.%u.%u.%u", a, b, c, d) ;
return s;
}
}
return "" ;
}
static void getPage(const std::string& server_name,std::string& page)
{
page = "" ;
int sockfd,n=0; // socket descriptor
struct sockaddr_in serveur; // server's parameters
memset(&serveur.sin_zero, 0, sizeof(serveur.sin_zero));
char buf[1024];
char request[1024];
#ifdef EXTADDRSEARCH_DEBUG
std::cout << "ExtAddrFinder: connecting to " << server_name << std::endl ;
#endif
// socket creation
sockfd = unix_socket(PF_INET,SOCK_STREAM,0);
if (sockfd < 0)
{
std::cerr << "ExtAddrFinder: Failed to create socket" << std::endl;
return ;
}
serveur.sin_family = AF_INET;
// get server's ipv4 adress
in_addr in ;
if(!rsGetHostByName(server_name.c_str(),in)) /* l'hôte n'existe pas */
{
std::cerr << "ExtAddrFinder: Unknown host " << server_name << std::endl;
unix_close(sockfd);
return ;
}
serveur.sin_addr = in ;
serveur.sin_port = htons(80);
#ifdef EXTADDRSEARCH_DEBUG
printf("Connection attempt\n");
#endif
std::cerr << "ExtAddrFinder: resolved hostname " << server_name << " to " << rs_inet_ntoa(in) << std::endl;
sockaddr_storage server;
sockaddr_storage_setipv4(server, &serveur);
sockaddr_storage_setport(server, 80);
if(unix_connect(sockfd, server) == -1)
{
std::cerr << "ExtAddrFinder: Connection error to " << server_name << std::endl ;
unix_close(sockfd);
return ;
}
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Connection established to " << server_name << std::endl ;
#endif
// envoi
if(snprintf( request,
1024,
"GET / HTTP/1.0\r\n"
"Host: %s:%d\r\n"
"Connection: Close\r\n"
"\r\n",
server_name.c_str(), 80) > 1020)
{
std::cerr << "ExtAddrFinder: buffer overrun. The server name \"" << server_name << "\" is too long. This is quite unexpected." << std::endl;
unix_close(sockfd);
return ;
}
if(send(sockfd,request,strlen(request),0)== -1)
{
std::cerr << "ExtAddrFinder: Could not send request to " << server_name << std::endl ;
unix_close(sockfd);
return ;
}
// recéption
while((n = recv(sockfd, buf, sizeof buf - 1, 0)) > 0)
{
buf[n] = '\0';
page += std::string(buf,n) ;
}
// fermeture de la socket
unix_close(sockfd);
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Got full page from " << server_name << std::endl ;
#endif
}
void* doExtAddrSearch(void *p)
void ExtAddrFinder::run()
{
std::vector<std::string> res ;
ExtAddrFinder *af = (ExtAddrFinder*)p ;
for(std::list<std::string>::const_iterator it(af->_ip_servers.begin());it!=af->_ip_servers.end();++it)
for(auto& it : _ip_servers)
{
std::string page ;
getPage(*it,page) ;
std::string ip = scan_ip(page) ;
std::string ip = "";
rsGetHostByNameSpecDNS(it,"myip.opendns.com",ip,2);
if(ip != "")
res.push_back(ip) ;
#ifdef EXTADDRSEARCH_DEBUG
std::cout << "ip found through " << *it << ": \"" << ip << "\"" << std::endl ;
RS_DBG("ip found through DNS ", it, ": \"", ip, "\"");
#endif
}
if(res.empty())
{
// thread safe copy results.
//
reset();
return ;
}
std::map<sockaddr_storage,ZeroInt> addrV4_votes;
std::map<sockaddr_storage,ZeroInt> addrV6_votes;
std::string all_addrV4_Found;
std::string all_addrV6_Found;
for(auto curRes : res)
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = false ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
}
return NULL ;
}
sort(res.begin(),res.end()) ; // eliminates outliers.
if(!sockaddr_storage_ipv4_aton(af->mAddr, res[res.size()/2].c_str()))
sockaddr_storage addr;
sockaddr_storage_clear(addr);
bool validIP = sockaddr_storage_inet_pton(addr, curRes)
&& sockaddr_storage_isValidNet(addr);
bool isIPv4 = sockaddr_storage_ipv6_to_ipv4(addr);
if( validIP && isIPv4 )
{
std::cerr << "ExtAddrFinder: Could not convert " << res[res.size()/2] << " into an address." << std::endl ;
addr.ss_family = AF_INET;
addrV4_votes[addr].n++ ;
all_addrV4_Found += sockaddr_storage_tostring(addr) + "\n";
}
else if( validIP && !isIPv4)
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = false ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
addr.ss_family = AF_INET6;
addrV6_votes[addr].n++ ;
all_addrV6_Found += sockaddr_storage_tostring(addr) + "\n";
}
return NULL ;
else
RS_ERR("Invalid addresse reported: ", curRes) ;
}
if( (0 == addrV4_votes.size()) && (0 == addrV6_votes.size()) )
{
RS_ERR("Could not find any external address.");
reset();
return ;
}
if( 1 < addrV4_votes.size() )
RS_ERR("Multiple external IPv4 addresses reported: "
, all_addrV4_Found ) ;
if( 1 < addrV6_votes.size() )
RS_ERR("Multiple external IPv6 addresses reported: "
, all_addrV6_Found ) ;
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = true ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
RS_STACK_MUTEX(mAddrMtx);
mSearching = false ;
mFoundTS = time(NULL) ;
// Only save more reported address if not only once.
uint32_t admax = 0 ;
sockaddr_storage_clear(mAddrV4);
for (auto it : addrV4_votes)
if (admax < it.second.n)
{
mAddrV4 = it.first ;
mFoundV4 = true ;
admax = it.second.n ;
}
return NULL ;
admax = 0 ;
sockaddr_storage_clear(mAddrV6);
for (auto it : addrV6_votes)
if (admax < it.second.n)
{
mAddrV6 = it.first ;
mFoundV6 = true ;
admax = it.second.n ;
}
}
return ;
}
void ExtAddrFinder::start_request()
{
void *data = (void *)this;
pthread_t tid ;
if(! pthread_create(&tid, 0, &doExtAddrSearch, data))
pthread_detach(tid); /* so memory is reclaimed in linux */
else
std::cerr << "(EE) Could not start ExtAddrFinder thread." << std::endl;
if (!isRunning())
start("ExtAddrFinder");
}
bool ExtAddrFinder::hasValidIP(struct sockaddr_storage &addr)
bool ExtAddrFinder::hasValidIPV4(struct sockaddr_storage &addr)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Getting ip." << std::endl ;
RS_DBG("Getting ip.");
#endif
{
RsStackMutex mut(mAddrMtx) ;
if(mFound)
RS_STACK_MUTEX(mAddrMtx) ;
if(mFoundV4)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Has stored ip: responding with this ip." << std::endl ;
RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV4)) ;
#endif
sockaddr_storage_copyip(addr,mAddr); // just copy the IP so we dont erase the port.
sockaddr_storage_copyip(addr,mAddrV4); // just copy the IP so we dont erase the port.
}
}
rstime_t delta;
testTimeOut();
RS_STACK_MUTEX(mAddrMtx) ;
return mFoundV4;
}
bool ExtAddrFinder::hasValidIPV6(struct sockaddr_storage &addr)
{
RsStackMutex mut(mAddrMtx) ;
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("Getting ip.");
#endif
{
RS_STACK_MUTEX(mAddrMtx) ;
if(mFoundV6)
{
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV6)) ;
#endif
sockaddr_storage_copyip(addr,mAddrV6); // just copy the IP so we dont erase the port.
}
}
testTimeOut();
RS_STACK_MUTEX(mAddrMtx) ;
return mFoundV6;
}
void ExtAddrFinder::testTimeOut()
{
bool timeOut;
{
RS_STACK_MUTEX(mAddrMtx) ;
//timeout the current ip
delta = time(NULL) - mFoundTS;
timeOut = (mFoundTS + MAX_IP_STORE < time(NULL));
}
if((uint32_t)delta > MAX_IP_STORE) {//launch a research
if(timeOut || mFirstTime) {//launch a research
if( mAddrMtx.trylock())
{
if(!mSearching)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: No stored ip: Initiating new search." << std::endl ;
RS_DBG("No stored ip: Initiating new search.");
#endif
mSearching = true ;
start_request() ;
}
#ifdef EXTADDRSEARCH_DEBUG
else
std::cerr << "ExtAddrFinder: Already searching." << std::endl ;
RS_DBG("Already searching.");
#endif
mFirstTime = false;
mAddrMtx.unlock();
}
#ifdef EXTADDRSEARCH_DEBUG
else
std::cerr << "ExtAddrFinder: (Note) Could not acquire lock. Busy." << std::endl ;
RS_DBG("(Note) Could not acquire lock. Busy.");
#endif
}
RsStackMutex mut(mAddrMtx) ;
return mFound ;
}
void ExtAddrFinder::reset()
void ExtAddrFinder::reset(bool firstTime /*=false*/)
{
RsStackMutex mut(mAddrMtx) ;
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("firstTime=", firstTime);
#endif
RS_STACK_MUTEX(mAddrMtx) ;
mFound = false ;
mSearching = false ;
mFoundTS = time(NULL) - MAX_IP_STORE;
mFoundV4 = false ;
mFoundV6 = false ;
mFirstTime = firstTime;
mFoundTS = time(nullptr);
sockaddr_storage_clear(mAddrV4);
sockaddr_storage_clear(mAddrV6);
}
ExtAddrFinder::~ExtAddrFinder()
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Deleting ExtAddrFinder." << std::endl ;
RS_DBG("Deleting ExtAddrFinder.");
#endif
}
ExtAddrFinder::ExtAddrFinder() : mAddrMtx("ExtAddrFinder")
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Creating new ExtAddrFinder." << std::endl ;
RS_DBG("Creating new ExtAddrFinder.");
#endif
RsStackMutex mut(mAddrMtx) ;
reset( true );
mFound = false;
mSearching = false;
mFoundTS = time(NULL) - MAX_IP_STORE;
sockaddr_storage_clear(mAddr);
_ip_servers.push_back(std::string( "checkip.dyndns.org" )) ;
_ip_servers.push_back(std::string( "www.myip.dk" )) ;
_ip_servers.push_back(std::string( "showip.net" )) ;
_ip_servers.push_back(std::string( "www.displaymyip.com")) ;
//https://unix.stackexchange.com/questions/22615/how-can-i-get-my-external-ip-address-in-a-shell-script
//Enter direct ip so local DNS cannot change it.
//DNS servers must recognize "myip.opendns.com"
_ip_servers.push_back(std::string( "208.67.222.222" )) ;//resolver1.opendns.com
_ip_servers.push_back(std::string( "208.67.220.220" )) ;//resolver2.opendns.com
_ip_servers.push_back(std::string( "208.67.222.220" )) ;//resolver3.opendns.com
_ip_servers.push_back(std::string( "208.67.220.222" )) ;//resolver4.opendns.com
_ip_servers.push_back(std::string( "2620:119:35::35" )) ;//resolver1.opendns.com
_ip_servers.push_back(std::string( "2620:119:53::53" )) ;//resolver2.opendns.com
}

19
libretroshare/src/util/extaddrfinder.h
View File

@ -30,26 +30,31 @@
struct sockaddr ;
class ExtAddrFinder
class ExtAddrFinder: public RsThread
{
public:
ExtAddrFinder() ;
~ExtAddrFinder() ;
bool hasValidIP(struct sockaddr_storage &addr) ;
bool hasValidIPV4(struct sockaddr_storage &addr) ;
bool hasValidIPV6(struct sockaddr_storage &addr) ;
void getIPServersList(std::list<std::string>& ip_servers) { ip_servers = _ip_servers ; }
void start_request() ;
void reset() ;
void reset(bool firstTime = false) ;
private:
friend void* doExtAddrSearch(void *p) ;
virtual void run();
void testTimeOut();
RsMutex mAddrMtx;
rstime_t mFoundTS;
struct sockaddr_storage mAddr;
bool mFound ;
bool mSearching;
bool mFoundV4;
bool mFoundV6;
bool mFirstTime;
rstime_t mFoundTS;
struct sockaddr_storage mAddrV4;
struct sockaddr_storage mAddrV6;
std::list<std::string> _ip_servers;
};

7
libretroshare/src/util/folderiterator.cc
View File

@ -38,8 +38,11 @@
namespace librs { namespace util {
FolderIterator::FolderIterator(const std::string& folderName, bool allow_symlinks, bool allow_files_from_the_future)
: mFolderName(folderName),mAllowSymLinks(allow_symlinks),mAllowFilesFromTheFuture(allow_files_from_the_future)
FolderIterator::FolderIterator(
const std::string& folderName, bool allow_symlinks,
bool allow_files_from_the_future ):
mFolderName(folderName), mAllowSymLinks(allow_symlinks),
mAllowFilesFromTheFuture(allow_files_from_the_future)
{
is_open = false ;
validity = false ;

4
libretroshare/src/util/folderiterator.h
View File

@ -44,7 +44,9 @@ namespace librs { namespace util {
class FolderIterator
{
public:
FolderIterator(const std::string& folderName,bool allow_symlinks,bool allow_files_from_the_future = true);
FolderIterator(
const std::string& folderName, bool allow_symlinks,
bool allow_files_from_the_future = true );
~FolderIterator();
enum { TYPE_UNKNOWN = 0x00,

122
libretroshare/src/util/i2pcommon.cpp
View File

@ -50,8 +50,10 @@ std::string publicKeyFromPrivate(std::string const &priv)
* https://geti2p.net/spec/common-structures#keysandcert
* https://geti2p.net/spec/common-structures#certificate
*/
if (priv.length() < 884) // base64 ( = 663 bytes = KeyCert + priv Keys)
if (priv.length() < privKeyMinLenth_b64) {
RS_WARN("key to short!");
return std::string();
}
// creat a copy to work on, need to convert it to standard base64
auto priv_copy(priv);
@ -70,13 +72,14 @@ std::string publicKeyFromPrivate(std::string const &priv)
uint8_t certType = 0;
uint16_t len = 0;
uint16_t signingKeyType = 0;
uint16_t cryptKey = 0;
uint16_t cryptKeyType = 0;
// only used for easy break
do {
try {
// jump to certificate
p += publicKeyLen;
// try to read type and length
certType = *p++;
len = readTwoBytesBE(p);
@ -87,7 +90,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
/*
* CertType.Null
* type null is followed by 0x00 0x00 <END>
* so has to be 0!
* so len has to be 0!
*/
RS_DBG("cert is CertType.Null");
publicKeyLen += 3; // add 0x00 0x00 0x00
@ -119,7 +122,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
// likely 7
signingKeyType = readTwoBytesBE(p);
RS_DBG("signing pubkey type ", certType);
RS_DBG("signing pubkey type ", signingKeyType);
if (signingKeyType >= 3 && signingKeyType <= 6) {
RS_DBG("signing pubkey type ", certType, " has oversize");
// calculate oversize
@ -137,18 +140,18 @@ std::string publicKeyFromPrivate(std::string const &priv)
return std::string();
}
publicKeyLen += values.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted
publicKeyLen += values.first - 128; // 128 = default DSA key length = the space that can be used before the key must be splitted
}
// Crypto Public Key
// likely 0
cryptKey = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKey);
cryptKeyType = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKeyType);
// info: these are all smaller than the default 256 bytes, so no oversize calculation is needed
break;
} catch (const std::out_of_range &e) {
RS_DBG("hit exception! ", e.what());
RS_DBG("hit an exception! ", e.what());
return std::string();
}
} while(false);
@ -160,4 +163,107 @@ std::string publicKeyFromPrivate(std::string const &priv)
return pub;
}
bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey)
{
if (key.length() < pubKeyMinLenth_b64) {
RS_WARN("key to short!");
return false;
}
// creat a copy to work on, need to convert it to standard base64
auto key_copy(key);
std::replace(key_copy.begin(), key_copy.end(), '~', '/');
// replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too
// std::replace(copy.begin(), copy.end(), '-', '+');
// get raw data
std::vector<uint8_t> data;
RsBase64::decode(key_copy, data);
auto p = data.cbegin();
constexpr size_t publicKeyLen = 256 + 128; // default length (bytes)
uint8_t certType = 0;
uint16_t signingKeyType = 0;
uint16_t cryptKeyType = 0;
// try to read types
try {
// jump to certificate
p += publicKeyLen;
// try to read type and skip length
certType = *p++;
p += 2;
// only 0 and 5 are used / valid at this point
// check for == 0
if (certType == static_cast<typename std::underlying_type<CertType>::type>(CertType::Null)) {
RS_DBG("cert is CertType.Null");
signingKey = "DSA_SHA1";
cryptoKey = "ElGamal";
return true;
}
// check for != 5
if (certType != static_cast<typename std::underlying_type<CertType>::type>(CertType::Key)) {
// unsupported
RS_DBG("cert type ", certType, " is unsupported");
return false;
}
RS_DBG("cert is CertType.Key");
// Signing Public Key
// likely 7
signingKeyType = readTwoBytesBE(p);
RS_DBG("signing pubkey type ", signingKeyType);
// Crypto Public Key
// likely 0
cryptKeyType = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKeyType);
} catch (const std::out_of_range &e) {
RS_DBG("hit an exception! ", e.what());
return false;
}
// now convert to string (this would be easier with c++17)
#define HELPER(a, b, c) \
case static_cast<typename std::underlying_type<a>::type>(a::c): \
b = #c; \
break;
switch (signingKeyType) {
HELPER(SigningKeyType, signingKey, DSA_SHA1)
HELPER(SigningKeyType, signingKey, ECDSA_SHA256_P256)
HELPER(SigningKeyType, signingKey, ECDSA_SHA384_P384)
HELPER(SigningKeyType, signingKey, ECDSA_SHA512_P521)
HELPER(SigningKeyType, signingKey, RSA_SHA256_2048)
HELPER(SigningKeyType, signingKey, RSA_SHA384_3072)
HELPER(SigningKeyType, signingKey, RSA_SHA512_4096)
HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519)
HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519ph)
HELPER(SigningKeyType, signingKey, RedDSA_SHA512_Ed25519)
default:
RsWarn("unkown signing key type:", signingKeyType);
return false;
}
switch (cryptKeyType) {
HELPER(CryptoKeyType, cryptoKey, ElGamal)
HELPER(CryptoKeyType, cryptoKey, P256)
HELPER(CryptoKeyType, cryptoKey, P384)
HELPER(CryptoKeyType, cryptoKey, P521)
HELPER(CryptoKeyType, cryptoKey, X25519)
default:
RsWarn("unkown crypto key type:", cryptKeyType);
return false;
}
#undef HELPER
return true;
}
} // namespace i2p

42
libretroshare/src/util/i2pcommon.h
View File

@ -186,6 +186,39 @@ static const std::array<std::pair<uint16_t, uint16_t>, 12> signingKeyLengths {
/*SigningKeyType::RedDSA_SHA512_Ed25519 */ std::make_pair<uint16_t, uint16_t>( 32, 32),
};
/*
* Key length infos:
*
* BOB private key
* len b64: 884
* len pln: 663
*
* BOB public key / destination
* len b64: 516
* len pln: 387
*
* SAMv3 private key
* len b64: 908
* len pln: 679
*
* SAMv3 public key
* len b64: 516
* len pln: 387
*
* Example:
* in bytes, public key only
* 384 (Key) + 3 (Null certificate) = 387 bytes
* 384 (Key) + 7 (key certificate) = 391 bytes
*
* in bytes public + private key
* 384 (Key) + 3 (Null certificate) + 256 (ElGamal) + 20 (DSA_SHA1) = 663 bytes
* 384 (Key) + 7 (key certificate) + 256 (ElGamal) + 32 (EdDSA_SHA512_Ed25519) = 679 bytes
*/
constexpr size_t pubKeyMinLenth_b64 = 516;
constexpr size_t pubKeyMinLenth_bin = 387;
constexpr size_t privKeyMinLenth_b64 = 884;
constexpr size_t privKeyMinLenth_bin = 663;
/**
* @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs
* @param lhs option to set
@ -208,6 +241,15 @@ std::string keyToBase32Addr(const std::string &key);
*/
std::string publicKeyFromPrivate(const std::string &priv);
/**
* @brief getKeyTypes returns the name of the utilized algorithms used by the key
* @param key public key
* @param signingKey name of the signing key, e.g. DSA_SHA1
* @param cryptoKey name of the crpyto key, e.g. ElGamal
* @return true on success, false otherwise
*/
bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey);
} // namespace i2p
#endif // I2PCOMMON_H

60
libretroshare/src/util/rsdebug.cc
View File

@ -3,9 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -22,6 +22,8 @@
* *
*******************************************************************************/
#include <iomanip>
#include "util/rsdebug.h"
std::ostream &operator<<(std::ostream& out, const std::error_condition& err)
@ -36,6 +38,54 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
" not available in category: " + categoryName;
}
std::error_condition rs_errno_to_condition(int errno_code)
{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
std::ostream& hex_dump(std::ostream& os, const void *buffer,
std::size_t bufsize, bool showPrintableChars /*= true*/)
{
if (buffer == nullptr) {
return os;
}
auto oldFormat = os.flags();
auto oldFillChar = os.fill();
constexpr std::size_t maxline{8};
// create a place to store text version of string
char renderString[maxline+1];
char *rsptr{renderString};
// convenience cast
const unsigned char *buf{reinterpret_cast<const unsigned char *>(buffer)};
for (std::size_t linecount=maxline; bufsize; --bufsize, ++buf) {
os << std::setw(2) << std::setfill('0') << std::hex
<< static_cast<unsigned>(*buf) << ' ';
*rsptr++ = std::isprint(*buf) ? *buf : '.';
if (--linecount == 0) {
*rsptr++ = '\0'; // terminate string
if (showPrintableChars) {
os << " | " << renderString;
}
os << '\n';
rsptr = renderString;
linecount = std::min(maxline, bufsize);
}
}
// emit newline if we haven't already
if (rsptr != renderString) {
if (showPrintableChars) {
for (*rsptr++ = '\0'; rsptr != &renderString[maxline+1]; ++rsptr) {
os << " ";
}
os << " | " << renderString;
}
os << '\n';
}
os.fill(oldFillChar);
os.flags(oldFormat);
return os;
}
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
@ -208,3 +258,7 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
lineCount++;
}
}
/// All the lines before are DEPRECATED!!
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

62
libretroshare/src/util/rsdebug.h
View File

@ -2,8 +2,8 @@
* RetroShare debugging utilities *
* *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
* the message around */
std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
/** Convert C errno codes to modern C++11 std::error_condition, this is quite
* useful to use toghether with C functions used around the code like `malloc`,
* `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
*/
std::error_condition rs_errno_to_condition(int errno_code);
template <RsLoggerCategories CATEGORY>
struct t_RsLogger : std::ostringstream
@ -204,6 +210,53 @@ struct RsNoDbg
};
//From https://codereview.stackexchange.com/a/165162
/**
* @brief hex_dump: Send Hexadecimal Dump to stream
* @param os: Output Stream
* @param buffer: Buffer to send
* @param bufsize: Buffer's size
* @param showPrintableChars: If must send printable Char too
* @return
* basic string:
* 61 62 63 64 65 66 31 32 | abcdef12
* 33 34 35 36 00 7a 79 78 | 3456.zyx
* 77 76 75 39 38 37 36 35 | wvu98765
* 34 45 64 77 61 72 64 00 | 4Edward.
*
* wide string:
* 41 00 00 00 20 00 00 00 | A... ...
* 77 00 00 00 69 00 00 00 | w...i...
* 64 00 00 00 65 00 00 00 | d...e...
* 20 00 00 00 73 00 00 00 | ...s...
* 74 00 00 00 72 00 00 00 | t...r...
* 69 00 00 00 6e 00 00 00 | i...n...
* 67 00 00 00 2e 00 00 00 | g.......
*
* a double
* 49 92 24 49 92 24 09 40 | I.$I.$.@
*/
std::ostream& hex_dump(std::ostream& os, const void *buffer,
std::size_t bufsize, bool showPrintableChars = true);
/**
* @brief The hexDump struct
* Enable to print dump calling like that:
* const char test[] = "abcdef123456\0zyxwvu987654Edward";
* RsDbg()<<hexDump(test, sizeof(test))<<std::endl;
*/
struct hexDump {
const void *buffer;
std::size_t bufsize;
hexDump(const void *buf, std::size_t bufsz) : buffer{buf}, bufsize{bufsz} {}
friend std::ostream &operator<<(std::ostream &out, const hexDump &hd) {
return hex_dump(out, hd.buffer, hd.bufsize, true);
}
};
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
@ -312,3 +365,8 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
#define PQL_DEBUG_ALERT RSL_DEBUG_ALERT
#define PQL_DEBUG_BASIC RSL_DEBUG_BASIC
#define PQL_DEBUG_ALL RSL_DEBUG_ALL
/// All the lines before are DEPRECATED!!
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

30
libretroshare/src/util/rsdir.cc
View File

@ -458,6 +458,36 @@ bool RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d
return true;
}
rstime_t RsDirUtil::lastWriteTime(
const std::string& path,
std::error_condition& errc )
{
if(!fileExists(path))
{
errc = std::errc::no_such_file_or_directory;
return 0;
}
#ifdef WINDOWS_SYS
struct _stati64 buf;
std::wstring wPath;
librs::util::ConvertUtf8ToUtf16(path, wPath);
if ( 0 == _wstati64(wPath.c_str(), &buf))
#else
struct stat64 buf;
if ( 0 == stat64(path.c_str(), &buf))
#endif
{
/* errc output param is guaranted to be meaningful only if an error
* happens so is not strictly necessary but we clean it anyway just
* in case */
errc = std::error_condition();
return buf.st_mtime;
}
errc = std::error_condition(errno, std::generic_category());
return 0;
}
bool RsDirUtil::checkDirectory(const std::string& dir)
{

Some files were not shown because too many files have changed in this diff Show More