Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-04-22 16:09:18 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed merge with upstream/master

Browse Source
This commit is contained in:
csoler 2021-12-13 21:01:06 +01:00
commit 56e17f077e
1005 changed files with 22644 additions and 19062 deletions
.gitignore.gitlab-ci.yml.gitmodules.travis.ymlREADME.asciidoc
build_scripts
Android
DockerfileREADME.asciidocprepare-toolchain-clang.sh
GitlabCI
Android.Dockerfilebase.DockerfilegetRepoArgs.sh
OBS
Windows-msys2
build
pack.bat
env/tools
prepare-msys2.bat
root
update-msys2.bat
installer
retroshare-Qt5.nsi
Windows
build
pack.bat
env/tools
prepare-msys2.bat
root
update-msys2.bat
installer
retroshare-Qt5.nsi
libbitdht
CMakeLists.txt
libretroshare
CMakeLists.txt
src
CMakeLists.txt
chat
distributedchat.ccdistributedchat.h
deep_search
commonutils.cppcommonutils.hppfilesindex.cppfilesindex.hppforumsindex.cppforumsindex.hpp
file_sharing
dir_hierarchy.ccdirectory_storage.ccdirectory_updater.cchash_cache.ccp3filelists.cc
ft
ftcontroller.ccftcontroller.hftdatamultiplex.ccftdatamultiplex.hftserver.cc
gxs
rsgenexchange.ccrsgxsnetservice.ccrsgxsnetservice.hrsgxsnettunnel.ccrsgxsnettunnel.hrsgxsnotify.hrsgxsutil.ccrsgxsutil.hrsnxs.hrsnxsobserver.cpprsnxsobserver.h
jsonapi
jsonapi.cpp
libretroshare.pro
pgp
rscertificate.ccrscertificate.h
pqi
p3netmgr.ccp3netmgr.hp3peermgr.ccpqinetwork.ccpqinetwork.hpqissl.ccpqissli2psam3.cpppqissli2psam3.hpqissllistener.hpqisslpersongrp.cc
retroshare
rsevents.hrsfiles.hrsgxscircles.hrsgxsforums.hrsgxsiface.hrsgxsifacehelper.hrsgxsifacetypes.hrsinit.hrsmsgs.hrspeers.hrstypes.h
rs_android
LocalArray.hREADME-ifaddrs-android.adocScopedFd.handroidcoutcerrcatcher.hpperrorconditionwrap.cppifaddrs-android.h
org/retroshare/service
AssetHelper.javaErrorConditionWrap.javaRetroShareServiceAndroid.java
retroshareserviceandroid.cppretroshareserviceandroid.hpprsjni.cpprsjni.hpp
rsitems
itempriorities.hrsgxsforumitems.hrsitem.hrsnxsitems.ccrsnxsitems.hrsserviceids.h
rsserver
p3face.hp3msgs.ccp3msgs.hp3peers.cc
Show More

2
.gitignore vendored

@ -15,3 +15,5 @@ Thumbs.db
*.pro.user
.kdev4
*.kdev4
!supportlibs/libsam3/Makefile

84
.gitlab-ci.yml

@ -1,32 +1,48 @@
image: docker:stable
services:
- docker:stable-dind
- docker:stable-dind
stages:
- build
- test
workflow:
rules:
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_BRANCH
build-and-test:
variables:
UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
build-ubuntu-test-image:
stage: build
script:
- >
if [ -n "$CI_MERGE_REQUEST_ID" ]; then
REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
else
REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
fi ;
export REPO_ARGS ;
echo REPO_ARGS=$REPO_ARGS ;
- >
docker login "$CI_REGISTRY"
--username "$CI_REGISTRY_USER"
--password "$CI_REGISTRY_PASSWORD"
- mkdir Dockercontext
- >
docker build -t retroshare:testing $REPO_ARGS
docker build
-t $UBUNTU_TESTING_IMAGE_TAG
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
Dockercontext
- docker push $UBUNTU_TESTING_IMAGE_TAG
- echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
test-ubuntu:
stage: test
script:
- >
docker login "$CI_REGISTRY"
--username "$CI_REGISTRY_USER"
--password "$CI_REGISTRY_PASSWORD"
- echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
- docker pull $UBUNTU_TESTING_IMAGE_TAG
- >
docker run --name retroshare --detach --tty retroshare:testing
docker run --name retroshare --detach --tty $UBUNTU_TESTING_IMAGE_TAG
retroshare-service --jsonApiPort 9092
- apk add jq
- >
@ -51,3 +67,43 @@ build-and-test:
docker exec retroshare
curl --verbose http://127.0.0.1:9092/rsMsgs/getChatLobbyList | jq
- docker container stop retroshare
build-android-arm-apk:
stage: build
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
# see https://stackoverflow.com/a/59055906
- >
docker cp
$(docker create --rm retroshare:android_arm_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm.apk
artifacts:
paths:
- RetroShare_Android_Service-arm.apk
# Use separate runner to avoid no space left on device
build-android-arm64-apk:
stage: build
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm64_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg ANDROID_PLATFORM_VER=21
--build-arg ANDROID_NDK_ARCH=arm64
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
- >
docker cp
$(docker create --rm retroshare:android_arm64_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm64.apk
artifacts:
paths:
- RetroShare_Android_Service-arm64.apk

6
.gitmodules vendored

@ -14,3 +14,9 @@
[submodule "supportlibs/rapidjson"]
path = supportlibs/rapidjson
url = https://github.com/Tencent/rapidjson.git
[submodule "supportlibs/libsam3"]
path = supportlibs/libsam3
url = https://github.com/i2p/libsam3.git
[submodule "supportlibs/jni.hpp"]
path = supportlibs/jni.hpp
url = https://github.com/RetroShare/jni.hpp.git

12
.travis.yml

@ -8,10 +8,10 @@ language: cpp
matrix:
include:
- os: linux
dist: bionic
sudo: required
compiler: gcc
# - os: linux
# dist: bionic
# sudo: required
# compiler: gcc
- os: osx
osx_image: xcode10.2
compiler: clang
@ -68,11 +68,11 @@ before_script:
fi
script:
- if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j4; fi
- if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then travis_wait 30 make -j$(nproc); fi
after_success:
- if [ $TRAVIS_OS_NAME == osx ]; then build_scripts/OSX/travis_makeOSXPackage.sh ; fi
- if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
# - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
# branches:

4
README.asciidoc

@ -9,9 +9,9 @@ RetroShare provides file sharing, chat, messages, forums, channels and more.
.Build Status
|===============================================================================
|GNU/Linux (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|GNU/Linux, macOS, (via Travis CI) | image:https://travis-ci.org/RetroShare/RetroShare.svg?branch=master[link="https://travis-ci.org/RetroShare/RetroShare"]
|GNU/Linux, Android (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|Windows (via AppVeyor) | image:https://ci.appveyor.com/api/projects/status/github/RetroShare/RetroShare?svg=true[link="https://ci.appveyor.com/project/RetroShare58622/retroshare"]
|macOS, (via Travis CI) | image:https://app.travis-ci.com/RetroShare/RetroShare.svg?branch=master[link="https://app.travis-ci.com/github/RetroShare/RetroShare"]
|===============================================================================

117
build_scripts/Android/Dockerfile Normal file

@ -0,0 +1,117 @@
## To prepare an image suitable as base for Gitlab CI use
## image name must match gitlab repository name, you can play just with the tag
## the part after :
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm_base"
# docker build --squash --tag "${CI_IMAGE_NAME}" \
# --build-arg QT_INSTALLER_JWT_TOKEN="your qt JWT token goes here" .
#
# To build Android ARMv8 (64 bit) package pass also
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm64_base"
# --build-arg ANDROID_NDK_ARCH=arm64 --build-arg ANDROID_PLATFORM_VER=21
## --squash is very important in case of GitlabCI shared runners as they are
## limited to 25GB disk size
## To push it to gitlab CI registry you need first to login and the to push
# docker login registry.gitlab.com
# docker push ${CI_IMAGE_NAME}
## To extract the generated APK easily you can run after the build complete
# docker cp \
# $(docker create --rm ${CI_IMAGE_NAME}):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk \
# /tmp/RetroShare_Android_Service.apk
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive
ENV APT_UNAT="--assume-yes --quiet"
RUN apt-get update $APT_UNAT && apt-get upgrade --show-upgraded $APT_UNAT && \
apt-get clean $APT_UNAT
RUN apt-get install --no-install-recommends $APT_UNAT \
bash build-essential bzip2 cmake curl chrpath doxygen \
git p7zip python qt5-default qttools5-dev tclsh unzip wget zip
# Dependencies to create Android pkg
RUN apt-get install --no-install-recommends $APT_UNAT \
openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle
ARG FRESHCLONE=0
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
ARG REPO_DEPTH="--depth 2000"
RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
git fetch --tags
ENV PREPARE_TOOLCHAIN="/RetroShare/build_scripts/Android/prepare-toolchain-clang.sh"
ENV NATIVE_LIBS_TOOLCHAIN_PATH="/android-toolchain/"
ARG ANDROID_PLATFORM_VER=16
ARG ANDROID_NDK_ARCH=arm
ENV ANDROID_SDK_PATH="/opt/android-sdk"
ENV ANDROID_HOME="$ANDROID_SDK_PATH"
ENV ANDROID_SDK_ROOT="$ANDROID_SDK_PATH"
ENV ANDROID_NDK_PATH="/opt/android-ndk"
ENV ANDROID_NDK_ROOT="$ANDROID_NDK_PATH"
ENV PATH="$PATH:$ANDROID_HOME/tools"
ENV PATH="$PATH:$ANDROID_HOME/platform-tools"
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN mkdir /bCache
WORKDIR /bCache
## Quick HACK to ease development
#COPY prepare-toolchain-clang.sh $PREPARE_TOOLCHAIN
RUN $PREPARE_TOOLCHAIN install_android_sdk
RUN $PREPARE_TOOLCHAIN install_android_ndk
RUN $PREPARE_TOOLCHAIN
ARG QT_INSTALLER_JWT_TOKEN
RUN $PREPARE_TOOLCHAIN install_qt_android
# Avoid Qt account details leak into the image
RUN rm -f /root/.local/share/Qt/qtaccount.ini
# Shrink image by removing unneeded Qt components
RUN rm -r \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Docs/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Examples/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Tools/
RUN mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
RUN mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle
RUN rm -rf /bCache
# Clean apt cache
RUN apt-get clean && rm -rf /var/lib/apt/lists/*

1
build_scripts/Android/README.asciidoc

@ -81,6 +81,7 @@ JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
CONFIG+=no_rs_service_terminal_login
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3
-------------------------------------------------------------------------------
TIP: Some versions of QtCreator try to find the Android SDK in

802
build_scripts/Android/prepare-toolchain-clang.sh

@ -2,7 +2,8 @@
# Script to prepare RetroShare Android package building toolchain
#
# Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
@ -38,6 +39,15 @@ define_default_value ANDROID_PLATFORM_VER "16"
define_default_value NATIVE_LIBS_TOOLCHAIN_PATH "${HOME}/Builds/android-toolchains/retroshare-android-${ANDROID_PLATFORM_VER}-${ANDROID_NDK_ARCH}/"
define_default_value HOST_NUM_CPU $(nproc)
define_default_value ANDROID_SDK_INSTALL "false"
define_default_value ANDROID_SDK_TOOLS_VERSION "3859397"
define_default_value ANDROID_SDK_TOOLS_SHA256 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
define_default_value ANDROID_SDK_VERSION "29.0.3"
define_default_value ANDROID_NDK_INSTALL "false"
define_default_value ANDROID_NDK_VERSION "r21"
define_default_value ANDROID_NDK_SHA256 b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b
define_default_value BZIP2_SOURCE_VERSION "1.0.6"
define_default_value BZIP2_SOURCE_SHA256 a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
@ -48,16 +58,22 @@ define_default_value SQLITE_SOURCE_YEAR "2018"
define_default_value SQLITE_SOURCE_VERSION "3250200"
define_default_value SQLITE_SOURCE_SHA256 da9a1484423d524d3ac793af518cdf870c8255d209e369bd6a193e9f9d0e3181
define_default_value SQLCIPHER_SOURCE_VERSION "4.2.0"
define_default_value SQLCIPHER_SOURCE_SHA256 105c1b813f848da038c03647a8bfc9d42fb46865e6aaf4edfd46ff3b18cdccfc
define_default_value SQLCIPHER_SOURCE_VERSION "4.4.3"
define_default_value SQLCIPHER_SOURCE_SHA256 b8df69b998c042ce7f8a99f07cf11f45dfebe51110ef92de95f1728358853133
define_default_value LIBUPNP_SOURCE_VERSION "1.8.4"
define_default_value LIBUPNP_SOURCE_SHA256 976c3e4555604cdd8391ed2f359c08c9dead3b6bf131c24ce78e64d6669af2ed
define_default_value INSTALL_QT_ANDROID "false"
define_default_value QT_VERSION "5.12.0"
define_default_value QT_ANDROID_VIA_INSTALLER "false"
define_default_value QT_VERSION "5.12.11"
define_default_value QT_INSTALLER_VERSION "4.1.1"
define_default_value QT_INSTALLER_SHA256 1266ffd0d1b0e466244e3bc8422975c1aa9d96745b6bb28d422f7f92df11f34c
define_default_value QT_INSTALLER_JWT_TOKEN ""
define_default_value QT_INSTALL_PATH "${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt/"
define_default_value QT_ANDROID_INSTALLER_SHA256 a214084e2295c9a9f8727e8a0131c37255bf724bfc69e80f7012ba3abeb1f763
define_default_value RESTBED_SOURCE_REPO "https://github.com/Corvusoft/restbed.git"
define_default_value RESTBED_SOURCE_VERSION f74f9329dac82e662c1d570b7cd72c192b729eb4
define_default_value UDP_DISCOVERY_CPP_SOURCE "https://github.com/truvorskameikin/udp-discovery-cpp.git"
@ -72,6 +88,67 @@ define_default_value RAPIDJSON_SOURCE_SHA256 bf7ced29704a1e696fbccf2a2b4ea068e77
define_default_value MINIUPNPC_SOURCE_VERSION "2.1.20190625"
define_default_value MINIUPNPC_SOURCE_SHA256 8723f5d7fd7970de23635547700878cd29a5c2bb708b5e5475b2d1d2510317fb
# zlib and libpng versions walks toghether
define_default_value ZLIB_SOURCE_VERSION "1.2.11"
define_default_value ZLIB_SOURCE_SHA256 4ff941449631ace0d4d203e3483be9dbc9da454084111f97ea0a2114e19bf066
define_default_value LIBPNG_SOURCE_VERSION "1.6.37"
define_default_value LIBPNG_SOURCE_SHA256 505e70834d35383537b6491e7ae8641f1a4bed1876dbfe361201fc80868d88ca
define_default_value LIBJPEG_SOURCE_VERSION "9d"
define_default_value LIBJPEG_SOURCE_SHA256 6c434a3be59f8f62425b2e3c077e785c9ce30ee5874ea1c270e843f273ba71ee
define_default_value TIFF_SOURCE_VERSION "4.2.0"
define_default_value TIFF_SOURCE_SHA256 eb0484e568ead8fa23b513e9b0041df7e327f4ee2d22db5a533929dfc19633cb
define_default_value CIMG_SOURCE_VERSION "2.9.7"
define_default_value CIMG_SOURCE_SHA256 595dda9718431a123b418fa0db88e248c44590d47d9b1646970fa0503e27fa5c
define_default_value PHASH_SOURCE_REPO "https://gitlab.com/g10h4ck/pHash.git"
define_default_value PHASH_SOURCE_VERSION origin/android-ndk
define_default_value MVPTREE_SOURCE_REPO "https://github.com/starkdg/mvptree.git"
define_default_value MVPTREE_SOURCE_VERSION origin/master
define_default_value REPORT_DIR "$(pwd)/$(basename ${NATIVE_LIBS_TOOLCHAIN_PATH})_build_report/"
define_default_value RS_SRC_DIR "$(realpath $(dirname $BASH_SOURCE)/../../)"
cArch=""
eABI=""
cmakeABI=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
;;
"arm64")
cArch="aarch64"
eABI=""
;;
"x86")
cArch="i686"
eABI=""
;;
"x86_64")
echo "ANDROID_NDK_ARCH=${ANDROID_NDK_ARCH} not supported yet"
exit -1
cArch="??"
eABI=""
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
# Used to instruct cmake to explicitely ignore host libraries
export HOST_IGNORE_PREFIX="/usr/"
## $1 filename, $2 sha256 hash
function check_sha256()
@ -104,159 +181,269 @@ function verified_download()
}
}
cArch=""
eABI=""
# This function is the result of reading and testing many many stuff be very
# careful editing it
function andro_cmake()
{
# Using android.toolchain.cmake as documented here
# https://developer.android.com/ndk/guides/cmake seens to break more things then
# it fixes :-\
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
cmakeProc=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cmakeProc="armv7-a"
;;
"arm64")
cArch="aarch64"
eABI=""
"arm64")
cmakeProc="aarch64"
;;
"x86")
cArch="i686"
eABI=""
"x86")
cmakeProc="i686"
;;
esac
"x86_64")
cmakeProc="x86_64"
;;
*)
echo "Unhandled NDK architecture ${ANDROID_NDK_ARCH}"
exit -1
;;
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
_hi="$HOST_IGNORE_PREFIX"
cmake \
-DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_PREFIX_PATH="${PREFIX}" \
-DCMAKE_SYSTEM_PREFIX_PATH="${PREFIX}" \
-DCMAKE_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_SYSTEM_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_SYSTEM_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" \
-DCMAKE_IGNORE_PATH="$_hi/include;$_hi/lib;$_hi/lib64" \
$@
# It is probably ok to do not touch CMAKE_PROGRAM_PATH and
# CMAKE_SYSTEM_PROGRAM_PATH
}
function git_source_get()
{
sourceDir="$1" ; shift #$1
sourceRepo="$1" ; shift #$2
sourceVersion="$1" ; shift #$3
# extra paramethers are treated as submodules
[ -d "$sourceDir" ] &&
{
pushd "$sourceDir"
actUrl="$(git remote get-url origin)"
[ "$actUrl" != "$sourceRepo" ] && rm -rf "${sourceDir}"
popd
} || true
[ -d $sourceDir ] || git clone "$sourceRepo" "$sourceDir"
pushd $sourceDir
git fetch --all
git reset --hard ${sourceVersion}
while [ "$1" != "" ] ; do
git submodule update --init "$1"
pushd "$1"
git reset --hard
shift
popd
done
popd
}
declare -A TASK_REGISTER
function task_register()
{
TASK_REGISTER[$1]=true
}
function task_unregister()
{
# we may simply wipe them but we could benefit from keeping track of
# unregistered tasks too
TASK_REGISTER[$1]=false
}
function task_logfile()
{
echo "$REPORT_DIR/$1.log"
}
function task_run()
{
mTask="$1" ; shift
[ "${TASK_REGISTER[$mTask]}" != "true" ] &&
{
echo "Attempt to run not registered task $mTask $@"
return -1
}
logFile="$(task_logfile $mTask)"
if [ -f "$logFile" ] ; then
echo "Task $mTask already run more details at $logFile"
else
date | tee > "$logFile"
$mTask $@ |& tee --append "$logFile"
mRetval="${PIPESTATUS[0]}"
echo "Task $mTask return ${mRetval} more details at $logFile"
date | tee --append "$logFile"
return ${mRetval}
fi
}
function task_zap()
{
rm -f "$(task_logfile $1)"
}
DUPLICATED_INCLUDES_LIST_FILE="${REPORT_DIR}/duplicated_includes_list"
DUPLICATED_INCLUDES_DIR="${REPORT_DIR}/duplicated_includes/"
task_register install_android_sdk
install_android_sdk()
{
tFile="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
verified_download "${tFile}" "${ANDROID_SDK_TOOLS_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_SDK_PATH"
rm -rf "$ANDROID_SDK_PATH/tools/"
mv --verbose tools/ "$ANDROID_SDK_PATH/tools/"
# Install Android SDK
yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses && \
$ANDROID_SDK_PATH/tools/bin/sdkmanager --update
$ANDROID_SDK_PATH/tools/bin/sdkmanager "platforms;android-$ANDROID_PLATFORM_VER"
$ANDROID_SDK_PATH/tools/bin/sdkmanager "build-tools;$ANDROID_SDK_VERSION"
}
task_register install_android_ndk
install_android_ndk()
{
tFile="android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
verified_download "${tFile}" "${ANDROID_NDK_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_NDK_PATH"
rm -rf "$ANDROID_NDK_PATH"
mv --verbose "android-ndk-${ANDROID_NDK_VERSION}/" "$ANDROID_NDK_PATH/"
}
## More information available at https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html
build_toolchain()
task_register bootstrap_toolchain
bootstrap_toolchain()
{
echo "build_toolchain()
################################################################################
################################################################################
################################################################################
"
rm -rf ${NATIVE_LIBS_TOOLCHAIN_PATH}
rm -rf "${NATIVE_LIBS_TOOLCHAIN_PATH}"
${ANDROID_NDK_PATH}/build/tools/make_standalone_toolchain.py --verbose \
--arch ${ANDROID_NDK_ARCH} --install-dir ${NATIVE_LIBS_TOOLCHAIN_PATH} \
--api ${ANDROID_PLATFORM_VER}
find "${PREFIX}/include/" -not -type d > "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles"
# Avoid problems with arm64 some libraries installing on lib64
ln -s "${PREFIX}/lib/" "${PREFIX}/lib64"
pushd "${PREFIX}/include/"
find . -not -type d > "${DUPLICATED_INCLUDES_LIST_FILE}"
popd
}
## This avoid <cmath> include errors due to -isystem and -I ordering issue
delete_copied_includes()
task_register deduplicate_includes
deduplicate_includes()
{
echo "delete_copied_includes()
################################################################################
################################################################################
################################################################################
"
cat "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles" | while read delFile ; do
rm "$delFile"
done
while read delFile ; do
mNewPath="${DUPLICATED_INCLUDES_DIR}/$delFile"
mkdir --verbose --parents "$(dirname "$mNewPath")"
mv --verbose "${PREFIX}/include/$delFile" "$mNewPath"
done < "${DUPLICATED_INCLUDES_LIST_FILE}"
}
## More information available at https://gitlab.com/relan/provisioners/merge_requests/1 and http://stackoverflow.com/a/34032216
task_register reduplicate_includes
reduplicate_includes()
{
pushd "${DUPLICATED_INCLUDES_DIR}"
find . -not -type d | while read delFile ; do
mv --verbose "${delFile}" "${PREFIX}/include/$delFile"
done
popd
}
# $1 optional prefix prepended only if return value is not empty
# $2 optional suffix appended only if return value is not empty
task_register get_qt_arch
get_qt_arch()
{
local QT_VERSION_COMP="$(echo $QT_VERSION | awk -F. '{print $1*1000000+$2*1000+$3}')"
local QT_ARCH=""
# Qt >= 5.15.0 ships all Android architectures toghether
[ "$QT_VERSION_COMP" -lt "5015000" ] &&
{
case "${ANDROID_NDK_ARCH}" in
"arm")
QT_ARCH="armv7"
;;
"arm64")
QT_ARCH="arm64_v8a"
;;
"x86")
QT_ARCH="x86"
;;
esac
echo "$1$QT_ARCH$2"
}
}
task_register get_qt_dir
get_qt_dir()
{
echo "${QT_INSTALL_PATH}/${QT_VERSION}/android$(get_qt_arch _)/"
}
## More information available at https://wiki.qt.io/Online_Installer_4.x
task_register install_qt_android
install_qt_android()
{
echo "install_qt_android()
################################################################################
################################################################################
################################################################################
"
[ "$QT_INSTALLER_JWT_TOKEN" == "" ] &&
{
echo "To run Qt installer QT_INSTALLER_JWT_TOKEN environement variable \
need to be set to a valid JWT token see https://wiki.qt.io/Online_Installer_4.x"
return -1
}
QT_VERSION_CODE=$(echo $QT_VERSION | tr -d .)
QT_INSTALL_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt
QT_INSTALLER="qt-unified-linux-x64-3.0.2-online.run"
verified_download $QT_INSTALLER $QT_ANDROID_INSTALLER_SHA256 \
http://master.qt.io/archive/online_installers/3.0/${QT_INSTALLER}
QT_VERSION_CODE="$(echo $QT_VERSION | tr -d .)"
QT_INSTALLER="qt-unified-linux-x86_64-${QT_INSTALLER_VERSION}-online.run"
tMajDotMinVer="$(echo $QT_INSTALLER_VERSION | awk -F. '{print $1"."$2}')"
verified_download $QT_INSTALLER $QT_INSTALLER_SHA256 \
"https://master.qt.io/archive/online_installers/${tMajDotMinVer}/${QT_INSTALLER}"
chmod a+x ${QT_INSTALLER}
QT_INSTALLER_SCRIPT="qt_installer_script.js"
cat << EOF > "${QT_INSTALLER_SCRIPT}"
function Controller() {
installer.autoRejectMessageBoxes();
installer.installationFinished.connect(function() {
gui.clickButton(buttons.NextButton);
});
var welcomePage = gui.pageWidgetByObjectName("WelcomePage");
welcomePage.completeChanged.connect(function() {
if (gui.currentPageWidget().objectName == welcomePage.objectName)
gui.clickButton(buttons.NextButton);
});
}
Controller.prototype.WelcomePageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.CredentialsPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.IntroductionPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.TargetDirectoryPageCallback = function() {
gui.currentPageWidget().TargetDirectoryLineEdit.setText("$QT_INSTALL_PATH");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ComponentSelectionPageCallback = function() {
var widget = gui.currentPageWidget();
// You can get these component names by running the installer with the
// --verbose flag. It will then print out a resource tree.
widget.deselectComponent("qt.tools.qtcreator");
widget.deselectComponent("qt.tools.doc");
widget.deselectComponent("qt.tools.examples");
widget.selectComponent("qt.$QT_VERSION_CODE.android_armv7");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.LicenseAgreementPageCallback = function() {
gui.currentPageWidget().AcceptLicenseRadioButton.setChecked(true);
gui.clickButton(buttons.NextButton);
}
Controller.prototype.StartMenuDirectoryPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ReadyForInstallationPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.FinishedPageCallback = function() {
var checkBoxForm = gui.currentPageWidget().LaunchQtCreatorCheckBoxForm;
if (checkBoxForm && checkBoxForm.launchQtCreatorCheckBox)
checkBoxForm.launchQtCreatorCheckBox.checked = false;
gui.clickButton(buttons.FinishButton);
}
EOF
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} --script ${QT_INSTALLER_SCRIPT}
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} \
install qt.qt5.${QT_VERSION_CODE}.android$(get_qt_arch _) \
--accept-licenses --accept-obligations --confirm-command \
--default-answer --no-default-installations \
--root "${QT_INSTALL_PATH}"
}
## More information available at retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136
task_register build_bzlib
build_bzlib()
{
echo "build_bzlib()
################################################################################
################################################################################
################################################################################
"
B_dir="bzip2-${BZIP2_SOURCE_VERSION}"
rm -rf $B_dir
@ -279,14 +466,9 @@ build_bzlib()
}
## More information available at http://doc.qt.io/qt-5/opensslsupport.html
task_register build_openssl
build_openssl()
{
echo "build_openssl()
################################################################################
################################################################################
################################################################################
"
B_dir="openssl-${OPENSSL_SOURCE_VERSION}"
rm -rf $B_dir
@ -308,6 +490,10 @@ build_openssl()
--openssldir="${SYSROOT}/etc/ssl"
# sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile
# sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile
# Avoid documentation build which is unneded and time consuming
echo "exit 0; " > util/process_docs.pl
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libssl.so*
@ -315,14 +501,9 @@ build_openssl()
cd ..
}
task_register build_sqlite
build_sqlite()
{
echo "build_sqlite()
################################################################################
################################################################################
################################################################################
"
B_dir="sqlite-autoconf-${SQLITE_SOURCE_VERSION}"
rm -rf $B_dir
@ -335,18 +516,13 @@ build_sqlite()
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libsqlite3.so*
# ${CC} -shared -o libsqlite3.so -fPIC sqlite3.o -ldl
# cp libsqlite3.so "${SYSROOT}/usr/lib"
cd ..
}
task_register build_sqlcipher
build_sqlcipher()
{
echo "build_sqlcipher()
################################################################################
################################################################################
################################################################################
"
task_run build_sqlite
B_dir="sqlcipher-${SQLCIPHER_SOURCE_VERSION}"
rm -rf $B_dir
@ -358,14 +534,14 @@ build_sqlcipher()
tar -xf $T_file
cd $B_dir
case "${ANDROID_NDK_ARCH}" in
"arm64")
# SQLCipher config.sub is outdated and doesn't recognize newer architectures
rm config.sub
autoreconf --verbose --install --force
automake --add-missing --copy --force-missing
;;
esac
# case "${ANDROID_NDK_ARCH}" in
# "arm64")
# # SQLCipher config.sub is outdated and doesn't recognize newer architectures
# rm config.sub
# autoreconf --verbose --install --force
# automake --add-missing --copy --force-missing
# ;;
# esac
./configure --with-pic --build=$(sh ./config.guess) \
--host=${cArch}-linux \
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}" \
@ -377,14 +553,9 @@ build_sqlcipher()
cd ..
}
task_register build_libupnp
build_libupnp()
{
echo "build_libupnp()
################################################################################
################################################################################
################################################################################
"
B_dir="pupnp-release-${LIBUPNP_SOURCE_VERSION}"
B_ext=".tar.gz"
B_file="${B_dir}${B_ext}"
@ -408,14 +579,9 @@ build_libupnp()
cd ..
}
task_register build_rapidjson
build_rapidjson()
{
echo "build_rapidjson()
################################################################################
################################################################################
################################################################################
"
B_dir="rapidjson-${RAPIDJSON_SOURCE_VERSION}"
D_file="${B_dir}.tar.gz"
verified_download $D_file $RAPIDJSON_SOURCE_SHA256 \
@ -424,42 +590,25 @@ build_rapidjson()
cp -r "${B_dir}/include/rapidjson/" "${PREFIX}/include/rapidjson"
}
task_register build_restbed
build_restbed()
{
echo "build_restbed()
################################################################################
################################################################################
################################################################################
"
S_dir="restbed"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$RESTBED_SOURCE_REPO" "${RESTBED_SOURCE_VERSION}" \
"dependency/asio" "dependency/catch"
[ -d restbed ] || git clone --depth=2000 https://github.com/Corvusoft/restbed.git
cd restbed
# git fetch --tags
# git checkout tags/${RESTBED_SOURCE_VERSION}
git checkout ${RESTBED_SOURCE_VERSION}
git submodule update --init dependency/asio
git submodule update --init dependency/catch
git submodule update --init dependency/kashmir
cd ..
rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
rm -rf "$B_dir"; mkdir "$B_dir"
pushd "$B_dir"
andro_cmake -DBUILD_TESTS=OFF -DBUILD_SSL=OFF -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
cp "${PREFIX}/library/librestbed.a" "${PREFIX}/lib/"
cd ..
popd
}
task_register build_udp-discovery-cpp
build_udp-discovery-cpp()
{
echo "build_udp-discovery-cpp()
################################################################################
################################################################################
################################################################################
"
S_dir="udp-discovery-cpp"
[ -d $S_dir ] || git clone $UDP_DISCOVERY_CPP_SOURCE $S_dir
cd $S_dir
@ -468,23 +617,16 @@ build_udp-discovery-cpp()
B_dir="udp-discovery-cpp-build"
rm -rf ${B_dir}; mkdir ${B_dir}; cd ${B_dir}
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../udp-discovery-cpp
andro_cmake -B. -H../$S_dir
make -j${HOST_NUM_CPU}
cp libudp-discovery.a "${PREFIX}/lib/"
cp ../$S_dir/*.hpp "${PREFIX}/include/"
cd ..
}
task_register build_xapian
build_xapian()
{
echo "build_xapian()
################################################################################
################################################################################
################################################################################
"
B_dir="xapian-core-${XAPIAN_SOURCE_VERSION}"
D_file="$B_dir.tar.xz"
verified_download $D_file $XAPIAN_SOURCE_SHA256 \
@ -503,15 +645,12 @@ build_xapian()
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}"
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_miniupnpc
build_miniupnpc()
{
echo "build_miniupnpc()
################################################################################
################################################################################
################################################################################
"
S_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}"
B_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}-build"
D_file="$S_dir.tar.gz"
@ -521,29 +660,238 @@ build_miniupnpc()
tar -xf $D_file
mkdir $B_dir
cd $B_dir
cmake \
andro_cmake \
-DUPNPC_BUILD_STATIC=TRUE \
-DUPNPC_BUILD_SHARED=FALSE \
-DUPNPC_BUILD_TESTS=FALSE \
-DUPNPC_BUILD_SAMPLE=FALSE \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -S../$S_dir
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
build_toolchain
[ "${INSTALL_QT_ANDROID}X" != "trueX" ] || install_qt_android
build_bzlib
build_openssl
build_sqlite
build_sqlcipher
build_libupnp
build_rapidjson
build_restbed
build_udp-discovery-cpp
build_xapian
build_miniupnpc
delete_copied_includes
task_register build_zlib
build_zlib()
{
S_dir="zlib-${ZLIB_SOURCE_VERSION}"
B_dir="zlib-${ZLIB_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $ZLIB_SOURCE_SHA256 \
http://www.zlib.net/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
cd $B_dir
andro_cmake -B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libz.so*
cd ..
}
echo NATIVE_LIBS_TOOLCHAIN_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}
task_register build_libpng
build_libpng()
{
task_run build_zlib
S_dir="libpng-${LIBPNG_SOURCE_VERSION}"
B_dir="libpng-${LIBPNG_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $LIBPNG_SOURCE_SHA256 \
https://download.sourceforge.net/libpng/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
# libm is part of bionic An android
sed -i -e 's/find_library(M_LIBRARY m)/set(M_LIBRARY "")/' $S_dir/CMakeLists.txt
# Disable hardware acceleration as they are problematic for Android
# compilation and are not supported by all phones, it is necessary to fiddle
# with CMakeLists.txt as libpng 1.6.37 passing it as cmake options seems not
# working properly
# https://github.com/imagemin/optipng-bin/issues/97
# https://github.com/opencv/opencv/issues/7600
echo "add_definitions(-DPNG_ARM_NEON_OPT=0)" >> $S_dir/CMakeLists.txt
mkdir $B_dir
pushd $B_dir
HW_OPT="OFF"
# [ "$ANDROID_PLATFORM_VER" -ge "22" ] && HW_OPT="ON"
andro_cmake \
-DPNG_SHARED=OFF \
-DPNG_STATIC=ON \
-DPNG_TESTS=OFF \
-DPNG_HARDWARE_OPTIMIZATIONS=$HW_OPT \
-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
popd
}
task_register build_libjpeg
build_libjpeg()
{
S_dir="jpeg-${LIBJPEG_SOURCE_VERSION}"
D_file="jpegsrc.v${LIBJPEG_SOURCE_VERSION}.tar.gz"
verified_download $D_file $LIBJPEG_SOURCE_SHA256 \
https://www.ijg.org/files/$D_file
rm -rf $S_dir
tar -xf $D_file
cd $S_dir
./configure --with-pic --prefix="${PREFIX}" --host=${cArch}-linux
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libjpeg.so*
cd ..
}
task_register build_tiff
build_tiff()
{
S_dir="tiff-${TIFF_SOURCE_VERSION}"
B_dir="${S_dir}-build"
D_file="tiff-${TIFF_SOURCE_VERSION}.tar.gz"
verified_download $D_file $TIFF_SOURCE_SHA256 \
https://download.osgeo.org/libtiff/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
# Disable tools building, not needed for retroshare, and depending on some
# OpenGL headers not available on Android
echo "" > $S_dir/tools/CMakeLists.txt
# Disable tests building, not needed for retroshare, and causing linker
# errors
echo "" > $S_dir/test/CMakeLists.txt
# Disable extra tools building, not needed for retroshare, and causing
# linker errors
echo "" > $S_dir/contrib/CMakeLists.txt
# Disable more unneded stuff
echo "" > $S_dir/build/CMakeLists.txt
echo "" > $S_dir/html/CMakeLists.txt
echo "" > $S_dir/man/CMakeLists.txt
echo "" > $S_dir/port/CMakeLists.txt
# Change to static library build
sed -i 's\add_library(tiff\add_library(tiff STATIC\' \
$S_dir/libtiff/CMakeLists.txt
cd $B_dir
#TODO: build dependecies to support more formats
andro_cmake \
-Dlibdeflate=OFF -Djbig=OFF -Dlzma=OFF -Dzstd=OFF -Dwebp=OFF \
-Djpeg12=OFF \
-Dcxx=OFF \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_cimg
build_cimg()
{
task_run build_libpng
task_run build_libjpeg
task_run build_tiff
S_dir="CImg-${CIMG_SOURCE_VERSION}"
D_file="CImg_${CIMG_SOURCE_VERSION}.zip"
verified_download $D_file $CIMG_SOURCE_SHA256 \
https://cimg.eu/files/${D_file}
unzip -o $D_file
cp --archive --verbose "$S_dir/CImg.h" "$PREFIX/include/"
}
task_register build_phash
build_phash()
{
task_run build_cimg
S_dir="pHash"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$PHASH_SOURCE_REPO" "${PHASH_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -DPHASH_DYNAMIC=OFF -DPHASH_STATIC=ON -B. -H../pHash
make -j${HOST_NUM_CPU}
make install
popd
}
task_register fetch_jni_hpp
fetch_jni_hpp()
{
local rDir="supportlibs/jni.hpp/"
[ "$(ls "${RS_SRC_DIR}/${rDir}" | wc -l)" -gt "4" ] ||
git -C ${RS_SRC_DIR} submodule update --init ${rDir}
}
task_register build_mvptree
build_mvptree()
{
S_dir="mvptree"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$MVPTREE_SOURCE_REPO" "${MVPTREE_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
popd
}
task_register get_native_libs_toolchain_path
get_native_libs_toolchain_path()
{
echo ${NATIVE_LIBS_TOOLCHAIN_PATH}
}
task_register build_default_toolchain
build_default_toolchain()
{
task_run bootstrap_toolchain || return $?
task_run build_bzlib || return $?
task_run build_openssl || return $?
task_run build_sqlcipher || return $?
task_run build_rapidjson || return $?
task_run build_restbed || return $?
task_run build_udp-discovery-cpp || return $?
task_run build_xapian || return $?
task_run build_miniupnpc || return $?
task_run build_phash || return $?
task_run fetch_jni_hpp || return $?
task_run deduplicate_includes || return $?
task_run get_native_libs_toolchain_path || return $?
}
if [ "$1" == "" ]; then
rm -rf "$REPORT_DIR"
mkdir -p "$REPORT_DIR"
cat "$0" > "$REPORT_DIR/build_script"
env > "$REPORT_DIR/build_env"
build_default_toolchain
else
# do not delete report directory in this case so we can reuse material
# produced by previous run, like deduplicated includes
mkdir -p "$REPORT_DIR"
while [ "$1" != "" ] ; do
task_zap $1
task_run $1 || exit $?
shift
done
fi

48
build_scripts/GitlabCI/Android.Dockerfile Normal file

@ -0,0 +1,48 @@
ARG ANDROID_NDK_ARCH=arm
FROM registry.gitlab.com/retroshare/retroshare:android_${ANDROID_NDK_ARCH}_base
ENV APT_UNAT="--assume-yes --quiet"
RUN apt-get update $APT_UNAT && apt-get upgrade $APT_UNAT --show-upgraded
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
WORKDIR /RetroShare
RUN git remote add testing $REPO_URL && \
git fetch --tags testing $REPO_BRANCH && \
git reset --hard testing/$REPO_BRANCH && \
git --no-pager log --max-count 1
RUN rm -rf /jsonapi-generator-build ; mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
# ARG declared before FROM get wiped after it, so we need declaring it again
ARG ANDROID_NDK_ARCH=arm
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
ARG ANDROID_PLATFORM_VER=16
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle

35
build_scripts/GitlabCI/base.Dockerfile

@ -2,42 +2,43 @@
## force cloning a new
## To prepare an image suitable as base for Gitlab CI use
# docker build -t "${CI_REGISTRY_IMAGE}:base" --build-arg KEEP_SOURCE=true --build-arg REPO_DEPTH="" -f base.Dockerfile .
## Now you need to tag it so you can later push it
# docker tag ${ID_OF_THE_CREATED_IMAGE} registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
# export CI_REGISTRY_IMAGE="registry.gitlab.com/retroshare/retroshare:base"
# docker build -t "${CI_REGISTRY_IMAGE}" -f base.Dockerfile .
## To push it to gitlab CI registry you need first to login and the to push
# docker login registry.gitlab.com
# docker push registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
# docker push "${CI_REGISTRY_IMAGE}"
## To run the container
# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}:base" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
FROM ubuntu
ARG CACHEBUST=0
RUN \
apt-get update -y && apt-get upgrade -y && \
apt-get install -y build-essential libssl-dev libbz2-dev libsqlite3-dev \
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get upgrade -y -qq && \
apt-get install -y -qq build-essential cimg-dev libssl-dev libbz2-dev \
libsqlite3-dev \
libsqlcipher-dev libupnp-dev pkg-config libz-dev \
qt5-default libxapian-dev qttools5-dev doxygen rapidjson-dev \
git cmake curl
RUN git clone --depth 1 https://github.com/aetilius/pHash.git && \
rm -rf pHash-build && mkdir pHash-build && cd pHash-build && \
cmake -B. -H../pHash -DCMAKE_INSTALL_PREFIX=/usr && \
make -j$(nproc) && make install && cd .. && \
rm -rf pHash-build pHash
ARG FRESHCLONE=0
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
ARG REPO_DEPTH="--depth 2000"
ARG KEEP_SOURCE=false
RUN apt-get update -y && apt-get upgrade -y
RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
git fetch --tags && cd ..
RUN \
mkdir RetroShare-build && cd RetroShare-build && \
RUN mkdir RetroShare-build && cd RetroShare-build && \
qmake ../RetroShare \
CONFIG+=no_retroshare_plugins CONFIG+=ipv6 \
CONFIG+=no_retroshare_plugins \
CONFIG+=retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi CONFIG+=rs_deep_search && \
(make -j$(nproc) || make -j$(nproc) || make) && make install && \
cd .. && rm -rf RetroShare-build && ($KEEP_SOURCE || rm -rf RetroShare)
cd .. && rm -rf RetroShare-build

9
build_scripts/GitlabCI/getRepoArgs.sh Executable file

@ -0,0 +1,9 @@
#!/bin/sh
[ -n "$CI_MERGE_REQUEST_ID" ] &&
echo \
--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
echo \
--build-arg REPO_URL="$CI_REPOSITORY_URL" \
--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

2
build_scripts/OBS

@ -1 +1 @@
Subproject commit 6d6ee4aab171c23c603f7e813ba66d22192393ad
Subproject commit df16cb915465d058c75277678799ce4dadeae287

6
build_scripts/Windows-msys2/build/pack.bat

@ -147,8 +147,10 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
call :copy_dependencies "%%D" "%RsDeployPath%"
)
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

13
build_scripts/Windows-msys2/env/tools/prepare-msys2.bat vendored

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
%cecho% info "Remove previous MSYS2 version"
call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
@ -41,12 +44,10 @@ if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download M
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows-msys2/env/tools/root/update-msys2.bat vendored

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi

@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${DEPLOYDIR}\qss\*.*"
File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

6
build_scripts/Windows/build/pack.bat

@ -132,8 +132,10 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

17
build_scripts/Windows/env/tools/prepare-msys2.bat vendored

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
set CMakeInstall=cmake-3.19.0-win32-x86.zip
set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.19.0/%CMakeInstall%
set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
@ -64,17 +67,11 @@ if "%FoundProfile%"=="0" (
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Update keyring"
"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows/env/tools/root/update-msys2.bat vendored

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows/installer/retroshare-Qt5.nsi

@ -334,7 +334,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

19
libbitdht/CMakeLists.txt Normal file

@ -0,0 +1,19 @@
# RetroShare decentralized communication platform
#
# Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
#
# SPDX-License-Identifier: CC0-1.0
cmake_minimum_required (VERSION 2.8.12)
project(libbitdht)
file(
GLOB BITDHT_SOURCES
src/bitdht/*.c src/bitdht/*.cc src/udp/*.cc src/util/*.cc )
add_library(${PROJECT_NAME} ${BITDHT_SOURCES})
target_include_directories(
${PROJECT_NAME}
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src )

243
libretroshare/CMakeLists.txt Normal file

@ -0,0 +1,243 @@
# RetroShare decentralized communication platform
#
# Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
#
# SPDX-License-Identifier: CC0-1.0
cmake_minimum_required (VERSION 3.18.0)
project(retroshare)
# sqlcipher
option(
RS_SQLCIPHER
"SQLCipher encryption for GXS database"
ON )
# rs_gxs_send_all
option(
RS_GXS_SEND_ALL
"GXS distribute all available messages on request, indipendently from \
local sync timer"
ON )
# bitdht
option(
RS_BITDHT
"Use bitdht (BitTorrent DHT own implementation) to look for online peers"
ON )
# use_dht_stunner
option(
RS_BITDHT_STUNNER
"Use bitdht (BitTorrent DHT own implementation) for NAT type discovery and \
attempt the STUN (Session Traversal Utilities for NAT)"
ON )
# use_dht_stunner_ext_ip
option(
RS_BITDHT_STUNNER_EXT_IP
"Use bitdht (BitTorrent DHT own implementation) stunner to figure out our \
external IP. As this purely relying on random DHT peers that answer our \
request, it can easily be abused. Therefore, it is turned off by default."
OFF )
# rs_jsonapi
option(
RS_JSON_API
"Use restbed to expose libretroshare as JSON API via HTTP"
OFF )
# rs_deep_forums_index
option(
RS_FORUM_DEEP_INDEX
"Xapian based full text index and search of GXS forums"
OFF )
# rs_broadcast_discovery
option(
RS_BRODCAST_DISCOVERY
"Local area network peer discovery via udp-discovery-cpp"
ON )
# rs_dh_init_check
option(
RS_DH_PRIME_INIT_CHECK
"Check Diffie Hellman prime at each startup. This is not necessary and on \
all Android mobile phones tested this take at least one minute at startup \
which is untolerable for most phone users."
ON )
option(
RS_MINIUPNPC
"Forward ports in NAT router via miniupnpc"
ON
)
include(CMakeDependentOption)
cmake_dependent_option(
RS_LIBUPNP
"Forward ports in NAT router via libupnp (unstable)"
OFF
"NOT RS_MINIUPNPC"
OFF
)
option(
RS_LIBRETROSHARE_STATIC
"Build RetroShare static library"
ON
)
cmake_dependent_option(
RS_LIBRETROSHARE_SHARED
"Build RetroShare shared library"
OFF
"NOT RS_LIBRETROSHARE_STATIC"
OFF
)
# rs_deprecatedwarning
option(
RS_WARN_DEPRECATED
"Print warning about RetroShare deprecated components usage during build"
ON
)
# rs_cppwarning
option(
RS_WARN_LESS
"Silence a few at the moment very common warnings about RetroShare \
components during build"
OFF
)
# rs_v07_changes
option(
RS_V07_BREAKING_CHANGES
"Enable retro-compatibility breaking changes planned for RetroShare 0.7.0"
OFF
)
set(
RS_DATA_DIR
"${CMAKE_INSTALL_PREFIX}/share/retroshare"
CACHE STRING
"Path where to install RetroShare system wide data" )
################################################################################
include(src/CMakeLists.txt)
list(TRANSFORM RS_SOURCES PREPEND src/)
if(RS_LIBRETROSHARE_STATIC)
add_library(${PROJECT_NAME} STATIC ${RS_SOURCES})
endif(RS_LIBRETROSHARE_STATIC)
if(RS_LIBRETROSHARE_SHARED)
add_library(${PROJECT_NAME} SHARED ${RS_SOURCES})
## Ensure statically linked libraries such as openpgpsdk are compiled with
## PIC Which is needed for shared library
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif(RS_LIBRETROSHARE_SHARED)
target_include_directories(
${PROJECT_NAME}
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src )
find_package(OpenSSL REQUIRED)
target_include_directories(${PROJECT_NAME} PRIVATE ${OPENSSL_INCLUDE_DIR})
target_link_libraries(${PROJECT_NAME} PRIVATE OpenSSL::SSL OpenSSL::Crypto)
################################################################################
add_subdirectory(../openpgpsdk ${CMAKE_BINARY_DIR}/openpgpsdk)
target_link_libraries(${PROJECT_NAME} PRIVATE openpgpsdk)
if(RS_BITDHT)
add_compile_definitions(RS_USE_BITDHT)
add_subdirectory(../libbitdht ${CMAKE_BINARY_DIR}/libbitdht)
target_link_libraries(${PROJECT_NAME} PRIVATE libbitdht)
endif(RS_BITDHT)
## TODO: Check if https://github.com/rbock/sqlpp11 or
## https://github.com/rbock/sqlpp17 may improve GXS code
if(RS_SQLCIPHER)
find_library(RS_SQL_LIB "sqlcipher" REQUIRED)
find_path(
RS_SQL_LIB_INCLUDE "sqlcipher/sqlite3.h"
PATH_SUFFIXES "include" "includes"
REQUIRED )
target_include_directories(
${PROJECT_NAME}
PRIVATE "${RS_SQL_LIB_INCLUDE}/sqlcipher" )
target_link_libraries(${PROJECT_NAME} PRIVATE ${RS_SQL_LIB})
else()
add_compile_definitions(NO_SQLCIPHER)
find_package(SQLite3 REQUIRED)
target_link_libraries(${PROJECT_NAME} PRIVATE SQLite::SQLite3)
endif()
add_compile_definitions(
SQLITE_HAS_CODEC
RS_ENABLE_GXS
GXS_ENABLE_SYNC_MSGS
RS_USE_GXS_DISTANT_SYNC
RS_GXS_TRANS
V07_NON_BACKWARD_COMPATIBLE_CHANGE_001
V07_NON_BACKWARD_COMPATIBLE_CHANGE_002
V07_NON_BACKWARD_COMPATIBLE_CHANGE_003 )
if(RS_V07_BREAKING_CHANGES)
add_compile_definitions(
V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
V07_NON_BACKWARD_COMPATIBLE_CHANGE_UNNAMED )
endif()
if(RS_MINIUPNPC)
add_compile_definitions(RS_USE_LIBMINIUPNPC)
endif(RS_MINIUPNPC)
if(RS_LIBUPNP)
add_compile_definitions(RS_USE_LIBUPNP)
endif(RS_LIBUPNP)
if(RS_GXS_SEND_ALL)
add_compile_definitions(RS_GXS_SEND_ALL)
endif(RS_GXS_SEND_ALL)
if(RS_BRODCAST_DISCOVERY)
add_subdirectory(
../supportlibs/udp-discovery-cpp/
${CMAKE_BINARY_DIR}/supportlibs/udp-discovery-cpp/ )
target_link_libraries(${PROJECT_NAME} PRIVATE udp-discovery)
## Temporary work around target_include_directories should be added upstream
include_directories(../supportlibs/udp-discovery-cpp/)
endif(RS_BRODCAST_DISCOVERY)
if(NOT RS_WARN_DEPRECATED)
add_compile_definitions(RS_NO_WARN_DEPRECATED)
target_compile_options(
${PROJECT_NAME} PRIVATE
-Wno-deprecated -Wno-deprecated-declarations )
endif(RS_WARN_DEPRECATED)
if(RS_WARN_LESS)
add_compile_definitions(RS_NO_WARN_CPP)
target_compile_options(
${PROJECT_NAME} PRIVATE
-Wno-cpp -Wno-inconsistent-missing-override )
endif(RS_WARN_LESS)
################################################################################
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_compile_definitions(RS_DATA_DIR="${RS_DATA_DIR}")
endif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
## Useful to debug CMake
#set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

346
libretroshare/src/CMakeLists.txt Normal file

@ -0,0 +1,346 @@
# RetroShare decentralized communication platform
#
# Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
#
# SPDX-License-Identifier: CC0-1.0
list(
APPEND RS_SOURCES
chat/distantchat.cc
chat/p3chatservice.cc
chat/rschatitems.cc
chat/distributedchat.cc
crypto/chacha20.cpp
crypto/hashstream.cc
crypto/rsaes.cc
crypto/rscrypto.cpp )
if(RS_BITDHT)
list(
APPEND RS_SOURCES
dht/connectstatebox.cc
dht/p3bitdht.cc
dht/p3bitdht_interface.cc
dht/p3bitdht_peernet.cc
dht/p3bitdht_peers.cc
dht/p3bitdht_relay.cc )
endif(RS_BITDHT)
list(
APPEND RS_SOURCES
file_sharing/filelist_io.cc
file_sharing/rsfilelistitems.cc
file_sharing/file_tree.cc
file_sharing/directory_updater.cc
file_sharing/p3filelists.cc
file_sharing/hash_cache.cc
file_sharing/dir_hierarchy.cc
file_sharing/directory_storage.cc
ft/ftchunkmap.cc
ft/ftfilecreator.cc
ft/ftfileprovider.cc
ft/ftfilesearch.cc
ft/ftturtlefiletransferitem.cc
ft/fttransfermodule.cc
ft/ftcontroller.cc
ft/ftdatamultiplex.cc
ft/ftextralist.cc
ft/ftserver.cc )
list(
APPEND RS_SOURCES
grouter/groutermatrix.cc
grouter/grouteritems.cc
grouter/p3grouter.cc )
list(
APPEND RS_SOURCES
gxs/rsgxsdata.cc
gxs/rsgxsrequesttypes.cc
gxs/gxssecurity.cc
gxs/gxstokenqueue.cc
gxs/rsdataservice.cc
gxs/rsgxsdataaccess.cc
gxs/rsgxsnetutils.cc
gxs/rsgxsnettunnel.cc
gxs/rsgxsutil.cc
gxs/rsnxsobserver.cpp
gxs/rsgenexchange.cc
gxs/rsgxsnetservice.cc )
list(
APPEND RS_SOURCES
gxstrans/p3gxstransitems.cc
gxstrans/p3gxstrans.cc )
list(
APPEND RS_SOURCES
gxstunnel/rsgxstunnelitems.cc
gxstunnel/p3gxstunnel.cc )
if(RS_JSON_API)
list(
APPEND RS_SOURCES
jsonapi/jsonapi.cpp )
endif(RS_JSON_API)
list(
APPEND RS_SOURCES
pgp/pgpkeyutil.cc
pgp/pgpauxutils.cc
pgp/pgphandler.cc
pgp/rscertificate.cc )
#./plugins/dlfcn_win32.cc
#./plugins/dlfcn_win32.h
#./plugins/pluginmanager.h
#./plugins/rscacheservice.h
#./plugins/rspqiservice.h
#./plugins/pluginmanager.cc
list(
APPEND RS_SOURCES
pqi/pqibin.cc
pqi/pqiipset.cc
pqi/pqiloopback.cc
pqi/pqimonitor.cc
pqi/pqipersongrp.cc
pqi/pqiqos.cc
pqi/pqiqosstreamer.cc
pqi/pqisslproxy.cc
pqi/pqistore.cc
pqi/authgpg.cc
pqi/p3cfgmgr.cc
pqi/p3notify.cc
pqi/p3servicecontrol.cc
pqi/pqinetstatebox.cc
pqi/pqiperson.cc
pqi/pqiservice.cc
pqi/pqissllistener.cc
pqi/pqissludp.cc
pqi/pqithreadstreamer.cc
pqi/sslfns.cc
pqi/authssl.cc
pqi/p3historymgr.cc
pqi/p3linkmgr.cc
pqi/pqihandler.cc
pqi/pqistreamer.cc
pqi/p3netmgr.cc
pqi/p3peermgr.cc
pqi/pqinetwork.cc
pqi/pqissl.cc
pqi/pqisslpersongrp.cc )
#./pqi/pqissli2psam3.cpp
#./pqi/pqissli2psam3.h
list(
APPEND RS_SOURCES
rsitems/rsbanlistitems.cc
rsitems/rsbwctrlitems.cc
rsitems/rsconfigitems.cc
rsitems/rsfiletransferitems.cc
rsitems/rsgxscommentitems.cc
rsitems/rsgxsforumitems.cc
rsitems/rsgxsiditems.cc
rsitems/rsgxsrecognitems.cc
rsitems/rsgxsreputationitems.cc
rsitems/rsgxsupdateitems.cc
rsitems/rshistoryitems.cc
rsitems/rsrttitems.cc
rsitems/rsserviceinfoitems.cc )
#./rsitems/rswikiitems.cc
#./rsitems/rswikiitems.h
#./rsitems/rswireitems.h
list(
APPEND RS_SOURCES
rsitems/rsgxschannelitems.cc
rsitems/rsgxscircleitems.cc
rsitems/rsgxsitems.cc
rsitems/rsmsgitems.cc )
#./rsitems/rsphotoitems.cc
#./rsitems/rsphotoitems.h
#./rsitems/rsposteditems.cc
#./rsitems/rsposteditems.h
#./rsitems/rswireitems.cc
list(
APPEND RS_SOURCES
rsitems/rsnxsitems.cc )
list(
APPEND RS_SOURCES
rsserver/p3status.cc
rsserver/p3face-config.cc
rsserver/p3face-info.cc
rsserver/p3history.cc
rsserver/p3serverconfig.cc
rsserver/rsloginhandler.cc
rsserver/p3face-server.cc
rsserver/p3msgs.cc
rsserver/p3peers.cc
rsserver/rsaccounts.cc
rsserver/rsinit.cc )
list(
APPEND RS_SOURCES
serialiser/rsbaseserial.cc
serialiser/rsserializable.cc
serialiser/rstlvaddrs.cc
serialiser/rstlvbanlist.cc
serialiser/rstlvbase.cc
serialiser/rstlvbinary.cc
serialiser/rstlvfileitem.cc
serialiser/rstlvgenericmap.inl
serialiser/rstlvgenericparam.cc
serialiser/rstlvidset.cc
serialiser/rstlvimage.cc
serialiser/rstlvitem.cc
serialiser/rstlvkeys.cc
serialiser/rstlvkeyvalue.cc
serialiser/rstlvstring.cc
serialiser/rsserializer.cc
serialiser/rstypeserializer.cc
serialiser/rsserial.cc )
# ./services/autoproxy
#./services/autoproxy/p3i2psam3.cpp
#./services/autoproxy/p3i2psam3.h
list(
APPEND RS_SOURCES
services/autoproxy/rsautoproxymonitor.cc
services/p3bwctrl.cc
services/p3heartbeat.cc
services/p3service.cc
services/p3serviceinfo.cc
services/p3statusservice.cc
services/p3banlist.cc
services/p3rtt.cc
services/rseventsservice.cc
services/p3gxscircles.cc
services/p3gxscommon.cc
services/p3gxsreputation.cc
services/p3msgservice.cc
services/p3idservice.cc
services/p3gxschannels.cc
services/p3gxsforums.cc )
#./services/p3wiki.cc
#./services/p3wiki.h
#./services/p3wire.cc
#./services/p3wire.h
#./services/p3photoservice.cc
#./services/p3photoservice.h
#./services/p3postbase.cc
#./services/p3postbase.h
#./services/p3posted.cc
#./services/p3posted.h
if(RS_BRODCAST_DISCOVERY)
list(
APPEND RS_SOURCES
services/broadcastdiscoveryservice.cc )
endif(RS_BRODCAST_DISCOVERY)
list(
APPEND RS_SOURCES
tcponudp/tcppacket.cc
tcponudp/tcpstream.cc
tcponudp/tou.cc
tcponudp/udppeer.cc
tcponudp/bss_tou.cc
tcponudp/udprelay.cc
tcponudp/udpstunner.cc )
list(
APPEND RS_SOURCES
turtle/rsturtleitem.cc
turtle/p3turtle.cc )
list(
APPEND RS_SOURCES
# util/contentvalue.cc
# util/exampletst.c
# util/rsdbbind.cc
# util/rsdiscspace.cc
util/rsexpr.cc
util/rsprint.cc
# util/rsrecogn.cc
# util/rssharedptr.h
# util/rstickevent.cc
util/rstime.cc
util/smallobject.cc
# util/retrodb.cc
util/rsbase64.cc
util/rsjson.cc
# util/rskbdinput.cc
util/rsrandom.cc
util/rsstring.cc
util/rsurl.cc
util/folderiterator.cc
util/rsdir.cc
util/dnsresolver.cc
util/extaddrfinder.cc
util/rsdebug.cc
util/rsdnsutils.cc
util/rsnet.cc
util/rsnet_ss.cc
util/rsthreads.cc )
# util/i2pcommon.cpp
# util/i2pcommon.h
if(RS_FORUM_DEEP_INDEX)
list(
APPEND RS_SOURCES
deep_search/commonutils.cpp
deep_search/forumsindex.cpp )
endif(RS_FORUM_DEEP_INDEX)
#./deep_search/filesflacindexer.hpp
#./deep_search/filesoggindexer.hpp
#./deep_search/filestaglibindexer.hpp
#./deep_search/filesindex.cpp
#./deep_search/filesindex.hpp
#./deep_search/channelsindex.cpp
#./deep_search/channelsindex.hpp
list(
APPEND RS_SOURCES
gossipdiscovery/gossipdiscoveryitems.cc
gossipdiscovery/p3gossipdiscovery.cc )
if(RS_MINIUPNPC)
list(
APPEND RS_SOURCES
rs_upnp/upnphandler_miniupnp.cc )
endif(RS_MINIUPNPC)
#./rs_upnp/UPnPBase.cpp
#./rs_upnp/upnphandler_libupnp.cc
#./rs_upnp/upnptest.cc
#./rs_upnp/upnputil.cc
#./rs_android/LocalArray.h
#./rs_android/README-ifaddrs-android.adoc
#./rs_android/ScopedFd.h
#./rs_android/androidcoutcerrcatcher.hpp
#./rs_android/errorconditionwrap.cpp
#./rs_android/ifaddrs-android.h
#./rs_android/org
#./rs_android/org/retroshare
#./rs_android/org/retroshare/service
#./rs_android/org/retroshare/service/AssetHelper.java
#./rs_android/org/retroshare/service/ErrorConditionWrap.java
#./rs_android/org/retroshare/service/RetroShareServiceAndroid.java
#./rs_android/retroshareserviceandroid.cpp
#./rs_android/retroshareserviceandroid.hpp
#./rs_android/rsjni.cpp
#./rs_android/rsjni.hpp

6
libretroshare/src/chat/distributedchat.cc

@ -1513,7 +1513,7 @@ ChatLobbyVirtualPeerId DistributedChatService::makeVirtualPeerId(ChatLobbyId lob
}
void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
bool DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
{
RsStackMutex stack(mDistributedChatMtx); /********** STACK LOCKED MTX ******/
@ -1525,10 +1525,12 @@ void DistributedChatService::denyLobbyInvite(const ChatLobbyId& lobby_id)
if(it == _lobby_invites_queue.end())
{
std::cerr << " (EE) lobby invite not in cache!!" << std::endl;
return ;
return false;
}
_lobby_invites_queue.erase(it) ;
return true;
}
bool DistributedChatService::joinVisibleChatLobby(const ChatLobbyId& lobby_id,const RsGxsId& gxs_id)

2
libretroshare/src/chat/distributedchat.h

@ -61,7 +61,7 @@ class DistributedChatService
void getChatLobbyList(std::list<ChatLobbyId>& clids) ;
bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& clinfo) ;
bool acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& identity) ;
void denyLobbyInvite(const ChatLobbyId& id) ;
bool denyLobbyInvite(const ChatLobbyId& id) ;
void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
void invitePeerToLobby(const ChatLobbyId&, const RsPeerId& peer_id,bool connexion_challenge = false) ;
void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;

191
libretroshare/src/deep_search/commonutils.cpp

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,54 +18,49 @@
* *
*******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
#ifndef XAPIAN_AT_LEAST
/// Added in Xapian 1.4.2.
#define XAPIAN_AT_LEAST(A,B,C) \
(XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
try
{
#if XAPIAN_AT_LEAST(1,3,2)
std::unique_ptr<Xapian::Database> dbPtr(
new Xapian::Database(path, flags) );
#else
std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
if(flags)
{
RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
"library version: ", XAPIAN_VERSION, " < 1.3.2" );
}
#endif
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
catch(Xapian::DatabaseOpeningError& e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
catch(Xapian::DatabaseLockError&)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
return date;
}
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(rstime_t(50), maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
// strip also CSS inside <style></style>
oSize = retVal.size();
auto styleTagBegin(retVal.find("<style"));
if(styleTagBegin < oSize)
{
auto styleEnd(retVal.find("</style>", styleTagBegin));
if(styleEnd < oSize)
retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
}
std::string::size_type oPos;
std::string::size_type cPos;
int itCount = 0;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
// Avoid infinite loop with crafty input
if(itCount > 1000)
{
RS_WARN( "Breaking stripping loop due to max allowed iterations ",
"rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
break;
}
++itCount;
}
return retVal;
}
}

34
libretroshare/src/deep_search/commonutils.hpp

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h>
#include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
}

83
libretroshare/src/deep_search/filesindex.cpp

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,47 +18,47 @@
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
if(db)
{
oldDoc = db.get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
if( pIt != db->postlist_end(idTerm) )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
oldDoc = db->get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
}
db.reset(); // Release DB read lock ASAP
}
Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
RS_DBG3(hash);
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
mWriteQueue.push([hash](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + hash.toStdString()); });
db->delete_document("Q" + hash.toStdString());
return true;
return std::error_condition();
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
std::error_condition DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}

23
libretroshare/src/deep_search/filesindex.hpp

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
@ -29,6 +26,9 @@
#include <map>
#include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
std::error_condition indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

210
libretroshare/src/deep_search/forumsindex.cpp Normal file

@ -0,0 +1,210 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/forumsindex.hpp"
#include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "retroshare/rsgxsforums.h"
#include "util/rsdebuglevel4.h"
std::error_condition DeepForumsIndex::search(
const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
{
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
using XQP = Xapian::QueryParser;
Xapian::Query query = queryparser.parse_query(
queryStr, XQP::FLAG_WILDCARD | XQP::FLAG_DEFAULT );
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepForumsSearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
{
RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
forumIndexId.setQueryKV(
RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
return forumIndexId.toString();
}
/*static*/ std::string DeepForumsIndex::postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
return postIndexId.toString();
}
std::error_condition DeepForumsIndex::indexForumGroup(
const RsGxsForumGroup& forum )
{
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
termgenerator.index_text(
DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
termgenerator.index_text(forum.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(forum.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(forum.mDescription);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink);
/* Store some fields for display purposes. Retrieved later to provide the
* matching snippet on search */
doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
/* We use the identifier to ensure each object ends up in the database only
* once no matter how many times we run the indexer.
* "Q" prefix is a Xapian convention for unique id term. */
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumFromIndex(
const RsGxsGroupId& grpId )
{
mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + forumIndexId(grpId)); });
return std::error_condition();
}
std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
{
RS_DBG4(post);
const auto& groupId = post.mMeta.mGroupId;
const auto& msgId = post.mMeta.mMsgId;
if(groupId.isNull() || msgId.isNull())
{
RS_ERR("Got post with invalid id ", post);
print_stacktrace();
return std::errc::invalid_argument;
}
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// Avoid indexing RetroShare-gui HTML tags
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
termgenerator.index_text(cleanMsg, 1, "XD" );
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(groupId, msgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

81
libretroshare/src/deep_search/forumsindex.hpp Normal file

@ -0,0 +1,81 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <system_error>
#include <vector>
#include <xapian.h>
#include "util/rstime.h"
#include "retroshare/rsgxsforums.h"
#include "retroshare/rsevents.h"
#include "deep_search/commonutils.hpp"
struct DeepForumsSearchResult
{
std::string mUrl;
double mWeight;
std::string mSnippet;
};
struct DeepForumsIndex
{
explicit DeepForumsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
std::error_condition search( const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results,
uint32_t maxResults = 100 );
std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
std::error_condition indexForumPost(const RsGxsForumMsg& post);
std::error_condition removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
static std::string dbDefaultPath();
private:
static std::string forumIndexId(const RsGxsGroupId& grpId);
static std::string postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
};

214
libretroshare/src/file_sharing/dir_hierarchy.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,6 +31,7 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -70,7 +73,8 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
mTotalFiles = 0 ;
}
bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
bool InternalFileHierarchyStorage::getDirHashFromIndex(
const DirectoryStorage::EntryIndex& index, RsFileHash& hash ) const
{
if(!checkIndex(index,FileStorageNode::TYPE_DIR))
return false ;
@ -79,6 +83,7 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E
return true;
}
bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mDirHashes.find(hash) ;
@ -88,35 +93,39 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di
index = it->second;
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them.
if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast<DirEntry*>(mNodes[index])->dir_hash != hash)
{
std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl;
mDirHashes.erase(it) ;
return false ;
}
/* make sure the hash actually points to some existing directory. If not,
* remove it. This is an opportunistic update of dir hashes: when we need
* them, we check them. */
if( !checkIndex(index, FileStorageNode::TYPE_DIR) ||
static_cast<DirEntry*>(mNodes[index])->dir_hash != hash )
{
RS_INFO("removing non existing dir hash: ", hash, " from dir hash list");
mDirHashes.erase(it);
return false;
}
return true;
}
bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
bool InternalFileHierarchyStorage::getIndexFromFileHash(
const RsFileHash& hash, DirectoryStorage::EntryIndex& index )
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mFileHashes.find(hash) ;
auto it = std::as_const(mFileHashes).find(hash);
if(it == mFileHashes.end()) return false;
if(it == mFileHashes.end())
return false;
index = it->second;
index = it->second;
/* make sure the hash actually points to some existing file. If not, remove
* it. This is an opportunistic update of file hashes: when we need them,
* we check them. */
if( !checkIndex(it->second, FileStorageNode::TYPE_FILE) ||
static_cast<FileEntry*>(mNodes[index])->file_hash != hash )
{
RS_INFO("removing non existing file hash: ", hash, " from file hash list");
mFileHashes.erase(it);
return false;
}
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them.
if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast<FileEntry*>(mNodes[index])->file_hash != hash)
{
std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl;
mFileHashes.erase(it) ;
return false ;
}
return true;
return true;
}
bool InternalFileHierarchyStorage::getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const
@ -155,10 +164,13 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
return e < mNodes.size() && mNodes[e] != NULL ;
}
bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::set<std::string>& subdirs, const RsFileHash& random_hash_seed)
bool InternalFileHierarchyStorage::updateSubDirectoryList(
const DirectoryStorage::EntryIndex& indx,
const std::set<std::string>& subdirs,
const RsFileHash& random_hash_seed )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
@ -284,10 +296,17 @@ bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,
return true;
}
bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
bool InternalFileHierarchyStorage::updateSubFilesList(
const DirectoryStorage::EntryIndex& indx,
const std::map<std::string,DirectoryStorage::FileTS>& subfiles,
std::map<std::string,DirectoryStorage::FileTS>& new_files )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
if(!checkIndex(indx, FileStorageNode::TYPE_DIR))
{
RS_ERR("indx: ", indx, std::errc::not_a_directory);
print_stacktrace();
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
new_files = subfiles ;
@ -312,9 +331,11 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
continue;
}
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size
// file is newer and/or has different size
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)
{
f.file_hash.clear(); // hash needs recomputing
// hash needs recomputing
f.file_hash.clear();
f.file_modtime = it->second.modtime;
f.file_size = it->second.size;
@ -342,13 +363,16 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
}
return true;
}
bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
bool InternalFileHierarchyStorage::updateHash(
const DirectoryStorage::EntryIndex& file_index, const RsFileHash& hash )
{
if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
{
std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
return false;
}
if(!checkIndex(file_index, FileStorageNode::TYPE_FILE))
{
RS_ERR( "Cannot update file at index ", file_index,
". Not a valid index, or not a file." );
print_stacktrace();
return false;
}
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl;
#endif
@ -436,14 +460,20 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
return mNodes.size()-1 ;
}
bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,rstime_t most_recent_time,rstime_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
bool InternalFileHierarchyStorage::updateDirEntry(
const DirectoryStorage::EntryIndex& indx, const std::string& dir_name,
rstime_t most_recent_time, rstime_t dir_modtime,
const std::vector<RsFileHash>& subdirs_hash,
const std::vector<FileEntry>& subfiles_array )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
{
std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
{
RS_ERR( "cannot update dir at index ", indx, ". Not a valid index, or "
"not an existing dir." );
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx]));
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl;
@ -703,14 +733,14 @@ const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorag
return NULL ;
}
const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::DirEntry*
InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return NULL ;
return static_cast<DirEntry*>(mNodes[indx]) ;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return nullptr;
return static_cast<DirEntry*>(mNodes[indx]);
}
const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::FileEntry*
InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
return NULL ;
@ -754,7 +784,10 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,DirectorySt
class DirectoryStorageExprFileEntry: public RsRegularExpression::ExpFileEntry
{
public:
DirectoryStorageExprFileEntry(const InternalFileHierarchyStorage::FileEntry& fe,const InternalFileHierarchyStorage::DirEntry& parent) : mFe(fe),mDe(parent) {}
DirectoryStorageExprFileEntry(
const InternalFileHierarchyStorage::FileEntry& fe,
const InternalFileHierarchyStorage::DirEntry& parent ) :
mFe(fe), mDe(parent) {}
inline virtual const std::string& file_name() const { return mFe.file_name ; }
inline virtual uint64_t file_size() const { return mFe.file_size ; }
@ -768,14 +801,18 @@ private:
const InternalFileHierarchyStorage::DirEntry& mDe ;
};
int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const
int InternalFileHierarchyStorage::searchBoolExp(
RsRegularExpression::Expression* exp,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL && exp->eval(
DirectoryStorageExprFileEntry(*static_cast<const FileEntry*>(mNodes[it->second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it->second]->parent_index])
)))
results.push_back(it->second);
for(auto& it: std::as_const(mFileHashes))
if(mNodes[it.second])
if(exp->eval(
DirectoryStorageExprFileEntry(
*static_cast<const FileEntry*>(mNodes[it.second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it.second]->parent_index])
) ))
results.push_back(it.second);
return 0;
}
@ -784,27 +821,43 @@ int InternalFileHierarchyStorage::searchTerms(
const std::list<std::string>& terms,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
// most entries are likely to be files, so we could do a linear search over the entries tab.
// instead we go through the table of hashes.
/* most entries are likely to be files, so we could do a linear search over
* the entries tab. Instead we go through the table of hashes.*/
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL)
{
const std::string &str1 = static_cast<FileEntry*>(mNodes[it->second])->file_name;
for(auto& it : std::as_const(mFileHashes))
{
// node may be null for some hash waiting to be deleted
if(mNodes[it.second])
{
rs_view_ptr<FileEntry> tFileEntry =
static_cast<FileEntry*>(mNodes[it.second]);
for(std::list<std::string>::const_iterator iter(terms.begin()); iter != terms.end(); ++iter)
{
/* always ignore case */
const std::string &str2 = (*iter);
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
std::string tFilename = tFileEntry->file_name;
if(tFileEntry->file_name.find("/") != std::string::npos)
{
std::string _tParentDir;
RsDirUtil::splitDirFromFile(
tFileEntry->file_name, _tParentDir, tFilename );
}
if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() ))
{
results.push_back(it->second);
break;
}
}
}
return 0 ;
for(auto& termIt : std::as_const(terms))
{
/* always ignore case */
if(tFilename.end() != std::search(
tFilename.begin(), tFilename.end(),
termIt.begin(), termIt.end(),
RsRegularExpression::CompareCharIC() ))
{
results.push_back(it.second);
break;
}
}
}
}
return 0;
}
bool InternalFileHierarchyStorage::check(std::string& error_string) // checks consistency of storage.
@ -958,8 +1011,9 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
{
std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
return false ;
RS_ERR(s);
print_stacktrace();
return false;
}
// Removes the given subdirectory from the parent node and all its pendign subdirs and files.

369
libretroshare/src/file_sharing/directory_storage.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,6 +32,7 @@
#include "directory_storage.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -67,8 +70,11 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
{ return mStorage->getSubFileIndex(mParentIndex, mFileTabIndex); }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
{ return mStorage->getSubDirIndex(mParentIndex, mDirTabIndex); }
DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
@ -139,7 +145,9 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx, const std:
mChanged = true ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
bool DirectoryStorage::updateSubFilesList(
const EntryIndex& indx, const std::map<std::string,FileTS>& subfiles,
std::map<std::string,FileTS>& new_files )
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
@ -348,7 +356,8 @@ int LocalDirectoryStorage::searchHash(const RsFileHash& hash, RsFileHash& real_h
return false ;
}
void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
void LocalDirectoryStorage::setSharedDirectoryList(
const std::list<SharedDirInfo>& lst )
{
std::set<std::string> dirs_with_new_virtualname ;
bool dirs_with_changed_flags = false ;
@ -379,7 +388,9 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo
virtual_names.insert(candidate_virtual_name) ;
}
// now for each member of the processed list, check if it is an existing shared directory that has been changed. If so, we need to update the dir TS of that directory
/* now for each member of the processed list, check if it is an existing
* shared directory that has been changed. If so, we need to update the
* dir TS of that directory */
std::map<std::string,SharedDirInfo> new_dirs ;
@ -529,7 +540,7 @@ bool LocalDirectoryStorage::updateHash(
#endif
ret = (!update_internal_hierarchy) ||
mFileHierarchy->updateHash(index,hash);
mFileHierarchy->updateHash(index, hash);
} // RS_STACK_MUTEX(mDirStorageMtx);
#ifdef RS_DEEP_FILES_INDEX
@ -538,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX
@ -617,48 +628,78 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil
return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
}
bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
bool LocalDirectoryStorage::locked_getFileSharingPermissions(
const EntryIndex& indx, FileStorageFlags& flags,
std::list<RsNodeGroupId>& parent_groups )
{
flags.clear() ;
parent_groups.clear();
flags.clear();
parent_groups.clear();
std::string base_dir;
/* We got a request for root directory no need to do anything more after
* clearing outputs */
if(!indx) return true;
const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
using FileStorageNode = InternalFileHierarchyStorage::FileStorageNode;
using EntryIndex = DirectoryStorage::EntryIndex;
if(n == NULL)
return false ;
rs_view_ptr<const FileStorageNode> n = mFileHierarchy->getNode(indx);
if(!n)
{
RS_ERR("Node for index: ", indx, "not found");
print_stacktrace();
return false;
}
for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
{
const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
// Climb down node tree up to root + 1
EntryIndex curIndex = indx;
while (n->parent_index)
{
curIndex = n->parent_index;
n = mFileHierarchy->getNode(curIndex);
}
if(e == NULL)
break ;
// Retrieve base name
std::string tBaseName;
switch (n->type())
{
// Handle single file shared case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE:
tBaseName = mFileHierarchy->getFileEntry(curIndex)->file_name;
break;
// Handle shared directory case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR:
tBaseName = mFileHierarchy->getDirEntry(curIndex)->dir_name;
break;
default:
RS_ERR("Got unhandled node type: ", n->type());
print_stacktrace();
return false;
}
if(e->parent_index == 0)
{
base_dir = e->dir_name ;
break ;
}
i = e->parent_index ;
}
// Use base name to retrieve sharing permissions
if(!tBaseName.empty())
{
auto it = std::as_const(mLocalDirs).find(tBaseName);
if(!base_dir.empty())
{
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
if(it == mLocalDirs.end())
{
RS_ERR( "base name \"", tBaseName,
"\" for index: ", indx, " not found in shared dir list." );
print_stacktrace();
return false;
}
if(it == mLocalDirs.end())
{
std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
return false ;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
else
{
RS_ERR("base name for indx: ", indx, " is empty");
print_stacktrace();
return false;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
return true;
return true;
}
std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const
@ -705,134 +746,214 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
return it->second.virtualname + "/" + res;
}
bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
bool LocalDirectoryStorage::serialiseDirEntry(
const EntryIndex& indx, RsTlvBinaryData& bindata,
const RsPeerId& client_id )
{
RS_STACK_MUTEX(mDirStorageMtx) ;
RS_STACK_MUTEX(mDirStorageMtx);
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
const InternalFileHierarchyStorage::DirEntry* dir =
mFileHierarchy->getDirEntry(indx);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl;
#endif
if(dir == NULL)
{
std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
return false;
}
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs ;
FileStorageFlags node_flags ;
std::list<RsNodeGroupId> node_groups ;
if(!dir)
{
RS_ERR("Cannot find entry ", indx);
return false;
}
// for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs;
FileStorageFlags node_flags;
std::list<RsNodeGroupId> node_groups;
/* for each subdir, compute the node flags and groups, then ask rsPeers to
* compute the mask that result from these flags for the particular peer
* supplied in parameter */
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (
locked_getFileSharingPermissions(
dir->subdirs[i], node_flags, node_groups ) &&
( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
{
RsFileHash hash;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
RS_ERR( "Cannot get hash from subdir index: ",
dir->subdirs[i], ". Weird bug." );
print_stacktrace();
return false;
}
allowed_subdirs.push_back(hash);
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
RsFileHash hash ;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
return false;
}
allowed_subdirs.push_back(hash) ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl;
#endif
}
}
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
else
std::cerr << " not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl;
#endif
// now count the files that do not have a null hash (meaning the hash has indeed been computed)
/* now count the files that do not have a null hash (meaning the hash has
* indeed been computed), also in case files are shared singularly (without
* a shared directory) so they are child of root check browsability
* permission */
uint32_t allowed_subfiles = 0;
for(uint32_t i=0; i<dir->subfiles.size(); ++i)
{
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
if(file != nullptr && !file->file_hash.isNull()
&& ( indx !=0 || (
locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ) &&
rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
allowed_subfiles++;
}
uint32_t allowed_subfiles = 0 ;
unsigned char* section_data = (unsigned char *)
rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!section_data) return false;
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file != NULL && !file->file_hash.isNull())
allowed_subfiles++ ;
}
uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
uint32_t section_offset = 0;
unsigned char *section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
/* we need to send:
* - the name of the directory, its TS
* - the index entry for each subdir (the updte TS are exchanged at a
* higher level)
* - the file info for each subfile */
if(!section_data)
return false ;
std::string virtual_dir_name = locked_getVirtualDirName(indx);
uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
uint32_t section_offset = 0;
/* Manual serialization AGAIN! This is terrible and should be ported to the
* new serialization system ASAP! */
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_DIR_NAME, virtual_dir_name ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RECURS_MODIF_TS,
(uint32_t)dir->dir_most_recent_time ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)dir->dir_modtime ))
{ free(section_data); return false;}
// we need to send:
// - the name of the directory, its TS
// - the index entry for each subdir (the updte TS are exchanged at a higher level)
// - the file info for each subfile
//
std::string virtual_dir_name = locked_getVirtualDirName(indx) ;
// serialise number of subdirs and number of subfiles
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subdirs.size() ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subfiles ))
{ free(section_data); return false; }
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) { free(section_data); return false ;}
// serialise subdirs entry indexes
for(uint32_t i=0; i<allowed_subdirs.size(); ++i)
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_ENTRY_INDEX, allowed_subdirs[i] ))
{ free(section_data); return false; }
// serialise number of subdirs and number of subfiles
// serialise directory subfiles, with info for each of them
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles )) { free(section_data); return false ;}
unsigned char* file_section_data =
(unsigned char *) rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!file_section_data)
{
free(section_data);
return false;
}
// serialise subdirs entry indexes
uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE;
for(uint32_t i=0;i<allowed_subdirs.size();++i)
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i] )) { free(section_data); return false ;}
for(uint32_t i=0; i<dir->subfiles.size(); ++i)
{
uint32_t file_section_offset = 0;
// serialise directory subfiles, with info for each of them
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
unsigned char *file_section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
if(file == nullptr || file->file_hash.isNull())
{
RS_INFO( "skipping unhashed or Null file entry ",
dir->subfiles[i], " to get/send file info." );
continue;
}
if(!file_section_data)
{
free(section_data);
return false ;
}
if(indx == 0)
{
if(!locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ))
{
RS_ERR( "Failure getting sharing permission for single file: ",
dir->subfiles[i] );
print_stacktrace();
continue;
}
uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE ;
if(!( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ))
{
RS_INFO( "Skipping single file shared without browse "
"permission" );
continue;
}
}
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
uint32_t file_section_offset = 0 ;
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_NAME, file->file_name ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SIZE, file->file_size ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SHA1_HASH, file->file_hash ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)file->file_modtime ))
{ free(section_data); free(file_section_data); return false; }
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file == NULL || file->file_hash.isNull())
{
std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
continue ;
}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) { free(section_data);free(file_section_data);return false ;}
// now write the whole string into a single section in the file
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) { free(section_data); free(file_section_data);return false ;}
// now write the whole string into a single section in the file
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,
file_section_data, file_section_offset ))
{ free(section_data); free(file_section_data); return false; }
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl;
#endif
}
free(file_section_data) ;
}
free(file_section_data);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
#endif
bindata.bin_data = realloc(section_data,section_offset) ; // This discards the possibly unused trailing bytes in the end of section_data
bindata.bin_len = section_offset ;
// Discards the possibly unused trailing bytes in the end of section_data
bindata.bin_data = realloc(section_data,section_offset);
bindata.bin_len = section_offset;
return true ;
return true;
}

356
libretroshare/src/file_sharing/directory_updater.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,13 +21,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
******************************************************************************/
#include "util/cxx17retrocompat.h"
#include "util/folderiterator.h"
#include "util/rstime.h"
#include "rsserver/p3face.h"
#include "directory_storage.h"
#include "directory_updater.h"
#include "file_sharing_defaults.h"
#include "util/rsdebuglevel3.h"
//#define DEBUG_LOCAL_DIR_UPDATER 1
@ -121,185 +125,238 @@ void LocalDirectoryUpdater::forceUpdate(bool add_safe_delay)
bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
{
if(mHashSalt.isNull())
{
std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
return false;
}
if(mHashSalt.isNull())
{
RS_ERR("no salt value in LocalDirectoryUpdater");
print_stacktrace();
return false;
}
mIsChecking = true ;
mIsChecking = true;
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
/* recursive update algorithm works that way:
* - the external loop starts on the shared directory list and goes through
* sub-directories
* - at the same time, it updates the local list of shared directories.
* A single sweep is performed over the whole directory structure.
* - the information that is costly to compute (the hash) is stored
* externally into a separate structure.
* - doing so, changing directory names or moving files between directories
* does not cause a re-hash of the content. */
std::list<SharedDirInfo> shared_directory_list;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
std::set<std::string> sub_dir_list;
std::set<std::string> sub_dir_list ;
/* Support also single files sharing as it make much more sense on some
* platforms like Android */
std::map<std::string, DirectoryStorage::FileTS> singleFilesMap;
// We re-check that each dir actually exists. It might have been removed from the disk.
/* We re-check that each dir actually exists. It might have been removed
* from the disk. Accept also single files not just directories. */
for(auto& realDir: std::as_const(shared_directory_list))
{
const auto& fPath = realDir.filename;
if(RsDirUtil::checkDirectory(fPath))
sub_dir_list.insert(fPath);
else if (RsDirUtil::fileExists(fPath))
{
rstime_t lastWrite= RsDirUtil::lastWriteTime(fPath);
if(time(nullptr) >= lastWrite + MIN_TIME_AFTER_LAST_MODIFICATION)
{
uint64_t fSize = 0;
RsDirUtil::checkFile(fPath,fSize);
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
if(RsDirUtil::checkDirectory( (*real_dir_it).filename ) )
sub_dir_list.insert( (*real_dir_it).filename ) ;
singleFilesMap[fPath].modtime = lastWrite;
singleFilesMap[fPath].size = fSize;
}
else
{
some_files_not_ready = true;
RS_INFO( "file: \"", fPath, "\" is "
"probably being written to. Keep it for later");
}
}
else RS_WARN( "Got non existent file \"", fPath,
"\" in shared directories list. Ignored." );
}
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
{
const auto tRoot = mSharedDirectories->root();
std::map<std::string, DirectoryStorage::FileTS> needsUpdate;
mSharedDirectories->updateSubFilesList(
tRoot, singleFilesMap, needsUpdate);
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
for( DirectoryStorage::FileIterator storedSingleFilesIt(
mSharedDirectories, mSharedDirectories->root() );
storedSingleFilesIt; ++storedSingleFilesIt )
{
const auto& it = storedSingleFilesIt;
RsFileHash hash;
if( mHashCache->requestHash(
it.name(), it.size(), it.modtime(), hash,
this, *it ) )
mSharedDirectories->updateHash(*it, hash, it.hash() != hash);
}
}
// now for each of them, go recursively and match both files and dirs
/* make sure that entries in stored_dir_it are the same than paths in
* real_dir_it, and in the same order. */
mSharedDirectories->updateSubDirectoryList(
mSharedDirectories->root(), sub_dir_list, mHashSalt );
std::set<std::string> existing_dirs ;
// now for each of them, go recursively and match both files and dirs
std::set<std::string> existing_dirs;
for( DirectoryStorage::DirIterator stored_dir_it(
mSharedDirectories, mSharedDirectories->root() );
stored_dir_it; ++stored_dir_it )
{
RS_DBG4("recursing into \"", stored_dir_it.name());
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
recursUpdateSharedDir(
stored_dir_it.name(), *stored_dir_it,
existing_dirs, 1, some_files_not_ready );
/* here we need to use the list that was stored, instead of the shared
* dir list, because the two are not necessarily in the same order. */
}
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
mIsChecking = false;
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
mIsChecking = false ;
return true ;
return true;
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
void LocalDirectoryUpdater::recursUpdateSharedDir(
const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,
std::set<std::string>& existing_directories, uint32_t current_depth,
bool& some_files_not_ready )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
RS_DBG4("parsing directory \"", cumulated_path, "\" index: ", indx);
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
/* make sure list of subdirs is the same
* make sure list of subfiles is the same
* request all hashes to the hashcache */
librs::util::FolderIterator dirIt(cumulated_path,mFollowSymLinks,false); // disallow symbolic links and files from the future.
// disallow symbolic links and files from the future.
librs::util::FolderIterator dirIt(cumulated_path, mFollowSymLinks, false);
rstime_t dir_local_mod_time ;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
return;
}
rstime_t dir_local_mod_time;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
RS_ERR("Cannot get local mod time for dir index: ", indx);
print_stacktrace();
return;
}
rstime_t now = time(NULL) ;
rstime_t now = time(nullptr);
/* the > is because we may have changed the virtual name, and therefore the
* TS wont match. We only want to detect when the directory has changed on
* the disk */
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)
{
// collect subdirs and subfiles
std::map<std::string, DirectoryStorage::FileTS> subfiles;
std::set<std::string> subdirs;
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match.
// we only want to detect when the directory has changed on the disk
{
// collect subdirs and subfiles
for( ; dirIt.isValid(); dirIt.next() )
if(filterFile(dirIt.file_name()))
{
const auto fType = dirIt.file_type();
switch(fType)
{
case librs::util::FolderIterator::TYPE_FILE:
if(now >= dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime();
subfiles[dirIt.file_name()].size = dirIt.file_size();
RS_DBG4("adding sub-file \"", dirIt.file_name(), "\"");
}
else
{
some_files_not_ready = true;
RS_INFO( "file: \"", dirIt.file_fullpath(), "\" is "
"probably being written to. Keep it for later");
}
break;
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true;
/* 64 is here as a safe limit, to make infinite loops
* impossible.
* TODO: Make it a visible constexpr in the header */
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth)
|| (mMaxShareDepth == 0 && current_depth >= 64) )
dir_is_accepted = false;
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::set<std::string> subdirs ;
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(
cumulated_path + "/" + dirIt.file_name() );
for(;dirIt.isValid();dirIt.next())
if(filterFile(dirIt.file_name()))
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE:
if( existing_directories.end() !=
existing_directories.find(real_path) )
{
RS_WARN( "Directory: \"", cumulated_path,
"\" has real path: \"", real_path,
"\" which already belongs to another "
"shared directory. Ignoring" );
dir_is_accepted = false;
}
else existing_directories.insert(real_path);
}
if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
else
{
some_files_not_ready = true ;
if(dir_is_accepted) subdirs.insert(dirIt.file_name());
std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being written to. Keeping it for later." << std::endl;
}
RS_DBG4("adding sub-dir \"", dirIt.file_name(), "\"");
break;
break;
}
default:
RS_ERR( "Got Dir entry of unknown type:", fType,
"with path \"", cumulated_path, "/",
dirIt.file_name(), "\"" );
print_stacktrace();
break;
}
}
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true ;
/* update folder modificatoin time, which is the only way to detect
* e.g. removed or renamed files. */
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime());
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64)) // 64 is here as a safe limit, to make loops impossible.
dir_is_accepted = false ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt);
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
std::map<std::string, DirectoryStorage::FileTS> new_files;
mSharedDirectories->updateSubFilesList(indx, subfiles, new_files);
if(existing_directories.end() != existing_directories.find(real_path))
{
std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
dir_is_accepted = false ;
}
else
existing_directories.insert(real_path) ;
}
if(dir_is_accepted)
subdirs.insert(dirIt.file_name());
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
RsFileHash hash ;
// mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed.
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit))
mSharedDirectories->updateHash(*dit,hash,hash != dit.hash());
}
}
#ifdef DEBUG_LOCAL_DIR_UPDATER
else
std::cerr << " directory is unchanged. Keeping existing files and subdirs list." << std::endl;
#endif
// go through the list of sub-dirs and recursively update
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
// now go through list of subfiles and request the hash to hashcache
for( DirectoryStorage::FileIterator dit(mSharedDirectories,indx);
dit; ++dit )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
/* ask about the hash. If not present, ask HashCache.
* If not present, or different, the callback will update it. */
RsFileHash hash;
/* mSharedDirectories does two things: store H(F), and
* compute H(H(F)), which is used in FT.
* The later is always needed. */
if( mHashCache->requestHash(
cumulated_path + "/" + dit.name(),
dit.size(), dit.modtime(), hash, this, *dit ) )
mSharedDirectories->updateHash(*dit, hash, hash != dit.hash());
}
}
// go through the list of sub-dirs and recursively update
for( DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories, indx);
stored_dir_it; ++stored_dir_it )
recursUpdateSharedDir( cumulated_path + "/" + stored_dir_it.name(),
*stored_dir_it, existing_directories,
current_depth+1, some_files_not_ready );
}
bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@ -347,7 +404,8 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
{
return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
return mSharedDirectories->getEntryType(
DirectoryStorage::EntryIndex(client_param) ) == DIR_TYPE_FILE;
}
void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)

25
libretroshare/src/file_sharing/hash_cache.cc

@ -187,9 +187,10 @@ void HashStorage::threadTick()
else
rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
//RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
if(rsEvents)
{
/* Emit deprecated event only for retrocompatibility
* TODO: create a proper event with structured data instead of a
* formatted string */
auto ev = std::make_shared<RsSharedDirectoriesEvent>();
ev->mEventCode = RsSharedDirectoriesEventCode::HASHING_FILE;
ev->mMessage = tmpout;
@ -198,7 +199,7 @@ void HashStorage::threadTick()
double seconds_origin = rstime::RsScopeTimer::currentTime() ;
if(RsDirUtil::getFileHash(job.full_path, hash,size, this))
if(RsDirUtil::getFileHash(job.full_path, hash, size, this))
{
// store the result
@ -218,8 +219,7 @@ void HashStorage::threadTick()
mChanged = true ;
mTotalHashedSize += size ;
}
else
std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
else RS_ERR("Failure hashing file: ", job.full_path);
mHashingTime += rstime::RsScopeTimer::currentTime() - seconds_origin ;
mHashedBytes += size ;
@ -233,11 +233,18 @@ void HashStorage::threadTick()
++mHashCounter ;
}
}
// call the client
}
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
// call the client
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
/* Notify we completed hashing a file */
auto ev = std::make_shared<RsFileHashingCompletedEvent>();
ev->mFilePath = job.full_path;
ev->mHashingSpeed = mCurrentHashingSpeed;
ev->mFileHash = hash;
rsEvents->postEvent(ev);
}
bool HashStorage::requestHash(const std::string& full_path,uint64_t size,rstime_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)

344
libretroshare/src/file_sharing/p3filelists.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,7 +32,7 @@
#include "retroshare/rsids.h"
#include "retroshare/rspeers.h"
#include "retroshare/rsinit.h"
#include "util/cxx17retrocompat.h"
#include "rsserver/p3face.h"
#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : "
@ -1044,31 +1046,36 @@ void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIn
}
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
int p3FileDatabase::RequestDirDetails(
void* ref, DirDetails& d, FileSearchFlags flags ) const
{
RS_STACK_MUTEX(mFLSMtx) ;
RS_STACK_MUTEX(mFLSMtx);
d.children.clear();
d.children.clear();
// Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
// or at the top of our own list of shared directories, depending on the flags.
/* Case where the pointer is NULL, which means we're at the top of the list
* of shared directories for all friends (including us) or at the top of our
* own list of shared directories, depending on the flags.
*
* Friend index is used as follows:
* 0 : own id
* 1...n : other friends
*
* entry_index: starts at 0.
*
* The point is: we cannot use (0,0) because it encodes to NULL. No existing
* combination should encode to NULL.
* So we need to properly convert the friend index into 0 or into a friend
* tab index in mRemoteDirectories.
*
* We should also check the consistency between flags and the content of ref.
*/
// Friend index is used as follows:
// 0 : own id
// 1...n : other friends
//
// entry_index: starts at 0.
//
// The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
// So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
//
// We should also check the consistency between flags and the content of ref.
if (ref == NULL)
{
d.ref = NULL ;
d.type = DIR_TYPE_ROOT;
d.parent = NULL;
if (ref == nullptr)
{
d.ref = nullptr;
d.type = DIR_TYPE_ROOT;
d.parent = nullptr;
d.prow = -1;
d.name = "root";
d.hash.clear() ;
@ -1078,12 +1085,13 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
d.max_mtime = 0 ;
if(flags & RS_FILE_HINTS_LOCAL)
{
void *p;
{
void *p = nullptr;
{
convertEntryIndexToPointer<sizeof(void*)>(0,0,p); // root of own directories
DirStub stub;
{
// root of own directories
convertEntryIndexToPointer<sizeof(void*)>(0, 0, p);
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = mServCtrl->getOwnId().toStdString();
stub.ref = p;
@ -1092,9 +1100,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(mExtraFiles->size() > 0)
{
convertEntryIndexToPointer<sizeof(void*)>(0,1,p); // local shared files from extra list
DirStub stub;
stub.type = DIR_TYPE_PERSON; // not totally exact, but used as a trick.
// local shared files from extra list
convertEntryIndexToPointer<sizeof(void*)>(0, 1, p);
DirStub stub;
// not totally exact, but used as a trick.
stub.type = DIR_TYPE_PERSON;
stub.name = "[Extra List]";
stub.ref = p;
@ -1127,18 +1137,19 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
}
uint32_t fi;
DirectoryStorage::EntryIndex e ;
DirectoryStorage::EntryIndex e;
convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
{
P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
return false ;
}
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||
(fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
{
RS_ERR("Remote request on local index or local request on remote index");
return false;
}
if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list
if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list
{
getExtraFilesDirDetails(ref,e,d);
@ -1150,25 +1161,28 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
}
DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
DirectoryStorage* storage =
(flags & RS_FILE_HINTS_LOCAL) ?
((DirectoryStorage*) mLocalSharedDirs) :
((DirectoryStorage*) mRemoteDirectories[fi-1]);
// Case where the index is the top of a single person. Can be us, or a friend.
/* Case where the index is the top of a single person.
* Can be us, or a friend. */
if(!storage || !storage->extractData(e,d))
{
RS_WARN( "request on index; ", e, ", for directory ID:",
( (!storage)? ("[NULL]") : (storage->peerId().toStdString()) ),
" failed" );
return false;
}
if(storage==NULL || !storage->extractData(e,d))
{
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
#endif
return false ;
}
/* update indexes. This is a bit hacky, but does the job. The cast to
* intptr_t is the proper way to convert a pointer into an int. */
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref);
// update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
// a pointer into an int.
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref) ;
for(uint32_t i=0;i<d.children.size();++i)
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
for(uint32_t i=0; i<d.children.size(); ++i)
convertEntryIndexToPointer<sizeof(void*)>(
(intptr_t) d.children[i].ref, fi, d.children[i].ref );
if(e == 0) // root
{
@ -1178,9 +1192,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
else
{
if(d.parent == 0) // child of root node
d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1);
d.prow = (flags & RS_FILE_HINTS_LOCAL) ? 0 : (fi-1);
else
d.prow = storage->parentRow(e) ;
d.prow = storage->parentRow(e);
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.parent,fi,d.parent) ;
}
@ -1307,7 +1321,9 @@ uint32_t p3FileDatabase::watchPeriod()
return mLocalDirWatcher->fileWatchPeriod();
}
int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
int p3FileDatabase::SearchKeywords(
const std::list<std::string>& keywords, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& client_peer_id )
{
if(flags & RS_FILE_HINTS_LOCAL)
{
@ -1319,15 +1335,15 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
mLocalSharedDirs->searchTerms(keywords,firesults) ;
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
pointers.push_back(p) ;
}
for(auto& it: std::as_const(firesults))
{
void *p = nullptr;
convertEntryIndexToPointer<sizeof(void*)>(it, 0, p);
pointers.push_back(p);
}
}
filterResults(pointers,results,flags,client_peer_id) ;
filterResults(pointers, results, flags, client_peer_id);
}
if(flags & RS_FILE_HINTS_REMOTE)
@ -1476,7 +1492,9 @@ bool p3FileDatabase::search(
return false;
}
int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
int p3FileDatabase::filterResults(
const std::list<void*>& firesults, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& peer_id ) const
{
results.clear();
@ -1484,29 +1502,33 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
/* translate/filter results */
for(std::list<void*>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
{
DirDetails cdetails ;
for(void* rit: std::as_const(firesults))
{
DirDetails cdetails;
if(!RequestDirDetails (*rit,cdetails,RS_FILE_HINTS_LOCAL))
{
P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << *rit << std::endl;
continue ;
}
if(!RequestDirDetails(rit, cdetails, RS_FILE_HINTS_LOCAL))
{
RS_ERR("Cannot retrieve dir details for entry: ", rit);
print_stacktrace();
continue ;
}
RS_DBG( "Filtering candidate: ", rit,
", name: ", cdetails.name, ", path: ", cdetails.path,
", flags: ", cdetails.flags, ", peer: ", peer_id );
if(!peer_id.isNull())
{
FileSearchFlags permission_flags =
rsPeers->computePeerPermissionFlags(
peer_id, cdetails.flags, cdetails.parent_groups );
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
cdetails.id.clear();
results.push_back(cdetails);
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Filtering candidate " << *rit << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
#endif
if(!peer_id.isNull())
{
FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
cdetails.id.clear() ;
results.push_back(cdetails);
#ifdef DEBUG_P3FILELISTS
std::cerr << ": kept" << std::endl ;
std::cerr << ": kept" << std::endl ;
#endif
}
#ifdef DEBUG_P3FILELISTS
@ -1518,7 +1540,19 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
results.push_back(cdetails);
}
return !results.empty() ;
for(auto& res: results)
{
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
if(res.name.find("/") != std::string::npos )
{
std::string _tDirPath;
RsDirUtil::splitDirFromFile(res.name, _tDirPath, res.name);
}
}
return !results.empty();
}
bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath)
@ -1592,78 +1626,90 @@ void p3FileDatabase::tickSend()
void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
{
RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem;
RsFileListsSyncResponseItem* ritem = new RsFileListsSyncResponseItem;
// look at item TS. If local is newer, send the full directory content.
{
RS_STACK_MUTEX(mFLSMtx) ;
// look at item TS. If local is newer, send the full directory content.
{
RS_STACK_MUTEX(mFLSMtx);
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
#endif
RS_DBG( "Received directory sync request from peer ", item->PeerId(),
". hash=", item->entry_hash, ", flags=", item->flags,
", request id: ", item->request_id, ", last known TS: ",
item->last_known_recurs_modf_TS );
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
RS_DBG("Cannot find entry index for hash ", item->entry_hash,
" cannot respond to sync request." );
return;
}
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#endif
return;
}
uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index);
ritem->PeerId(item->PeerId());
ritem->request_id = item->request_id;
ritem->entry_hash = item->entry_hash;
uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ;
ritem->PeerId(item->PeerId()) ;
ritem->request_id = item->request_id;
ritem->entry_hash = item->entry_hash ;
std::list<RsNodeGroupId> node_groups;
FileStorageFlags node_flags;
std::list<RsNodeGroupId> node_groups;
FileStorageFlags node_flags;
if(entry_type != DIR_TYPE_DIR)
{
RS_DBG( "Directory does not exist anymore, or is not a directory, "
"or permission denied. Answering with proper flags." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else if( entry_index != 0 &&
(!mLocalSharedDirs->getFileSharingPermissions(
entry_index, node_flags,node_groups ) ||
!(rsPeers->computePeerPermissionFlags(
item->PeerId(), node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE)) )
{
RS_ERR("cannot get file permissions for entry index: ", entry_index,
", or permission denied." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else
{
rstime_t local_recurs_max_time;
mLocalSharedDirs->getDirectoryRecursModTime(
entry_index, local_recurs_max_time );
if(entry_type != DIR_TYPE_DIR)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else
{
rstime_t local_recurs_max_time ;
mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
/* normally, should be "<", but since we provided the TS it should
* be equal, so != is more robust. */
if(item->last_known_recurs_modf_TS != local_recurs_max_time)
{
RS_DBG( "Directory is more recent than what the friend knows. "
"Sending full dir content as response.");
if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
/* We supply the peer id, in order to possibly remove some
* subdirs, if entries are not allowed to be seen by this peer.
*/
mLocalSharedDirs->serialiseDirEntry(
entry_index, ritem->directory_content_data,
item->PeerId() );
}
else
{
RS_DBG3( "Directory is up to date w.r.t. what the friend knows."
" Sending ACK." );
// We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
}
else
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
}
}
}
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
ritem->last_known_recurs_modf_TS = local_recurs_max_time ;
}
}
}
// sends the response.
splitAndSendItem(ritem) ;
// sends the response.
splitAndSendItem(ritem);
}
void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem)

8
libretroshare/src/ft/ftcontroller.cc

@ -735,8 +735,7 @@ bool ftController::completeFile(const RsFileHash& hash)
RsDirUtil::splitDirFromFile(fc->mDestination,dst_dir,dst_file) ;
// We use this intermediate file in case the destination directory is not available, so as to not keep the partial file name.
std::string intermediate_file_name = src_dir+'/'+dst_file ;
std::string intermediate_file_name = RsDirUtil::makePath(src_dir, dst_file);
// I don't know how the size can be zero, but believe me, this happens,
// and it causes an error on linux because then the file may not even exist.
@ -1542,6 +1541,11 @@ std::string ftController::getPartialsDirectory()
return mPartialsPath;
}
bool ftController::FileServerCancel(const RsFileHash& hash)
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
return mDataplex->deleteServer(hash);
}
bool ftController::setDestinationDirectory(const RsFileHash& hash,const std::string& dest_dir)
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/

1
libretroshare/src/ft/ftcontroller.h

@ -157,6 +157,7 @@ public:
bool getFileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info);
bool setDestinationName(const RsFileHash& hash,const std::string& dest_name) ;
bool setDestinationDirectory(const RsFileHash& hash,const std::string& dest_name) ;
bool FileServerCancel(const RsFileHash& hash);
// Download speed
bool getPriority(const RsFileHash& hash,DwlSpeed& p);

17
libretroshare/src/ft/ftdatamultiplex.cc

@ -1039,6 +1039,23 @@ void ftDataMultiplex::handlePendingCrcRequests()
}
}
bool ftDataMultiplex::deleteServer(const RsFileHash& hash)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
auto sit = mServers.find(hash);
if(sit == mServers.end())
return false;
// We don't delete servers that are clients at the same time !
if(dynamic_cast<ftFileCreator*>(sit->second) == NULL)
delete sit->second;
mServers.erase(sit);
return true;
}
void ftDataMultiplex::deleteUnusedServers()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

6
libretroshare/src/ft/ftdatamultiplex.h

@ -112,9 +112,9 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
bool FileDownloads(std::list<RsFileHash> &hashs);
bool FileDetails(const RsFileHash &hash, FileSearchFlags hintsflag, FileInfo &info);
void deleteUnusedServers() ;
void handlePendingCrcRequests() ;
void deleteUnusedServers() ;
bool deleteServer(const RsFileHash& hash); // deletes FtServers for the given hash. Used when removing an extra file from shares.
void handlePendingCrcRequests() ;
/*************** SEND INTERFACE (calls ftDataSend) *******************/

17
libretroshare/src/ft/ftserver.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -817,7 +819,10 @@ bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t
}
bool ftServer::ExtraFileRemove(const RsFileHash& hash)
{ return mFileDatabase->removeExtraFile(hash); }
{
mFtController->FileServerCancel(hash);
return mFileDatabase->removeExtraFile(hash);
}
bool ftServer::ExtraFileHash( std::string localpath, rstime_t period, TransferRequestFlags flags )
{
@ -873,9 +878,11 @@ int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetai
{
return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
}
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
int ftServer::SearchKeywords(
std::list<std::string> keywords, std::list<DirDetails> &results,
FileSearchFlags flags, const RsPeerId& peer_id )
{
return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id);
return mFileDatabase->SearchKeywords(keywords, results, flags, peer_id);
}
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
@ -2030,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{
RsFileSearchResultItem resIt;

87
libretroshare/src/gxs/rsgenexchange.cc

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -40,6 +41,7 @@
#include "rsserver/p3face.h"
#include "retroshare/rsevents.h"
#include "util/radix64.h"
#include "util/cxx17retrocompat.h"
#define PUB_GRP_MASK 0x000f
#define RESTR_GRP_MASK 0x00f0
@ -193,47 +195,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60)
{
mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
}
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@ -362,12 +323,12 @@ void RsGenExchange::tick()
{
mIntegrityCheck = new RsGxsIntegrityCheck( mDataStore, this,
*mSerialiser, mGixs);
mIntegrityCheck->start("gxs integrity");
mChecking = true;
std::string thisName = typeid(*this).name();
mChecking = mIntegrityCheck->start("gxs IC4 "+thisName);
}
}
if(mIntegrityCheck->isDone())
if(mIntegrityCheck->isDone() || !mChecking)
{
std::vector<RsGxsGroupId> grpIds;
GxsMsgReq msgIds;
@ -2325,8 +2286,9 @@ bool RsGenExchange::processGrpMask(const RsGxsGroupId& grpId, ContentValue &grpC
void RsGenExchange::publishMsgs()
{
bool atLeastOneMessageCreatedSuccessfully = false;
RS_STACK_MUTEX(mGenMtx) ;
RS_STACK_MUTEX(mGenMtx);
rstime_t now = time(NULL);
@ -2503,6 +2465,8 @@ void RsGenExchange::publishMsgs()
// add to published to allow acknowledgement
mMsgNotify.insert(std::make_pair(mit->first, std::make_pair(grpId, msgId)));
mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::COMPLETE);
atLeastOneMessageCreatedSuccessfully = true;
}
else
{
@ -2536,6 +2500,8 @@ void RsGenExchange::publishMsgs()
mNotifications.push_back(ch);
}
if(atLeastOneMessageCreatedSuccessfully) mNetService->requestPull();
}
RsGenExchange::ServiceCreate_Return RsGenExchange::service_CreateGroup(RsGxsGrpItem* /* grpItem */,
@ -2717,10 +2683,15 @@ void RsGenExchange::processMessageDelete()
msgDeleted.push_back(note.msgIds);
}
for(const auto& msgreq:msgDeleted)
for(const auto& msgit:msgreq)
for(const auto& msg:msgit.second)
mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
/* Three nested for looks like a performance bomb, but as Cyril says here
* https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
* this should actually not explode at all because it is just one message at
* time that get notified */
for(const auto& msd : mMsgDeletePublish)
for(auto& msgMap : msd.mMsgs)
for(auto& msgId : msgMap.second)
mNotifications.push_back(
new RsGxsMsgDeletedChange(msgMap.first, msgId) );
mMsgDeletePublish.clear();
}
@ -2758,7 +2729,8 @@ bool RsGenExchange::checkKeys(const RsTlvSecurityKeySet& keySet)
void RsGenExchange::publishGrps()
{
std::list<RsGxsGroupId> groups_to_subscribe ;
bool atLeastOneGroupCreatedSuccessfully = false;
std::list<RsGxsGroupId> groups_to_subscribe;
{
RS_STACK_MUTEX(mGenMtx) ;
@ -2989,6 +2961,8 @@ void RsGenExchange::publishGrps()
// add to published to allow acknowledgement
toNotify.insert(std::make_pair(token, GrpNote(true,ggps.mIsUpdate,grpId)));
atLeastOneGroupCreatedSuccessfully = true;
}
}
@ -3007,9 +2981,14 @@ void RsGenExchange::publishGrps()
// This is done off-mutex to avoid possible cross deadlocks with the net service.
if(mNetService!=NULL)
for(std::list<RsGxsGroupId>::const_iterator it(groups_to_subscribe.begin());it!=groups_to_subscribe.end();++it)
mNetService->subscribeStatusChanged((*it),true) ;
if(mNetService != nullptr)
{
for(auto& grpId : std::as_const(groups_to_subscribe))
mNetService->subscribeStatusChanged(grpId, true);
if(atLeastOneGroupCreatedSuccessfully)
mNetService->requestPull();
}
}
uint32_t RsGenExchange::generatePublicToken()

292
libretroshare/src/gxs/rsgxsnetservice.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -33,7 +35,7 @@
// |
// +----------- sharePublishKeys()
// |
// +----------- syncWithPeers()
// +----------- checkUpdatesFromPeers()
// | |
// | +--if AutoSync--- send global UpdateTS of each peer to itself => the peer knows the last
// | | time current peer has received an updated from himself
@ -125,14 +127,14 @@
// (Set at server side to be mGrpServerUpdateItem->grpUpdateTS)
//
// Only updated in processCompletedIncomingTransaction() from Grp list transaction.
// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers: peer will send data if
// Used in checkUpdatesFromPeers() sending in RsNxsSyncGrp once to all peers: peer will send data if
// has something new. All time comparisons are in the friends' clock time.
//
// mClientMsgUpdateMap: map< RsPeerId, map<grpId,TimeStamp > >
//
// Last msg list modification time sent by that peer Id
// Updated in processCompletedIncomingTransaction() from Grp list trans.
// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers.
// Used in checkUpdatesFromPeers() sending in RsNxsSyncGrp once to all peers.
// Set at server to be mServerMsgUpdateMap[grpId]->msgUpdateTS
//
// mGrpServerUpdateItem: TimeStamp Last group local modification timestamp over all groups
@ -148,7 +150,7 @@
//
// tick() tick()
// | |
// +---- SyncWithPeers +-- recvNxsItemQueue()
// +---- checkUpdatesFromPeers() +-- recvNxsItemQueue()
// | |
// +---------------- Send global UpdateTS of each peer to itself => the peer knows +---------> +------ handleRecvSyncGroup( RsNxsSyncGrp*)
// | the last msg sent (stored in mClientGrpUpdateMap[peer_id]), | | - parse all subscribed groups. For each, send a RsNxsSyncGrpItem with publish TS
@ -455,7 +457,7 @@ int RsGxsNetService::tick()
if((elapsed) < now)
{
syncWithPeers();
checkUpdatesFromPeers();
syncGrpStatistics();
checkDistantSyncState();
@ -568,39 +570,40 @@ RsGxsGroupId RsGxsNetService::hashGrpId(const RsGxsGroupId& gid,const RsPeerId&
return RsGxsGroupId( RsDirUtil::sha1sum(tmpmem,SIZE).toByteArray() );
}
void RsGxsNetService::syncWithPeers()
std::error_condition RsGxsNetService::checkUpdatesFromPeers(
std::set<RsPeerId> peers )
{
#ifdef NXS_NET_DEBUG_0
GXSNETDEBUG___ << "RsGxsNetService::syncWithPeers() this=" << (void*)this << ". serviceInfo=" << mServiceInfo << std::endl;
RS_DBG("this=", (void*)this, ". serviceInfo=", mServiceInfo);
#endif
static RsNxsSerialiser ser(mServType) ; // this is used to estimate bandwidth.
RS_STACK_MUTEX(mNxsMutex) ;
std::set<RsPeerId> peers;
mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers);
if(mAllowDistSync && mGxsNetTunnel != NULL)
/* If specific peers are passed as paramether ask only to them */
if(peers.empty())
{
// Grab all online virtual peers of distant tunnels for the current service.
mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers);
std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
mGxsNetTunnel->getVirtualPeers(vpids);
if(mAllowDistSync && mGxsNetTunnel != nullptr)
{
/* Grab all online virtual peers of distant tunnels for the current
* service. */
for(auto it(vpids.begin());it!=vpids.end();++it)
peers.insert(RsPeerId(*it)) ;
std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
mGxsNetTunnel->getVirtualPeers(vpids);
for(auto it(vpids.begin());it!=vpids.end();++it)
peers.insert(RsPeerId(*it)) ;
}
}
if (peers.empty()) {
// nothing to do
return;
}
// Still empty? Reports there are no available peers
if (peers.empty()) return std::errc::network_down;
RS_STACK_MUTEX(mNxsMutex);
// for now just grps
for(auto sit = peers.begin(); sit != peers.end(); ++sit)
{
const RsPeerId peerId = *sit;
ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId);
@ -622,8 +625,7 @@ void RsGxsNetService::syncWithPeers()
generic_sendItem(grp);
}
if(!mAllowMsgSync)
return ;
if(!mAllowMsgSync) return std::error_condition();
#ifndef GXS_DISABLE_SYNC_MSGS
@ -741,10 +743,12 @@ void RsGxsNetService::syncWithPeers()
}
}
#endif
#endif // ndef GXS_DISABLE_SYNC_MSGS
return std::error_condition();
}
void RsGxsNetService::generic_sendItem(RsNxsItem *si)
void RsGxsNetService::generic_sendItem(rs_owner_ptr<RsItem> si)
{
// check if the item is to be sent to a distant peer or not
@ -1020,32 +1024,32 @@ void RsGxsNetService::locked_resetClientTS(const RsGxsGroupId& grpId)
it->second.msgUpdateInfos.erase(grpId) ;
}
void RsGxsNetService::subscribeStatusChanged(const RsGxsGroupId& grpId,bool subscribed)
void RsGxsNetService::subscribeStatusChanged(
const RsGxsGroupId& grpId, bool subscribed )
{
RS_STACK_MUTEX(mNxsMutex) ;
if(!subscribed)
return ;
if(!subscribed) return;
// When we subscribe, we reset the time stamps, so that the entire group list
// gets requested once again, for a proper update.
RS_STACK_MUTEX(mNxsMutex);
#ifdef NXS_NET_DEBUG_0
GXSNETDEBUG__G(grpId) << "Changing subscribe status for grp " << grpId << " to " << subscribed << ": reseting all server msg time stamps for this group, and server global TS." << std::endl;
std::map<RsGxsGroupId,RsGxsServerMsgUpdate>::iterator it = mServerMsgUpdateMap.find(grpId) ;
RS_DBG( "Changing subscribe status for grp", grpId, " to ", subscribed,
": reseting all server msg time stamps for this group, and "
"server global TS." );
#endif
RsGxsServerMsgUpdate& item(mServerMsgUpdateMap[grpId]) ;
RsGxsServerMsgUpdate& item(mServerMsgUpdateMap[grpId]);
item.msgUpdateTS = static_cast<uint32_t>(time(nullptr));
item.msgUpdateTS = time(NULL) ;
/* We also update mGrpServerUpdateItem so as to trigger a new grp list
* exchange with friends (friends will send their known ClientTS which
* will be lower than our own grpUpdateTS, triggering our sending of the
* new subscribed grp list. */
mGrpServerUpdate.grpUpdateTS = static_cast<uint32_t>(time(nullptr));
// We also update mGrpServerUpdateItem so as to trigger a new grp list exchange with friends (friends will send their known ClientTS which
// will be lower than our own grpUpdateTS, triggering our sending of the new subscribed grp list.
mGrpServerUpdate.grpUpdateTS = time(NULL) ;
if(subscribed)
locked_resetClientTS(grpId) ;
locked_resetClientTS(grpId);
}
bool RsGxsNetService::fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const
@ -1716,13 +1720,25 @@ RsItem *RsGxsNetService::generic_recvItem()
void RsGxsNetService::recvNxsItemQueue()
{
RsItem *item ;
RsItem* item;
while(NULL != (item=generic_recvItem()))
{
while(nullptr != (item=generic_recvItem()))
{
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << " type=" << std::hex << item->PacketId() << std::dec << std::endl ;
RS_DBG( "Received RsGxsNetService Item: ", (void*)item, " type=",
item->PacketId() );
#endif
/* Handle pull request and other new items here to not mess with all the
* old nested code and items hell */
switch(static_cast<RsNxsSubtype>(item->PacketSubType()))
{
case RsNxsSubtype::PULL_REQUEST:
std::unique_ptr<RsNxsPullRequestItem> pullItem(
static_cast<RsNxsPullRequestItem*>(item) );
handlePullRequest(std::move(pullItem));
continue;
}
// RsNxsItem needs dynamic_cast, since they have derived siblings.
//
RsNxsItem *ni = dynamic_cast<RsNxsItem*>(item) ;
@ -5073,6 +5089,46 @@ void RsGxsNetService::handleRecvPublishKeys(RsNxsGroupPublishKeyItem *item)
}
}
std::error_condition RsGxsNetService::requestPull(std::set<RsPeerId> peers)
{
/* If specific peers are passed as paramether ask only to them */
if(peers.empty())
{
mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers);
if(mAllowDistSync && mGxsNetTunnel != nullptr)
{
/* Grab all online virtual peers of distant tunnels for the current
* service. */
std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
mGxsNetTunnel->getVirtualPeers(vpids);
for(auto it(vpids.begin());it!=vpids.end();++it)
peers.insert(RsPeerId(*it)) ;
}
}
// Still empty? Reports there are no available peers
if (peers.empty()) return std::errc::network_down;
for(auto& peerId: std::as_const(peers))
{
auto item = new RsNxsPullRequestItem(
static_cast<RsServiceType>(mServType) );
item->PeerId(peerId);
generic_sendItem(item);
}
return std::error_condition();
}
void RsGxsNetService::handlePullRequest(
std::unique_ptr<RsNxsPullRequestItem> item )
{
checkUpdatesFromPeers(std::set<RsPeerId>{item->PeerId()});
}
bool RsGxsNetService::getGroupServerUpdateTS(const RsGxsGroupId& gid,rstime_t& group_server_update_TS, rstime_t& msg_server_update_TS)
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5204,13 +5260,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
/* always ignore case */
return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{
@ -5246,7 +5303,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{
std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5334,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_CHANNEL_INDEX
#ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important
* to test that the group meta is both non null and actually corresponds to the group id we seek. */
* mDataStore may in some situations allocate an empty group meta
* data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId);
@ -5332,18 +5384,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId);
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
return;
}
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
RS_WARN("Received search results for unknown request: ", req);
return;
}
RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32];
@ -5417,56 +5470,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId);
}
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
if(!mGxsNetTunnel)
{
free(searchData);
return std::errc::function_not_supported;
}
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos )
{
group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,12 +5531,11 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif
return !group_infos.empty();
return !group_infos.empty();
}
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)

77
libretroshare/src/gxs/rsgxsnetservice.h

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGXSNETSERVICE_H
#define RSGXSNETSERVICE_H
#pragma once
#include <list>
#include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
/// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -214,6 +250,15 @@ public:
void threadTick() override; /// @see RsTickingThread
/// @see RsNetworkExchangeService
std::error_condition checkUpdatesFromPeers(
std::set<RsPeerId> peers = std::set<RsPeerId>() ) override;
/// @see RsNetworkExchangeService
std::error_condition requestPull(
std::set<RsPeerId> peers = std::set<RsPeerId>() ) override;
private:
/*!
@ -387,6 +432,8 @@ private:
*/
void handleRecvPublishKeys(RsNxsGroupPublishKeyItem*) ;
void handlePullRequest(std::unique_ptr<RsNxsPullRequestItem> item);
/** E: item handlers **/
@ -423,7 +470,7 @@ private:
void locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN);
void checkDistantSyncState();
void syncWithPeers();
void syncGrpStatistics();
void addGroupItemToList(NxsTransaction*& tr,
const RsGxsGroupId& grpId, uint32_t& transN,
@ -523,7 +570,7 @@ private:
void cleanRejectedMessages();
void processObserverNotifications();
void generic_sendItem(RsNxsItem *si);
void generic_sendItem(rs_owner_ptr<RsItem> si);
RsItem *generic_recvItem();
private:
@ -629,5 +676,3 @@ private:
bool mUseMetaCache;
};
#endif // RSGXSNETSERVICE_H

607
libretroshare/src/gxs/rsgxsnettunnel.cc

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
mLastDump = time(NULL) + (RSRandom::random_u32()%20);
/* adds some variance in order to avoid doing all this tasks at once across
* services */
auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
}
//===========================================================================================================================================//
// Transport Items //
//===========================================================================================================================================//
//============================================================================//
// Transport Items //
//============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem
struct RsGxsNetTunnelItem: RsItem
{
public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{
// no priority. All items are encapsulated into generic Turtle items anyway.
}
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
};
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
};
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{
public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
}
};
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{
public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer
{
public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
return NULL ;
RS_ERR( "received item with wrong service ID ", service);
print_stacktrace();
return nullptr;
}
switch(item_subtype)
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
return new RsGxsNetTunnelVirtualPeerItem;
case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
return new RsGxsNetTunnelKeepAliveItem;
case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
return new RsGxsNetTunnelRandomBiasItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
return NULL ;
RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return nullptr;
}
}
};
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ;
}
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ;
uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL)
return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
uint32_t size = tSerializer.size(&searchItem);
uint8_t* buf = rs_malloc<uint8_t>(size);
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ;
RsNetworkExchangeService *service = nullptr;
requestId = mTurtle->turtleSearch(buf, size, this);
if(!requestId) return std::errc::result_out_of_range;
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service ;
search_result_item.group_infos = results ;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
return true ;
}
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
search_result_item.service = substring_gr->service ;
search_result_item.encrypted_group_data = encrypted_group_data ;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
return true ;
}
}
delete item;
return false ;
return std::error_condition();
}
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl;
bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
RS_DBG3("");
if(result_gs != NULL)
RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
#ifdef DEBUG_RSGXSNETTUNNEL
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
delete item;
return ;
RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
print_stacktrace();
return false;
}
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
if(result_gd != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{
if(!search_result_data)
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
"results!" );
print_stacktrace();
break;
}
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
result_gd->encrypted_group_data = NULL ; // prevents deletion
delete item;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
return ;
}
tSerializer.serialise(
&search_result_item, search_result_data,
&search_result_data_size );
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
return false;
}
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RS_DBG3(request_id);
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
auto castFailedWarn = [](const uint8_t subtype)
{
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
return it->second;
};
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{
groups = mGroups ;
virtual_peers = mVirtualPeers ;

62
libretroshare/src/gxs/rsgxsnettunnel.h

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once
#include <map>
#include <system_error>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService:
@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
public RsGxsDistSync
{
public:
RsGxsNetTunnelService() ;
virtual ~RsGxsNetTunnelService() ;
RsGxsNetTunnelService();
~RsGxsNetTunnelService() override;
/*!
* \brief registerSearchableService
@ -181,24 +184,38 @@ public:
*/
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
/*!
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
void connectToTurtleRouter(p3turtle *tr) override;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/** Gxs services (channels, forums...) are supposed to use this to request
* searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
///@see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* search_request_data,
uint32_t search_request_data_len,
unsigned char*& search_result_data,
uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread
///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected:
// interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

10
libretroshare/src/gxs/rsgxsnotify.h

@ -97,3 +97,13 @@ private:
bool mMetaChange;
};
struct RsGxsMsgDeletedChange : RsGxsNotify
{
RsGxsMsgDeletedChange(
const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
RsGxsNotify(gid), messageId(msgId) {}
NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
const RsGxsMessageId messageId;
};

145
libretroshare/src/gxs/rsgxsutil.cc

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
, RsGixs* gixs )
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true;
}
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
bool RsGxsSinglePassIntegrityCheck::check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
{
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
}
else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(nxsMsg)
nxsMsgS.insert(nxsMsg->msgId);
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
msgsToDel[grpId].insert(msgId);
}
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete msg;

36
libretroshare/src/gxs/rsgxsutil.h

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType&,
RsGixs *gixs);
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
RsGixs* gixs );
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
bool isDone();
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
static bool check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
};
class GroupUpdate

80
libretroshare/src/gxs/rsnxs.h

@ -3,8 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker *
* Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGNP_H
#define RSGNP_H
#pragma once
#include <set>
#include <string>
#include "util/rstime.h"
#include <stdlib.h>
#include <cstdlib>
#include <list>
#include <map>
#include "util/rstime.h"
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService
{
public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() = default;
virtual uint16_t serviceType() const =0;
/*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*!
* \brief turtleGroupRequest
@ -115,13 +129,17 @@ public:
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*!
* \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
@ -305,6 +325,20 @@ public:
return RsReputationLevel::NEUTRAL;
}
}
};
#endif // RSGNP_H
/**
* @brief Check if new stuff is available from peers
* @param peers peers to check, if empty all available peers are checked
*/
virtual std::error_condition checkUpdatesFromPeers(
std::set<RsPeerId> peers = std::set<RsPeerId>() ) = 0;
/**
* @brief request online peers to pull updates from our node ASAP
* @param peers peers to which request pull from, if empty all available
* peers are requested to pull
* @return success or error details
*/
virtual std::error_condition requestPull(
std::set<RsPeerId> peers = std::set<RsPeerId>() ) = 0;
};

88
libretroshare/src/pqi/pqissli2pbob.h → libretroshare/src/gxs/rsnxsobserver.cpp

@ -1,52 +1,36 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef PQISSLI2PBOB_H
#define PQISSLI2PBOB_H
#include "pqi/pqissl.h"
/*
* This class is a minimal varied version of pqissl to work with I2P BOB tunnels.
* The only difference is that the [.b32].i2p addresses must be sent first.
*
* Everything else is untouched.
*/
class pqissli2pbob : public pqissl
{
public:
pqissli2pbob(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm) {}
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Basic_Connection_Complete();
private:
std::string mI2pAddr;
};
#endif // PQISSLI2PBOB_H
/*******************************************************************************
* RetroShare General eXchange System *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "gxs/rsnxsobserver.h"
const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
{
switch(static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return std::errc::operation_not_supported;
default:
return std::error_condition(ev, *this);
}
}

108
libretroshare/src/gxs/rsnxsobserver.h

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher *
* Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSOBSERVER_H
#define RSNXSOBSERVER_H
#pragma once
#include <set>
#include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId ;
typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver
{
public:
RsNxsObserver() {}
public:
/*!
@ -56,6 +99,46 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data storage for a pointer to search
* result reply data or nullptr if no mathing results where found
* @param[out] resultSize storage for results data size or 0 if no matching
* results where found
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
/* Avoid unused paramethers warning this way so doxygen can still parse
* paramethers documentation */
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*!
* @param grpId group id
*/
@ -70,6 +153,7 @@ public:
* @param grpId group id
*/
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H
RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

5
libretroshare/src/jsonapi/jsonapi.cpp

@ -796,8 +796,9 @@ void JsonApiServer::run()
}
catch(std::exception& e)
{
RsErr() << __PRETTY_FUNCTION__ << " Failure starting JSON API server: "
<< e.what() << std::endl;
/* TODO: find a way to report back programmatically if failed listening
* port */
RS_ERR("Failure starting JSON API server: ", e.what());
print_stacktrace();
return;
}

89
libretroshare/src/libretroshare.pro

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team <contact@retroshare.cc>
# SPDX-FileCopyrightText: (C) 2004-2021 Retroshare Team <contact@retroshare.cc>
# SPDX-License-Identifier: CC0-1.0
!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri")
@ -156,7 +156,7 @@ rs_webui {
HEADERS += plugins/pluginmanager.h \
plugins/dlfcn_win32.h \
rsitems/rspluginitems.h \
rsitems/rspluginitems.h \
util/i2pcommon.h \
util/rsinitedptr.h
@ -205,7 +205,7 @@ linux-* {
LIBS *= -ldl
DEFINES *= PLUGIN_DIR=\"\\\"$${PLUGIN_DIR}\\\"\"
DEFINES *= DATA_DIR=\"\\\"$${DATA_DIR}\\\"\"
DEFINES *= RS_DATA_DIR=\"\\\"$${RS_DATA_DIR}\\\"\"
}
linux-g++ {
@ -289,7 +289,7 @@ mac {
#LIBS += -lsqlite3
DEFINES *= PLUGIN_DIR=\"\\\"$${PLUGIN_DIR}\\\"\"
DEFINES *= DATA_DIR=\"\\\"$${DATA_DIR}\\\"\"
DEFINES *= RS_DATA_DIR=\"\\\"$${RS_DATA_DIR}\\\"\"
}
################################# FreeBSD ##########################################
@ -394,7 +394,6 @@ HEADERS += pqi/authssl.h \
pqi/pqissl.h \
pqi/pqissllistener.h \
pqi/pqisslpersongrp.h \
pqi/pqissli2pbob.h \
pqi/pqisslproxy.h \
pqi/pqistore.h \
pqi/pqistreamer.h \
@ -455,7 +454,7 @@ HEADERS += rsitems/rsitem.h \
rsitems/rsgxsupdateitems.h \
rsitems/rsserviceinfoitems.h \
HEADERS += services/autoproxy/p3i2pbob.h \
HEADERS += \
services/rseventsservice.h \
services/autoproxy/rsautoproxymonitor.h \
services/p3msgservice.h \
@ -509,8 +508,8 @@ HEADERS += util/folderiterator.h \
util/cxx11retrocompat.h \
util/cxx14retrocompat.h \
util/cxx17retrocompat.h \
util/rsurl.h \
util/rserrno.h
util/cxx23retrocompat.h \
util/rsurl.h
SOURCES += ft/ftchunkmap.cc \
ft/ftcontroller.cc \
@ -560,7 +559,6 @@ SOURCES += pqi/authgpg.cc \
pqi/pqissl.cc \
pqi/pqissllistener.cc \
pqi/pqisslpersongrp.cc \
pqi/pqissli2pbob.cpp \
pqi/pqisslproxy.cc \
pqi/pqistore.cc \
pqi/pqistreamer.cc \
@ -618,7 +616,6 @@ SOURCES += serialiser/rsbaseserial.cc \
SOURCES += services/autoproxy/rsautoproxymonitor.cc \
services/rseventsservice.cc \
services/autoproxy/p3i2pbob.cc \
services/p3msgservice.cc \
services/p3service.cc \
services/p3statusservice.cc \
@ -638,10 +635,10 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \
util/rsnet.cc \
util/rsnet_ss.cc \
util/rsdnsutils.cc \
util/extaddrfinder.cc \
util/dnsresolver.cc \
util/rsprint.cc \
@ -652,8 +649,7 @@ SOURCES += util/folderiterator.cc \
util/rsrecogn.cc \
util/rstime.cc \
util/rsurl.cc \
util/rsbase64.cc \
util/rserrno.cc
util/rsbase64.cc
equals(RS_UPNP_LIB, miniupnpc) {
HEADERS += rs_upnp/upnputil.h rs_upnp/upnphandler_miniupnp.h
@ -717,7 +713,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc
gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# gxs tunnels
HEADERS += gxstunnel/p3gxstunnel.h \
@ -941,6 +938,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
@ -1014,6 +1019,44 @@ rs_broadcast_discovery {
}
}
rs_sam3 {
SOURCES += \
services/autoproxy/p3i2psam3.cpp \
pqi/pqissli2psam3.cpp \
HEADERS += \
services/autoproxy/p3i2psam3.h \
pqi/pqissli2psam3.h \
}
rs_sam3_libsam3 {
DUMMYQMAKECOMPILERINPUT = FORCE
libsam3.name = Generating libsam3.
libsam3.input = DUMMYQMAKECOMPILERINPUT
libsam3.output = $$clean_path($${LIBSAM3_BUILD_PATH}/libsam3.a)
libsam3.CONFIG += target_predeps combine
libsam3.variable_out = PRE_TARGETDEPS
win32-g++:isEmpty(QMAKE_SH) {
LIBSAM3_MAKE_PARAMS = CC=gcc
libsam3.commands = \
cd /D $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/libsam3 || cd . $$escape_expand(\\n\\t) \
$(CHK_DIR_EXISTS) $$shell_path($$LIBSAM3_BUILD_PATH) $(MKDIR) $$shell_path($${LIBSAM3_BUILD_PATH}) $$escape_expand(\\n\\t) \
$(COPY_DIR) $$shell_path($${LIBSAM3_SRC_PATH}) $$shell_path($${LIBSAM3_BUILD_PATH}) || cd . $$escape_expand(\\n\\t)
} else {
LIBSAM3_MAKE_PARAMS =
libsam3.commands = \
cd $${RS_SRC_PATH} && ( \
git submodule update --init supportlibs/libsam3 || \
true ) && \
mkdir -p $${LIBSAM3_BUILD_PATH} && \
(cp -r $${LIBSAM3_SRC_PATH}/* $${LIBSAM3_BUILD_PATH} || true) &&
}
libsam3.commands += \
cd $$shell_path($${LIBSAM3_BUILD_PATH}) && \
$(MAKE) build $${LIBSAM3_MAKE_PARAMS}
QMAKE_EXTRA_COMPILERS += libsam3
}
###########################################################################################################
# OLD CONFIG OPTIONS.
# Not used much - but might be useful one day.
@ -1051,6 +1094,8 @@ test_bitdht {
################################# Android #####################################
android-* {
lessThan(ANDROID_API_VERSION, 24) {
## TODO: This probably disable largefile support and maybe is not necessary with
## __ANDROID_API__ >= 24 hence should be made conditional or moved to a
## compatibility header
@ -1058,12 +1103,26 @@ android-* {
DEFINES *= "fseeko64=fseeko"
DEFINES *= "ftello64=ftello"
## @See: rs_android/README-ifaddrs-android.adoc
HEADERS += \
rs_android/ifaddrs-android.h \
rs_android/LocalArray.h \
rs_android/ScopedFd.h
}
## Static library are very susceptible to order in command line
sLibs = bz2 $$RS_UPNP_LIB $$RS_SQL_LIB ssl crypto
LIBS += $$linkStaticLibs(sLibs)
PRE_TARGETDEPS += $$pretargetStaticLibs(sLibs)
HEADERS += util/androiddebug.h
HEADERS += \
rs_android/androidcoutcerrcatcher.hpp \
rs_android/retroshareserviceandroid.hpp \
rs_android/rsjni.hpp
SOURCES += rs_android/rsjni.cpp \
rs_android/retroshareserviceandroid.cpp \
rs_android/errorconditionwrap.cpp
}

20
libretroshare/src/pgp/rscertificate.cc

@ -552,26 +552,24 @@ unsigned short RsCertificate::loc_port_us() const
return (int)ipv4_internal_ip_and_port[4]*256 + (int)ipv4_internal_ip_and_port[5] ;
}
bool RsCertificate::cleanCertificate( const std::string& input, std::string& output, Format& format, uint32_t& error_code, bool check_content )
bool RsCertificate::cleanCertificate( const std::string& input, std::string& output, Format& format, uint32_t& error_code, bool check_content, RsPeerDetails& details)
{
if(cleanRadix64(input,output,error_code))
{
RsPeerDetails details;
if(rsPeers->parseShortInvite(output,details,error_code))
{
format = RS_CERTIFICATE_SHORT_RADIX;
return true;
}
if(rsPeers->parseShortInvite(output,details,error_code))
{
format = RS_CERTIFICATE_SHORT_RADIX;
return true;
}
//Clear details. As parseShortInvite may make it dirty.
details = RsPeerDetails();
format = RS_CERTIFICATE_RADIX;
if(!check_content) return true;
uint32_t errCode;
auto crt = RsCertificate::fromString(input, errCode);
error_code = static_cast<int>(errCode);
return crt != nullptr;
return rsPeers->loadDetailsFromStringCert(input,details,error_code);
}
return false;

2
libretroshare/src/pgp/rscertificate.h

@ -84,7 +84,7 @@ public:
static bool cleanCertificate(
const std::string& input, std::string& output,
RsCertificate::Format& format, uint32_t& error_code, bool check_content);
RsCertificate::Format& format, uint32_t& error_code, bool check_content, RsPeerDetails& details);
const std::set<RsUrl>& locators() const { return mLocators; }

244
libretroshare/src/pqi/p3netmgr.cc

@ -47,14 +47,15 @@ struct RsLog::logInfo p3netmgrzoneInfo = {RsLog::Default, "p3netmgr"};
/* Network setup States */
const uint32_t RS_NET_NEEDS_RESET = 0x0000;
const uint32_t RS_NET_UNKNOWN = 0x0001;
const uint32_t RS_NET_UPNP_INIT = 0x0002;
const uint32_t RS_NET_UPNP_SETUP = 0x0003;
const uint32_t RS_NET_EXT_SETUP = 0x0004;
const uint32_t RS_NET_DONE = 0x0005;
const uint32_t RS_NET_LOOPBACK = 0x0006;
//const uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_NEEDS_RESET = 0x0000;
constexpr uint32_t RS_NET_UNKNOWN = 0x0001;
constexpr uint32_t RS_NET_UPNP_INIT = 0x0002;
constexpr uint32_t RS_NET_UPNP_SETUP = 0x0003;
constexpr uint32_t RS_NET_EXT_SETUP = 0x0004;
constexpr uint32_t RS_NET_DONE = 0x0005;
constexpr uint32_t RS_NET_LOOPBACK = 0x0006;
//constexpr uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_SHUTDOWN = 0x00FF; //Highest value to not restart UPnP nor ExtAddrFinder
/* Stun modes (TODO) */
//const uint32_t RS_STUN_DHT = 0x0001;
@ -68,6 +69,8 @@ const uint32_t MAX_UPNP_COMPLETE = 600; /* 10 min... seems to take a while */
//const uint32_t MIN_TIME_BETWEEN_NET_RESET = 5;
const uint32_t MIN_TIME_EXT_FINDER_UPDATE = 300; /* 5min to check if external IP is changed */
/****
* #define NETMGR_DEBUG 1
* #define NETMGR_DEBUG_RESET 1
@ -107,19 +110,17 @@ void pqiNetStatus::print(std::ostream &out)
}
p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
mDoNotNetCheckUntilTs(0)
p3NetMgrIMPL::p3NetMgrIMPL()
: mPeerMgr(nullptr), mLinkMgr(nullptr)
, mNetMtx("p3NetMgr"), mNetMode(RS_NET_MODE_UDP), mVsDisc(RS_VS_DISC_FULL), mVsDht(RS_VS_DHT_FULL)// default to full.
, mNetInitTS(0), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
, mUseExtAddrFinder(true), mNetExtAddrFinderTs(0), mDoNotNetCheckUntilTs(0)
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetMode = RS_NET_MODE_UDP;
mUseExtAddrFinder = true;
mExtAddrFinder = new ExtAddrFinder();
mNetInitTS = 0;
mNetFlags = pqiNetStatus();
mOldNetFlags = pqiNetStatus();
@ -133,10 +134,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mLocalAddr.ss_family = AF_INET;
mExtAddr.ss_family = AF_INET;
// default to full.
mVsDisc = RS_VS_DISC_FULL;
mVsDht = RS_VS_DHT_FULL;
}
#ifdef NETMGR_DEBUG
@ -235,6 +232,10 @@ void p3NetMgrIMPL::netReset()
rslog(RSL_ALERT, p3netmgrzone, "p3NetMgr::netReset() Called");
shutdown(); /* blocking shutdown call */
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
}
// Will initiate a new call for determining the external ip.
if (mUseExtAddrFinder)
@ -242,7 +243,7 @@ void p3NetMgrIMPL::netReset()
#ifdef NETMGR_DEBUG_RESET
std::cerr << "p3NetMgrIMPL::netReset() restarting AddrFinder" << std::endl;
#endif
mExtAddrFinder->reset() ;
mExtAddrFinder->reset(true) ;
}
else
{
@ -265,7 +266,7 @@ void p3NetMgrIMPL::netReset()
* as it calls back to p3ConnMgr.
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
struct sockaddr_storage iaddr = mLocalAddr;
@ -283,8 +284,7 @@ void p3NetMgrIMPL::netReset()
}
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
netStatusReset_locked();
}
@ -311,8 +311,8 @@ bool p3NetMgrIMPL::shutdown() /* blocking shutdown call */
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_SHUTDOWN;
mNetInitTS = time(NULL);
netStatusReset_locked();
}
@ -347,7 +347,7 @@ void p3NetMgrIMPL::netStartup()
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetInitTS = time(NULL);
netStatusReset_locked();
@ -460,35 +460,51 @@ void p3NetMgrIMPL::netStatusTick()
uint32_t netStatus = 0;
rstime_t age = 0;
bool needExtFinderUpdate = false;
{
RsStackMutex stack(mNetMtx); /************** LOCK MUTEX ***************/
RS_STACK_MUTEX(mNetMtx); /************** LOCK MUTEX ***************/
netStatus = mNetStatus;
age = time(NULL) - mNetInitTS;
}
needExtFinderUpdate = netStatus == RS_NET_DONE;
needExtFinderUpdate &= mNetExtAddrFinderTs < time(nullptr);
if(needExtFinderUpdate)
mNetExtAddrFinderTs = time(nullptr) + MIN_TIME_EXT_FINDER_UPDATE;
}
if(netStatus <= RS_NET_UPNP_SETUP && mUseExtAddrFinder)
{
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
if( mUseExtAddrFinder
&& ( netStatus <= RS_NET_UPNP_SETUP
|| needExtFinderUpdate) )
{
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "Asking ExtAddrFinder for IP. Initializing port with " << sockaddr_storage_port(tmpip) << std::endl;
RS_DBG("Asking ExtAddrFinder for IP. Initializing port with ", sockaddr_storage_port(tmpip));
#endif
if(mExtAddrFinder->hasValidIP(tmpip) && sockaddr_storage_ipv6_to_ipv4(tmpip) && !sockaddr_storage_same(tmpip,mExtAddr))
{
if(mExtAddrFinder->hasValidIPV4(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
else if(mExtAddrFinder->hasValidIPV6(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
//Only if no IPv4 else, reset connections on setExtAddress()
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
setExtAddress(tmpip);
}
}
}
switch(netStatus)
{
@ -578,7 +594,7 @@ void p3NetMgrIMPL::netDhtInit()
uint32_t vs = 0;
{
RsStackMutex stack(mNetMtx); /*********** LOCKED MUTEX ************/
RS_STACK_MUTEX(mNetMtx); /*********** LOCKED MUTEX ************/
vs = mVsDht;
}
@ -745,25 +761,16 @@ void p3NetMgrIMPL::netExtCheck()
if (mUseExtAddrFinder)
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() checking ExtAddrFinder" << std::endl;
RS_DBG("checking ExtAddrFinder");
#endif
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
if (extFinderOk && sockaddr_storage_ipv6_to_ipv4(tmpip))
// Test for IPv4 first to be compatible with older versions.
if (mExtAddrFinder->hasValidIPV4(tmpip))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
#endif
sockaddr_storage_setport(tmpip, guessNewExtPort());
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
#endif
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
@ -773,10 +780,22 @@ void p3NetMgrIMPL::netExtCheck()
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
std::cerr << __PRETTY_FUNCTION__ << " ExtAddrFinder "
<< " reported external address "
<< sockaddr_storage_iptostring(tmpip)
<< std::endl;
RS_DBG("Reported external IPv4 address ", sockaddr_storage_iptostring(tmpip));
}
else if (mExtAddrFinder->hasValidIPV6(tmpip))
{
sockaddr_storage_setport(tmpip, guessNewExtPort());
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
/* XXX HACK TO FIX drbob: ALLOWING
* ExtAddrFinder -> ExtAddrStableOk = true
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
RS_DBG("Reported external IPv6 address ", sockaddr_storage_iptostring(tmpip));
}
}
}
@ -967,7 +986,7 @@ void p3NetMgrIMPL::netExtCheck()
if (netSetupDone)
{
std::cerr << "p3NetMgrIMPL::netExtCheck() netSetupDone" << std::endl;
RS_DBG("netSetupDone");
/* Setup NetStateBox with this info */
updateNetStateBox_startup();
@ -978,12 +997,14 @@ void p3NetMgrIMPL::netExtCheck()
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
/* inform DHT about our external address */
RsPeerId fakeId;
netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
/* inform DHT about our external IPV4 address, it doesn't support IPv6 for now.*/
if(sockaddr_storage_ipv6_to_ipv4(mExtAddr))
{
RsPeerId fakeId;
netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
}
std::cerr << __PRETTY_FUNCTION__ << " Network Setup Complete"
<< std::endl;
RS_INFO("Network Setup Complete");
}
}
@ -994,7 +1015,8 @@ void p3NetMgrIMPL::netExtCheck()
bool p3NetMgrIMPL::checkNetAddress()
{
bool addrChanged = false;
bool validAddr = false;
bool validAddr = false;
bool needOwnAddrUpdate = false;
sockaddr_storage prefAddr;
sockaddr_storage oldAddr;
@ -1067,8 +1089,7 @@ bool p3NetMgrIMPL::checkNetAddress()
if (!validAddr)
{
RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
<<" found. Report to developers." << std::endl;
RS_ERR("no valid local network address found. Report to developers.");
print_stacktrace();
return false;
@ -1109,6 +1130,7 @@ bool p3NetMgrIMPL::checkNetAddress()
sockaddr_storage_setport(mExtAddr, port); // this accounts for when the port was updated
addrChanged = true;
}
} // RS_STACK_MUTEX(mNetMtx);
if (addrChanged)
@ -1123,11 +1145,24 @@ bool p3NetMgrIMPL::checkNetAddress()
rsEvents->postEvent(ev);
}
if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
needOwnAddrUpdate = true;
netReset();
}
if (mPeerMgr)
{
// Retrieve last known IP, if none, update own addresse to get current.
peerState ps;
mPeerMgr->getOwnNetStatus(ps);
needOwnAddrUpdate |= ps.ipAddrs.mLocal.mAddrs.empty();
needOwnAddrUpdate |= ps.ipAddrs.mExt.mAddrs.empty();
if (needOwnAddrUpdate)
{
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
}
return true;
}
@ -1139,7 +1174,7 @@ bool p3NetMgrIMPL::checkNetAddress()
/* to allow resets of network stuff */
void p3NetMgrIMPL::addNetListener(pqiNetListener *listener)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetListeners.push_back(listener);
}
@ -1149,7 +1184,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mLocalAddr, addr))
{
changed = true;
@ -1171,7 +1206,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
}
bool p3NetMgrIMPL::getExtAddress(struct sockaddr_storage& addr)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if(mNetFlags.mExtAddrOk)
{
@ -1186,7 +1221,7 @@ bool p3NetMgrIMPL::setExtAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mExtAddr, addr))
{
changed = true;
@ -1220,7 +1255,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
{
uint32_t oldNetMode;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* only change TRY flags */
oldNetMode = mNetMode;
@ -1266,7 +1301,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
bool p3NetMgrIMPL::setVisState(uint16_t vs_disc, uint16_t vs_dht)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mVsDisc = vs_disc;
mVsDht = vs_dht;
@ -1628,7 +1663,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
uint32_t netsize = 0, rsnetsize = 0;
netAssistConnectStats(netsize, rsnetsize);
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* quick update of the stuff that can change! */
mNetFlags.mDhtOk = dhtOk;
@ -1652,7 +1687,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
bool p3NetMgrIMPL::getIPServersEnabled()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mUseExtAddrFinder;
}
@ -1661,15 +1696,32 @@ void p3NetMgrIMPL::getIPServersList(std::list<std::string>& ip_servers)
mExtAddrFinder->getIPServersList(ip_servers);
}
void p3NetMgrIMPL::getCurrentExtIPList(std::list<std::string>& ip_list)
{
ip_list.clear();
sockaddr_storage addr;
if(mExtAddrFinder->hasValidIPV4(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
if(mExtAddrFinder->hasValidIPV6(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
}
void p3NetMgrIMPL::setIPServersEnabled(bool b)
{
if (mUseExtAddrFinder != b)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mExtAddrFinder->reset(true);
if (b)
mExtAddrFinder->start_request();
}
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mUseExtAddrFinder = b;
}
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgr: setIPServers to " << b << std::endl ;
RS_DBG("set mUseExtAddrFinder to ", b);
#endif
}
@ -1682,31 +1734,31 @@ void p3NetMgrIMPL::setIPServersEnabled(bool b)
RsNetState p3NetMgrIMPL::getNetStateMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetStateMode();
}
RsNetworkMode p3NetMgrIMPL::getNetworkMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetworkMode();
}
RsNatTypeMode p3NetMgrIMPL::getNatTypeMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatTypeMode();
}
RsNatHoleMode p3NetMgrIMPL::getNatHoleMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatHoleMode();
}
RsConnectModes p3NetMgrIMPL::getConnectModes()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getConnectModes();
}
@ -1730,7 +1782,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunDht(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1749,7 +1801,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mProxyStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunProxy(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1768,7 +1820,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
bool dhtOn = netAssistConnectEnabled();
bool dhtActive = netAssistConnectActive();
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setDhtState(dhtOn, dhtActive);
}
@ -1777,7 +1829,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
#ifdef NETMGR_DEBUG_STATEBOX
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
auto netstate = mNetStateBox.getNetStateMode();
auto netMode = mNetStateBox.getNetworkMode();
@ -1823,7 +1875,7 @@ void p3NetMgrIMPL::updateNatSetting()
RsNatTypeMode natType = RsNatTypeMode::UNKNOWN;
RsNatHoleMode natHole = RsNatHoleMode::UNKNOWN;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
natType = mNetStateBox.getNatTypeMode();
natHole = mNetStateBox.getNatHoleMode();
@ -1925,7 +1977,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* fill in the data */
struct sockaddr_storage tmpip;
@ -1973,8 +2025,8 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
/* ExtAddrFinder */
if (mUseExtAddrFinder)
{
tmpip = mLocalAddr;
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
tmpip = mLocalAddr;
bool extFinderOk = mExtAddrFinder->hasValidIPV4(tmpip);
if (extFinderOk)
{
@ -2020,7 +2072,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
void p3NetMgrIMPL::updateNetStateBox_reset()
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.reset();

7
libretroshare/src/pqi/p3netmgr.h

@ -120,6 +120,7 @@ virtual bool shutdown() = 0; /* blocking shutdown call */
virtual bool getIPServersEnabled() = 0;
virtual void setIPServersEnabled(bool b) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status) = 0;
@ -171,6 +172,7 @@ virtual bool shutdown(); /* blocking shutdown call */
virtual bool getIPServersEnabled();
virtual void setIPServersEnabled(bool b);
virtual void getIPServersList(std::list<std::string>& ip_servers);
virtual void getCurrentExtIPList(std::list<std::string>& ip_list);
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status);
@ -307,12 +309,13 @@ void netStatusReset_locked();
uint16_t mVsDisc;
uint16_t mVsDht;
rstime_t mNetInitTS;
rstime_t mNetInitTS;
uint32_t mNetStatus;
bool mStatusChanged;
bool mUseExtAddrFinder;
bool mUseExtAddrFinder;
rstime_t mNetExtAddrFinderTs;
/* network status flags (read by rsiface) */
pqiNetStatus mNetFlags;

4
libretroshare/src/pqi/p3peermgr.cc

@ -1398,11 +1398,11 @@ bool p3PeerMgrIMPL::UpdateOwnAddress( const sockaddr_storage& pLocalAddr,
sockaddr_storage_copy(pExtAddr, extAddr);
sockaddr_storage_ipv6_to_ipv4(extAddr);
//#ifdef PEER_DEBUG
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgrIMPL::UpdateOwnAddress("
<< sockaddr_storage_tostring(localAddr) << ", "
<< sockaddr_storage_tostring(extAddr) << ")" << std::endl;
//#endif
#endif
if( rsBanList &&
!rsBanList->isAddressAccepted(localAddr,

83
libretroshare/src/pqi/pqinetwork.cc

@ -21,15 +21,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifdef WINDOWS_SYS
# include "util/rswin.h"
# include "util/rsmemory.h"
# include <ws2tcpip.h>
#endif // WINDOWS_SYS
#ifdef __ANDROID__
# include <android/api-level.h>
#endif // def __ANDROID__
#include <cerrno>
#include <iostream>
@ -43,6 +34,28 @@
#include "util/rsnet.h"
#include "util/stacktrace.h"
#ifdef WINDOWS_SYS
# include "util/rswin.h"
# include "util/rsmemory.h"
# include <ws2tcpip.h>
#endif // WINDOWS_SYS
/// @See: android_ifaddrs/README.adoc
#ifdef __ANDROID__
# include <android/api-level.h>
#endif // def __ANDROID__
#ifdef WINDOWS_SYS /* Windows - define errno */
int errno;
#else /* Windows - define errno */
#include <netdb.h>
#endif
#ifdef __HAIKU__
# include <sys/sockio.h>
# define IFF_RUNNING 0x0001
#endif
static struct RsLog::logInfo pqinetzoneInfo = {RsLog::Default, "pqinet"};
#define pqinetzone &pqinetzoneInfo
@ -50,21 +63,6 @@ static struct RsLog::logInfo pqinetzoneInfo = {RsLog::Default, "pqinet"};
* #define NET_DEBUG 1
****/
#ifdef WINDOWS_SYS /* Windows - define errno */
int errno;
#else /* Windows - define errno */
#include <netdb.h>
#endif
#ifdef __HAIKU__
#include <sys/sockio.h>
#define IFF_RUNNING 0x0001
#endif
/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
#ifndef WINDOWS_SYS
@ -270,18 +268,16 @@ int inet_aton(const char *name, struct in_addr *addr)
#endif
/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
#include "util/cxx17retrocompat.h"
#include <sys/types.h>
#ifdef WINDOWS_SYS
# include <winsock2.h>
# include <iphlpapi.h>
# pragma comment(lib, "IPHLPAPI.lib")
#elif defined(__ANDROID__) && __ANDROID_API__ < 24
# include <string>
# include <QString>
# include <QHostAddress>
# include <QNetworkInterface>
#else // not __ANDROID__ nor WINDOWS => Linux and other unixes
/// @See: android_ifaddrs/README.adoc
# include "rs_android/ifaddrs-android.h"
#else // not WINDOWS => Linux and other unixes
# include <ifaddrs.h>
# include <net/if.h>
#endif // WINDOWS_SYS
@ -324,20 +320,12 @@ bool getLocalAddresses(std::vector<sockaddr_storage>& addrs)
}
}
free(adapter_addresses);
#elif defined(__ANDROID__) && __ANDROID_API__ < 24
for(auto& qAddr: QNetworkInterface::allAddresses())
{
sockaddr_storage tmpAddr;
sockaddr_storage_clear(tmpAddr);
if(sockaddr_storage_ipv4_aton(tmpAddr, qAddr.toString().toStdString().c_str()))
addrs.push_back(tmpAddr);
}
#else // not WINDOWS_SYS not ANDROID => Linux and other unixes
#else // not WINDOWS_SYS => Linux and other unixes
struct ifaddrs *ifsaddrs, *ifa;
if(getifaddrs(&ifsaddrs) != 0)
{
std::cerr << __PRETTY_FUNCTION__ << " FATAL ERROR: " << errno << " "
<< strerror(errno) << std::endl;
RS_ERR( "getifaddrs failed with: ", errno, " ",
rs_errno_to_condition(errno) );
print_stacktrace();
return false;
}
@ -346,18 +334,19 @@ bool getLocalAddresses(std::vector<sockaddr_storage>& addrs)
{
sockaddr_storage tmp;
sockaddr_storage_clear(tmp);
if (sockaddr_storage_copyip(tmp, *reinterpret_cast<sockaddr_storage*>(ifa->ifa_addr)))
if(sockaddr_storage_copyip(
tmp,
*reinterpret_cast<sockaddr_storage*>(ifa->ifa_addr) ))
addrs.push_back(tmp);
}
freeifaddrs(ifsaddrs);
#endif // WINDOWS_SYS
#ifdef NET_DEBUG
std::list<sockaddr_storage>::iterator it;
std::cout << "getLocalAddresses(...) returning: <" ;
for(it = addrs.begin(); it != addrs.end(); ++it)
std::cout << sockaddr_storage_iptostring(*it) << ", ";
std::cout << ">" << std::endl;
auto&& dbg = RS_DBG("returning: [");
for(auto& addr: std::as_const(addrs))
dbg << sockaddr_storage_iptostring(addr) << ", ";
dbg << "]" << std::endl;
#endif
return !addrs.empty();

16
libretroshare/src/pqi/pqinetwork.h

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2015-2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2015-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef MRK_PQI_NETWORKING_HEADER
#define MRK_PQI_NETWORKING_HEADER
#pragma once
#include <vector>
@ -86,9 +86,13 @@ extern int errno; /* Define extern errno, to duplicate unix behaviour */
#include <string>
#include <list>
#include "util/rsdeprecate.h"
// Same def - different functions...
RS_DEPRECATED_FOR("use std::error_condition instead")
void showSocketError(std::string &out);
RS_DEPRECATED_FOR("use std::error_condition instead")
std::string socket_errorType(int err);
bool getLocalAddresses(std::vector<sockaddr_storage> & addrs);
@ -103,10 +107,6 @@ int unix_getsockopt_error(int sockfd, int *err);
#ifdef WINDOWS_SYS // WINDOWS
/******************* WINDOWS SPECIFIC PART ******************/
RS_DEPRECATED_FOR("use std::error_condition instead")
int WinToUnixError(int error);
#endif
#endif

7
libretroshare/src/pqi/pqissl.cc

@ -1741,6 +1741,13 @@ bool pqissl::moretoread(uint32_t usec)
{
rslog(RSL_ALERT, pqisslzone,
"pqissl::moretoread() Select ERROR!");
RS_WARN(strerror(errno));
if (errno == EBADF) {
// happens when SAM is shut down
rslog(RSL_ALERT, pqisslzone, "pqissl::moretoread() -> calling reset()");
reset_locked();
}
return 0;
}

261
libretroshare/src/pqi/pqissli2psam3.cpp Normal file

@ -0,0 +1,261 @@
#include "pqissli2psam3.h"
#include <libsam3.h>
RS_SET_CONTEXT_DEBUG_LEVEL(2)
static constexpr int pqiDone = 1;
static constexpr int pqiWait = 0;
static constexpr int pqiError = -1;
pqissli2psam3::pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm), mState(pqisslSam3State::NONE), mI2pAddrB32(), mI2pAddrLong()
{
RS_DBG4();
mConn = nullptr;
}
bool pqissli2psam3::connect_parameter(uint32_t type, const std::string &value)
{
RS_DBG4();
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_DBG1("got addr:", value);
RS_STACK_MUTEX(mSslMtx);
mI2pAddrB32 = value;
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2psam3::Initiate_Connection()
{
RS_DBG4();
if(waiting != WAITING_DELAY)
{
RS_ERR("Already Attempt in Progress!");
return pqiError;
}
switch (mState) {
case(pqisslSam3State::NONE):
RS_DBG2("NONE");
{
if(mConn) {
// how did we end up here?
RS_ERR("state is NONE but a connection is existing?!");
}
mConn = 0;
// get SAM session
mConn = 0;
samSettings ss;
ss.session = nullptr;
rsAutoProxyMonitor::taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, static_cast<void*>(&ss));
if (!!ss.session) {
RS_DBG3("NONE->DO_LOOKUP");
mState = pqisslSam3State::DO_LOOKUP;
} else {
RS_DBG3("NONE->DO_LOOKUP NOPE", ss.session);
}
}
break;
case(pqisslSam3State::DO_LOOKUP):
RS_DBG1("DO_LOOKUP");
if (!mI2pAddrLong.empty()) {
// skip lookup, it is highly unlikely/impossible for a public key to change (isn't it?)
mState = pqisslSam3State::WAIT_LOOKUP;
break;
}
{
i2p::address *addr = new i2p::address;
addr->clear();
addr->base32 = mI2pAddrB32;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::lookupKey, this, static_cast<void*>(addr));
}
mState = pqisslSam3State::WAIT_LOOKUP;
break;
case(pqisslSam3State::DO_CONNECT):
RS_DBG2("DO_CONNECT");
{
auto wrapper = new samEstablishConnectionWrapper();
wrapper->address.clear();
wrapper->address.publicKey = mI2pAddrLong;
wrapper->connection = nullptr;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::establishConnection, this, static_cast<void*>(wrapper));
}
mState = pqisslSam3State::WAIT_CONNECT;
break;
case(pqisslSam3State::DONE):
RS_DBG2("DONE");
if (setupSocket())
return pqiDone;
return pqiError;
/* waiting */
case(pqisslSam3State::WAIT_LOOKUP):
RS_DBG3("WAIT_LOOKUP");
break;
case(pqisslSam3State::WAIT_CONNECT):
RS_DBG3("WAIT_CONNECT");
break;
}
return pqiWait;
}
int pqissli2psam3::net_internal_close(int fd)
{
RS_DBG4();
// sanity check
if (mConn && fd != mConn->fd) {
// this should never happen!
RS_ERR("fd != mConn");
// sam3CloseConnection(mConn);
}
// now to the actuall closing
int ret = pqissl::net_internal_close(fd);
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::closeConnection, this, mConn);
// finally cleanup
mConn = 0;
mState = pqisslSam3State::NONE;
return ret;
}
void pqissli2psam3::taskFinished(taskTicket *&ticket)
{
RS_DBG4();
switch (ticket->task) {
case autoProxyTask::lookupKey:
{
auto addr = static_cast<i2p::address*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mI2pAddrLong = addr->publicKey;
mState = pqisslSam3State::DO_CONNECT;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete addr;
ticket->data = nullptr;
addr = nullptr;
}
break;
case autoProxyTask::establishConnection:
{
auto wrapper = static_cast<struct samEstablishConnectionWrapper*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mConn = wrapper->connection;
mState = pqisslSam3State::DONE;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete wrapper;
ticket->data = nullptr;
wrapper = nullptr;
}
break;
case autoProxyTask::closeConnection:
// nothing to do here
break;
default:
RS_WARN("unkown task", ticket->task);
}
// clean up!
delete ticket;
ticket = nullptr;
}
bool pqissli2psam3::setupSocket()
{
/*
* This function contains the generis part from pqissl::Initiate_Connection()
*/
int err;
int osock = mConn->fd;
err = unix_fcntl_nonblock(osock);
if (err < 0)
{
RS_ERR("Cannot make socket NON-Blocking:", err);
waiting = WAITING_FAIL_INTERFACE;
net_internal_close(osock);
return false;
}
#ifdef WINDOWS_SYS
/* Set TCP buffer size for Windows systems */
int sockbufsize = 0;
int size = sizeof(int);
err = getsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP receive buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP receive buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = 0;
err = getsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP send buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP send buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = WINDOWS_TCP_BUFFER_SIZE;
err = setsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP receive buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP receive buffer size. Error " << err << std::endl;
}
#endif
err = setsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP send buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP send buffer size. Error " << err << std::endl;
}
#endif
#endif // WINDOWS_SYS
mTimeoutTS = time(NULL) + mConnectTimeout;
//std::cerr << "Setting Connect Timeout " << mConnectTimeout << " Seconds into Future " << std::endl;
waiting = WAITING_SOCK_CONNECT;
sockfd = osock;
return true;
}

47
libretroshare/src/pqi/pqissli2psam3.h Normal file

@ -0,0 +1,47 @@
#ifndef PQISSLI2PSAM3_H
#define PQISSLI2PSAM3_H
#include "pqi/pqissl.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "services/autoproxy/p3i2psam3.h"
// Use a state machine as the whole pqi code is designed around them and some operation (like lookup) might be blocking
enum class pqisslSam3State : uint8_t {
NONE = 0,
DO_LOOKUP,
WAIT_LOOKUP,
DO_CONNECT,
WAIT_CONNECT,
DONE
};
class pqissli2psam3 : public pqissl, public autoProxyCallback
{
public:
pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm);
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Initiate_Connection();
int net_internal_close(int fd);
// autoProxyCallback interface
public:
void taskFinished(taskTicket *&ticket);
private:
bool setupSocket();
private:
pqisslSam3State mState;
std::string mI2pAddrB32;
std::string mI2pAddrLong;
Sam3Connection *mConn;
};
#endif // PQISSLI2PSAM3_H

4
libretroshare/src/pqi/pqissllistener.h

@ -186,8 +186,8 @@ public:
virtual int finaliseConnection(int fd, SSL *ssl, const RsPeerId& peerId,
const sockaddr_storage &raddr);
RS_SET_CONTEXT_DEBUG_LEVEL(2)
private:
std::map<RsPeerId, pqissl*> listenaddr;
RS_SET_CONTEXT_DEBUG_LEVEL(2)
};

22
libretroshare/src/pqi/pqisslpersongrp.cc

@ -46,7 +46,7 @@ static struct RsLog::logInfo pqipersongrpzoneInfo = {RsLog::Default, "pqipersong
#endif
#include "pqi/pqisslproxy.h"
#include "pqi/pqissli2pbob.h"
#include "pqi/pqissli2psam3.h"
pqilistener * pqisslpersongrp::locked_createListener(const struct sockaddr_storage &laddr)
{
@ -74,24 +74,26 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
std::cerr << std::endl;
#endif
// Use pqicI2PBOB for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2PBOB;
// Use pqicI2P for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2P;
{
pqisslproxy *pqis = new pqisslproxy((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicSOCKSProxy = new pqiconnect(pqip, rss, pqis);
}
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PBOB))
#ifdef RS_USE_I2P_SAM3
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PSAM3))
{
pqissli2pbob *pqis = new pqissli2pbob((pqissllistener *) listener, pqip, mLinkMgr);
pqissli2psam3 *pqis = new pqissli2psam3((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicI2PBOB = new pqiconnect(pqip, rss, pqis);
} else {
pqicI2PBOB = pqicSOCKSProxy;
pqicI2P = new pqiconnect(pqip, rss, pqis);
}
else
#endif // def RS_USE_I2P_SAM3
pqicI2P = pqicSOCKSProxy;
/* first select type based on peer */
@ -101,7 +103,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_TOR_TCP, pqicSOCKSProxy);
break;
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* peer is not a hidden one but we are */
@ -109,7 +111,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
uint32_t typeOwn = mPeerMgr->getHiddenType(AuthSSL::getAuthSSL()->OwnId());
switch (typeOwn) {
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* this case shouldn't happen! */

11
libretroshare/src/retroshare/rsevents.h

@ -91,8 +91,8 @@ enum class RsEventType : uint32_t
/// @see RsGxsPostedEvent
GXS_IDENTITY = 12,
/// @see RsFiles
SHARED_DIRECTORIES = 13,
/// @see RsFiles @deprecated
SHARED_DIRECTORIES = 13,
/// @see RsFiles
FILE_TRANSFER = 14,
@ -100,8 +100,11 @@ enum class RsEventType : uint32_t
/// @see RsMsgs
CHAT_MESSAGE = 15,
/// @see rspeers.h
NETWORK = 16,
/// @see rspeers.h
NETWORK = 16,
/** Emitted to update library clients about file hashing being completed */
FILE_HASHING_COMPLETED = 20,
__MAX /// Used internally, keep last
};

28
libretroshare/src/retroshare/rsfiles.h

@ -194,7 +194,8 @@ enum class RsFileTransferEventCode: uint8_t {
COMPLETED_FILES_REMOVED = 0x02, //
};
struct RsSharedDirectoriesEvent: RsEvent
struct RS_DEPRECATED_FOR("Packing arbitrary data into an std::string is bad idea")
RsSharedDirectoriesEvent: RsEvent
{
RsSharedDirectoriesEvent() : RsEvent(RsEventType::SHARED_DIRECTORIES), mEventCode(RsSharedDirectoriesEventCode::UNKNOWN) {}
~RsSharedDirectoriesEvent() override = default;
@ -212,6 +213,31 @@ struct RsSharedDirectoriesEvent: RsEvent
std::string mMessage;
};
struct RsFileHashingCompletedEvent: RsEvent
{
RsFileHashingCompletedEvent():
RsEvent(RsEventType::FILE_HASHING_COMPLETED), mHashingSpeed(0) {}
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mFilePath);
RS_SERIAL_PROCESS(mFileHash);
RS_SERIAL_PROCESS(mHashingSpeed);
}
/// Complete path of the file being hashed
std::string mFilePath;
/// File hash, null if error occurred
RsFileHash mFileHash;
/// Hashing speed in MB/s
double mHashingSpeed;
};
struct RsFileTransferEvent: RsEvent
{
RsFileTransferEvent() : RsEvent(RsEventType::FILE_TRANSFER), mFileTransferEventCode(RsFileTransferEventCode::UNKNOWN) {}

5
libretroshare/src/retroshare/rsgxscircles.h

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
{
UNKNOWN = 0, /// Used to detect uninizialized values.
PUBLIC = 1, /// Public distribution, based on GxsIds
PUBLIC = 1, /// Public distribution
EXTERNAL = 2, /// Restricted to an external circle, based on GxsIds
NODES_GROUP = 3, /// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

97
libretroshare/src/retroshare/rsgxsforums.h

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -115,9 +115,13 @@ enum class RsForumEventCode: uint8_t
READ_STATUS_CHANGED = 0x06, /// msg was read or marked unread
STATISTICS_CHANGED = 0x07, /// suppliers and how many messages they have changed
MODERATOR_LIST_CHANGED = 0x08, /// forum moderation list has changed.
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
DELETED_POST = 0x0d, /// Post deleted (usually by cleaning)
/// Distant search result received
DISTANT_SEARCH_RESULT = 0x0e
};
struct RsGxsForumEvent: RsEvent
@ -129,8 +133,8 @@ struct RsGxsForumEvent: RsEvent
RsForumEventCode mForumEventCode;
RsGxsGroupId mForumGroupId;
RsGxsMessageId mForumMsgId;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
///* @see RsEvent @see RsSerializable
void serial_process(
@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mForumGroupId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mModeratorsAdded);
RS_SERIAL_PROCESS(mModeratorsRemoved);
}
@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
~RsGxsForumEvent() override;
};
/** This event is fired once distant search results are received */
struct RsGxsForumsDistantSearchEvent: RsEvent
{
RsGxsForumsDistantSearchEvent():
RsEvent(RsEventType::GXS_FORUMS),
mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
RsForumEventCode mForumEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsForums: public RsGxsIfaceHelper
{
public:
@ -385,6 +411,61 @@ public:
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) = 0;
/**
* @brief Get forum content summaries
* @jsonapi{development}
* @param[in] forumId id of the forum of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries
* @return success or error details if something failed
*/
virtual std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/**
* @brief Search the whole reachable network for matching forums and
* posts
* @jsonapi{development}
* An @see RsGxsForumsDistantSearchEvent is emitted when matching results
* arrives from the network
* @param[in] matchString string to search into the forum and posts
* @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/
virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/**
* @brief Search the local index for matching forums and posts
* @jsonapi{development}
* @param[in] matchString string to search into the index
* @param[out] searchResults storage for searchr esults
* @return success or error details
*/
virtual std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) = 0;
/**
* @brief Request Synchronization with available peers
* Usually syncronization already happen automatically so be carefull
* to call this method only if necessary.
* It has been thinked for use cases like mobile phone where internet
* connection is intermittent and calling this may be useful when a system
* event about connection being available or about to go offline is received
* @jsonapi{development}
* @return Success or error details
*/
virtual std::error_condition requestSynchronization() = 0;
////////////////////////////////////////////////////////////////////////////
/* Following functions are deprecated and should not be considered a stable
* to use API */
/**
* @brief Create forum. Blocking API.
* @jsonapi{development}

62
libretroshare/src/retroshare/rsgxsiface.h

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*!
* This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSummary : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{
RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSearchResults : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default;
};
/*!
* Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes.

1097
libretroshare/src/retroshare/rsgxsifacehelper.h

File diff suppressed because it is too large Load Diff

2
libretroshare/src/retroshare/rsgxsifacetypes.h

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {}
virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

2
libretroshare/src/retroshare/rsinit.h

@ -195,7 +195,7 @@ public:
/*
* Setup Hidden Location;
*/
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob);
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p);
static bool LoadPassword(const std::string& passwd) ;

5
libretroshare/src/retroshare/rsmsgs.h

@ -906,8 +906,9 @@ virtual void getOwnAvatarData(unsigned char *& data,int& size) = 0 ;
* @brief denyLobbyInvite deny a chat lobby invite
* @jsonapi{development}
* @param[in] id chat lobby id you were invited into
*/
virtual void denyLobbyInvite(const ChatLobbyId &id) = 0 ;
* @return true on success
*/
virtual bool denyLobbyInvite(const ChatLobbyId &id) = 0 ;
/**
* @brief getPendingChatLobbyInvites get a list of all pending chat lobby invites

3
libretroshare/src/retroshare/rspeers.h

@ -788,6 +788,7 @@ public:
virtual bool setProxyServer(const uint32_t type, const std::string &addr, const uint16_t port) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
virtual void allowServerIPDetermination(bool) = 0;
virtual bool resetOwnExternalAddressList() = 0;
virtual bool getAllowServerIPDetermination() = 0 ;
@ -886,7 +887,7 @@ public:
// Certificate utils
virtual bool cleanCertificate(
const std::string& certstr, std::string& cleanCert,
bool& is_short_format, uint32_t& error_code ) = 0;
bool& is_short_format, uint32_t& error_code, RsPeerDetails& details) = 0;
virtual std::string saveCertificateToString(const RsPeerId &id) = 0;
virtual bool signGPGCertificate(const RsPgpId &gpg_id,const std::string& gpg_passphrase) = 0;

57
libretroshare/src/retroshare/rstypes.h

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -135,17 +137,10 @@ struct PeerBandwidthLimits : RsSerializable
}
};
//class SearchRequest // unused stuff.
//{
// public:
// int searchId;
// RsCertId toId; /* all zeros for everyone! */
// std::list<Condition> tests;
//};
/********************** For FileCache Interface *****************/
/* G10h4ck: Having this kind of stuff on public headers is pretty dangerous for
* name space pollution, a C++11 enum class should be used instad ASAP */
#define DIR_TYPE_UNKNOWN 0x00
#define DIR_TYPE_ROOT 0x01
#define DIR_TYPE_PERSON 0x02
@ -258,7 +253,10 @@ struct DirStub : RsSerializable
uint8_t type;
std::string name;
void *ref;
/* G10h4ck do we still need to keep this as void* instead of uint64_t for
* retroshare-gui sake? */
void* ref;
/// @see RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
@ -267,29 +265,12 @@ struct DirStub : RsSerializable
RS_SERIAL_PROCESS(type);
RS_SERIAL_PROCESS(name);
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // defined(__GNUC__) && !defined(__clang__)
// (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
// preferable to force type serialization to the correct size rather than letting the compiler choose for us.
// /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
if(sizeof(ref) == 4)
{
std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
RS_SERIAL_PROCESS(handle);
}
else if(sizeof(ref) == 8)
{
std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
RS_SERIAL_PROCESS(handle);
}
else
std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic pop
#endif // defined(__GNUC__) && !defined(__clang__)
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
ref = reinterpret_cast<void*>(handle);
}
};
@ -330,9 +311,9 @@ struct DirDetails : RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx) override
{
/* Enforce serialization as uint64_t because void* changes size
* depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
*/
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
ref = reinterpret_cast<void*>(handle);
@ -347,7 +328,7 @@ struct DirDetails : RsSerializable
RS_SERIAL_PROCESS(name);
RS_SERIAL_PROCESS(hash);
RS_SERIAL_PROCESS(path);
RS_SERIAL_PROCESS(size);
RS_SERIAL_PROCESS(size);
RS_SERIAL_PROCESS(mtime);
RS_SERIAL_PROCESS(flags);
RS_SERIAL_PROCESS(max_mtime);

75
libretroshare/src/rs_android/LocalArray.h Normal file

@ -0,0 +1,75 @@
/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LOCAL_ARRAY_H_included
#define LOCAL_ARRAY_H_included
#include <cstddef>
#include <new>
/**
* A fixed-size array with a size hint. That number of bytes will be allocated
* on the stack, and used if possible, but if more bytes are requested at
* construction time, a buffer will be allocated on the heap (and deallocated
* by the destructor).
*
* The API is intended to be a compatible subset of C++0x's std::array.
*/
template <size_t STACK_BYTE_COUNT>
class LocalArray {
public:
/**
* Allocates a new fixed-size array of the given size. If this size is
* less than or equal to the template parameter STACK_BYTE_COUNT, an
* internal on-stack buffer will be used. Otherwise a heap buffer will
* be allocated.
*/
LocalArray(size_t desiredByteCount) : mSize(desiredByteCount) {
if (desiredByteCount > STACK_BYTE_COUNT) {
mPtr = new char[mSize];
} else {
mPtr = &mOnStackBuffer[0];
}
}
/**
* Frees the heap-allocated buffer, if there was one.
*/
~LocalArray() {
if (mPtr != &mOnStackBuffer[0]) {
delete[] mPtr;
}
}
// Capacity.
size_t size() { return mSize; }
bool empty() { return mSize == 0; }
// Element access.
char& operator[](size_t n) { return mPtr[n]; }
const char& operator[](size_t n) const { return mPtr[n]; }
private:
char mOnStackBuffer[STACK_BYTE_COUNT];
char* mPtr;
size_t mSize;
// Disallow copy and assignment.
LocalArray(const LocalArray&);
void operator=(const LocalArray&);
};
#endif // LOCAL_ARRAY_H_included

49
libretroshare/src/rs_android/README-ifaddrs-android.adoc Normal file

@ -0,0 +1,49 @@
= README Android ifaddrs
Android API level < 24 doesn't provide `getifaddrs` and related functions, we
have tested multiple ways to overcome this issue.
== Non Weorking alternative implementations
https://github.com/kmackay/android-ifaddrs
https://github.com/morristech/android-ifaddrs
https://www.openhub.net/p/android-ifaddrs/
Compiles but then segfault at runtime.
== Qt implementation
Using `QNetworkInterface::allAddresses()` provided by Qt works but at time of
writing (Q4 2021) on newer Android the log is flooded by warnings as we reported
here
https://bugreports.qt.io/browse/QTBUG-78659
plus depending on Qt networking module just for this is frustrating.
Update: the warning flood seems have been fixed in later Qt versions
https://bugreports.qt.io/browse/QTBUG-86394
This solution was the first working we implemented in our code it has been
removed to avoid dependency on Qt, as lighter alternatives are possible.
== Code copied from Android Gingerbread release
As explained here
https://stackoverflow.com/a/57112520
even older Android have `getifaddrs` implementations but doesn't make them
accessible in the API, in particular the one included in Android Gingerbread
https://android.googlesource.com/platform/libcore/+/refs/heads/gingerbread-release/luni/src/main/native/ifaddrs-android.h
https://android.googlesource.com/platform/libcore/+/refs/heads/gingerbread-release/
is particularly easy to include in our code base and compile.
This solution seems the best fitting and doesn't introduce dependency on Qt.
Newer Android releases (expecially 11) have introduced multiple restrictions
on network information access so we suggest you to prepare different APK for
different API level in order to use the `getifaddrs` provided by Android NDK
which deal gracefully with those restrictions as soon as available.

46
libretroshare/src/rs_android/ScopedFd.h Normal file

@ -0,0 +1,46 @@
/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef SCOPED_FD_H_included
#define SCOPED_FD_H_included
#include <unistd.h>
// A smart pointer that closes the given fd on going out of scope.
// Use this when the fd is incidental to the purpose of your function,
// but needs to be cleaned up on exit.
class ScopedFd {
public:
explicit ScopedFd(int fd) : fd(fd) {
}
~ScopedFd() {
close(fd);
}
int get() const {
return fd;
}
private:
int fd;
// Disallow copy and assignment.
ScopedFd(const ScopedFd&);
void operator=(const ScopedFd&);
};
#endif // SCOPED_FD_H_included

26
libretroshare/src/util/androiddebug.h → libretroshare/src/rs_android/androidcoutcerrcatcher.hpp

@ -1,9 +1,9 @@
/*******************************************************************************
* libretroshare/src/util: androiddebug.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2016 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -36,10 +36,10 @@
* class at the beginning of the main of your program to get them (stdout and
* stderr) on logcat output.
*/
class AndroidStdIOCatcher
class AndroidCoutCerrCatcher
{
public:
AndroidStdIOCatcher(const std::string& dTag = "RetroShare",
AndroidCoutCerrCatcher(const std::string& dTag = "RetroShare",
android_LogPriority stdout_pri = ANDROID_LOG_INFO,
android_LogPriority stderr_pri = ANDROID_LOG_ERROR) :
tag(dTag), cout_pri(stdout_pri), cerr_pri(stderr_pri), should_stop(false)
@ -63,10 +63,10 @@ public:
pthread_detach(thr);
}
~AndroidStdIOCatcher()
~AndroidCoutCerrCatcher()
{
should_stop = true;
pthread_join(thr, NULL);
pthread_join(thr, nullptr);
}
private:
@ -79,11 +79,13 @@ private:
pthread_t thr;
std::atomic<bool> should_stop;
static void *thread_func(void* instance)
static void* thread_func(void* instance)
{
__android_log_write(ANDROID_LOG_INFO, "RetroShare", "Android debugging start");
__android_log_write(
ANDROID_LOG_INFO, "RetroShare",
"Android standard I/O catcher start" );
AndroidStdIOCatcher &i = *static_cast<AndroidStdIOCatcher*>(instance);
AndroidCoutCerrCatcher &i = *static_cast<AndroidCoutCerrCatcher*>(instance);
std::string out_buf;
std::string err_buf;
@ -113,9 +115,11 @@ private:
usleep(10000);
}
__android_log_write(ANDROID_LOG_INFO, "RetroShare", "Android debugging stop");
__android_log_write(
ANDROID_LOG_INFO, "RetroShare",
"Android standard I/O catcher stop" );
return NULL;
return nullptr;
}
};

44
libretroshare/src/pqi/pqissli2pbob.cpp → libretroshare/src/rs_android/errorconditionwrap.cpp

@ -1,9 +1,9 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,34 +19,24 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "pqissli2pbob.h"
bool pqissli2pbob::connect_parameter(uint32_t type, const std::string &value)
#include "rs_android/rsjni.hpp"
namespace jni
{
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_STACK_MUTEX(mSslMtx);
// a new line must be appended!
mI2pAddr = value + '\n';
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2pbob::Basic_Connection_Complete()
Local<Object<RsJni::ErrorConditionWrap>> MakeAnything(
ThingToMake<RsJni::ErrorConditionWrap>, JNIEnv& env,
const std::error_condition& ec )
{
int ret;
auto& clazz = jni::Class<RsJni::ErrorConditionWrap>::Singleton(env);
if ((ret = pqissl::Basic_Connection_Complete()) != 1)
{
// basic connection not complete.
return ret;
}
static auto method =
clazz.GetConstructor<jni::jint, jni::String, jni::String>(env);
// send addr. (new line is already appended)
ret = send(sockfd, mI2pAddr.c_str(), mI2pAddr.length(), 0);
if (ret != (int)mI2pAddr.length())
return -1;
return 1;
jni::jint value = ec.value();
auto message = jni::Make<jni::String>(env, ec.message());
auto category = jni::Make<jni::String>(env, ec.category().name());
return clazz.New(env, method, value, message, category);
}
}

228
libretroshare/src/rs_android/ifaddrs-android.h Normal file

@ -0,0 +1,228 @@
/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef IFADDRS_ANDROID_H_included
#define IFADDRS_ANDROID_H_included
#include <arpa/inet.h>
#include <cstring>
#include <errno.h>
#include <net/if.h>
#include <netinet/in.h>
#include <new>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
#include "LocalArray.h"
#include "ScopedFd.h"
// Android (bionic) doesn't have getifaddrs(3)/freeifaddrs(3).
// We fake it here, so java_net_NetworkInterface.cpp can use that API
// with all the non-portable code being in this file.
// Source-compatible subset of the BSD struct.
struct ifaddrs {
// Pointer to next struct in list, or NULL at end.
ifaddrs* ifa_next;
// Interface name.
char* ifa_name;
// Interface flags.
unsigned int ifa_flags;
// Interface network address.
sockaddr* ifa_addr;
// Interface netmask.
sockaddr* ifa_netmask;
ifaddrs(ifaddrs* next)
: ifa_next(next), ifa_name(NULL), ifa_flags(0), ifa_addr(NULL), ifa_netmask(NULL)
{
}
~ifaddrs() {
delete ifa_next;
delete[] ifa_name;
delete ifa_addr;
delete ifa_netmask;
}
// Sadly, we can't keep the interface index for portability with BSD.
// We'll have to keep the name instead, and re-query the index when
// we need it later.
bool setNameAndFlagsByIndex(int interfaceIndex) {
// Get the name.
char buf[IFNAMSIZ];
char* name = if_indextoname(interfaceIndex, buf);
if (name == NULL) {
return false;
}
ifa_name = new char[strlen(name) + 1];
strcpy(ifa_name, name);
// Get the flags.
ScopedFd fd(socket(AF_INET, SOCK_DGRAM, 0));
if (fd.get() == -1) {
return false;
}
ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
strcpy(ifr.ifr_name, name);
int rc = ioctl(fd.get(), SIOCGIFFLAGS, &ifr);
if (rc == -1) {
return false;
}
ifa_flags = ifr.ifr_flags;
return true;
}
// Netlink gives us the address family in the header, and the
// sockaddr_in or sockaddr_in6 bytes as the payload. We need to
// stitch the two bits together into the sockaddr that's part of
// our portable interface.
void setAddress(int family, void* data, size_t byteCount) {
// Set the address proper...
sockaddr_storage* ss = new sockaddr_storage;
memset(ss, 0, sizeof(*ss));
ifa_addr = reinterpret_cast<sockaddr*>(ss);
ss->ss_family = family;
uint8_t* dst = sockaddrBytes(family, ss);
memcpy(dst, data, byteCount);
}
// Netlink gives us the prefix length as a bit count. We need to turn
// that into a BSD-compatible netmask represented by a sockaddr*.
void setNetmask(int family, size_t prefixLength) {
// ...and work out the netmask from the prefix length.
sockaddr_storage* ss = new sockaddr_storage;
memset(ss, 0, sizeof(*ss));
ifa_netmask = reinterpret_cast<sockaddr*>(ss);
ss->ss_family = family;
uint8_t* dst = sockaddrBytes(family, ss);
memset(dst, 0xff, prefixLength / 8);
if ((prefixLength % 8) != 0) {
dst[prefixLength/8] = (0xff << (8 - (prefixLength % 8)));
}
}
// Returns a pointer to the first byte in the address data (which is
// stored in network byte order).
uint8_t* sockaddrBytes(int family, sockaddr_storage* ss) {
if (family == AF_INET) {
sockaddr_in* ss4 = reinterpret_cast<sockaddr_in*>(ss);
return reinterpret_cast<uint8_t*>(&ss4->sin_addr);
} else if (family == AF_INET6) {
sockaddr_in6* ss6 = reinterpret_cast<sockaddr_in6*>(ss);
return reinterpret_cast<uint8_t*>(&ss6->sin6_addr);
}
return NULL;
}
private:
// Disallow copy and assignment.
ifaddrs(const ifaddrs&);
void operator=(const ifaddrs&);
};
// FIXME: use iovec instead.
struct addrReq_struct {
nlmsghdr netlinkHeader;
ifaddrmsg msg;
};
inline bool sendNetlinkMessage(int fd, const void* data, size_t byteCount) {
ssize_t sentByteCount = TEMP_FAILURE_RETRY(send(fd, data, byteCount, 0));
return (sentByteCount == static_cast<ssize_t>(byteCount));
}
inline ssize_t recvNetlinkMessage(int fd, char* buf, size_t byteCount) {
return TEMP_FAILURE_RETRY(recv(fd, buf, byteCount, 0));
}
// Source-compatible with the BSD function.
inline int getifaddrs(ifaddrs** result) {
// Simplify cleanup for callers.
*result = NULL;
// Create a netlink socket.
ScopedFd fd(socket(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE));
if (fd.get() < 0) {
return -1;
}
// Ask for the address information.
addrReq_struct addrRequest;
memset(&addrRequest, 0, sizeof(addrRequest));
addrRequest.netlinkHeader.nlmsg_flags = NLM_F_REQUEST | NLM_F_MATCH;
addrRequest.netlinkHeader.nlmsg_type = RTM_GETADDR;
addrRequest.netlinkHeader.nlmsg_len = NLMSG_ALIGN(NLMSG_LENGTH(sizeof(addrRequest)));
addrRequest.msg.ifa_family = AF_UNSPEC; // All families.
addrRequest.msg.ifa_index = 0; // All interfaces.
if (!sendNetlinkMessage(fd.get(), &addrRequest, addrRequest.netlinkHeader.nlmsg_len)) {
return -1;
}
// Read the responses.
LocalArray<0> buf(65536); // We don't necessarily have std::vector.
ssize_t bytesRead;
while ((bytesRead = recvNetlinkMessage(fd.get(), &buf[0], buf.size())) > 0) {
nlmsghdr* hdr = reinterpret_cast<nlmsghdr*>(&buf[0]);
for (; NLMSG_OK(hdr, (size_t)bytesRead); hdr = NLMSG_NEXT(hdr, bytesRead)) {
switch (hdr->nlmsg_type) {
case NLMSG_DONE:
return 0;
case NLMSG_ERROR:
return -1;
case RTM_NEWADDR:
{
ifaddrmsg* address = reinterpret_cast<ifaddrmsg*>(NLMSG_DATA(hdr));
rtattr* rta = IFA_RTA(address);
size_t ifaPayloadLength = IFA_PAYLOAD(hdr);
while (RTA_OK(rta, ifaPayloadLength)) {
if (rta->rta_type == IFA_LOCAL) {
int family = address->ifa_family;
if (family == AF_INET || family == AF_INET6) {
*result = new ifaddrs(*result);
if (!(*result)->setNameAndFlagsByIndex(address->ifa_index)) {
return -1;
}
(*result)->setAddress(family, RTA_DATA(rta), RTA_PAYLOAD(rta));
(*result)->setNetmask(family, address->ifa_prefixlen);
}
}
rta = RTA_NEXT(rta, ifaPayloadLength);
}
}
break;
}
}
}
// We only get here if recv fails before we see a NLMSG_DONE.
return -1;
}
// Source-compatible with the BSD function.
inline void freeifaddrs(ifaddrs* addresses) {
delete addresses;
}
#endif // IFADDRS_ANDROID_H_included

98
libretroshare/src/rs_android/org/retroshare/service/AssetHelper.java Normal file

@ -0,0 +1,98 @@
/*
* RetroShare
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package org.retroshare.service;
import android.util.Log;
import android.content.Context;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
public class AssetHelper
{
public static boolean copyAsset(
Context ctx, String assetPath, String destinationFilePath )
{
Log.d(TAG, "copyAsset " + assetPath + " -> " + destinationFilePath);
InputStream in;
OutputStream out;
try { in = ctx.getAssets().open(assetPath); }
catch(Exception e)
{
Log.e(
TAG,
"Failure opening asset: " + assetPath + " " + e.getMessage() );
return false;
}
try { out = new FileOutputStream(destinationFilePath); }
catch(Exception e)
{
Log.e(
TAG,
"Failure opening destination: " + destinationFilePath + " " +
e.getMessage() );
return false;
}
try
{
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) > 0) out.write(buf, 0, len);
}
catch(IOException e)
{
Log.e(
TAG,
"Failure coping: " + assetPath + " -> " + destinationFilePath +
" " + e.getMessage() );
return false;
}
try { in.close(); }
catch(IOException e)
{
Log.e(TAG, "Failure closing: " + assetPath + " " + e.getMessage() );
return false;
}
try { out.close(); }
catch(IOException e)
{
Log.e(
TAG,
"Failure closing: " + destinationFilePath + " " +
e.getMessage() );
return false;
}
return true;
}
private static final String TAG = "RetroShare AssetHelper.java";
}

48
libretroshare/src/rs_android/org/retroshare/service/ErrorConditionWrap.java Normal file

@ -0,0 +1,48 @@
/*
* RetroShare
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package org.retroshare.service;
public class ErrorConditionWrap
{
public ErrorConditionWrap(
int value, String message, String categoryName )
{
mValue = value;
mMessage = message;
mCategoryName = categoryName;
}
public int value() { return mValue; }
public String message() { return mMessage; }
public String categoryName() { return mCategoryName; }
public boolean toBool() { return mValue != 0; }
@Override
public String toString()
{ return String.format("%d", mValue)+" "+mMessage+" [" + mCategoryName+ "]"; }
private int mValue = 0;
private String mMessage;
private String mCategoryName;
}

141
libretroshare/src/rs_android/org/retroshare/service/RetroShareServiceAndroid.java Normal file

@ -0,0 +1,141 @@
/*
* RetroShare
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package org.retroshare.service;
import android.app.Service;
import android.os.IBinder;
import android.os.Bundle;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
import android.app.ActivityManager;
public class RetroShareServiceAndroid extends Service
{
public static final int DEFAULT_JSON_API_PORT = 9092;
public static final String DEFAULT_JSON_API_BINDING_ADDRESS = "127.0.0.1";
static { System.loadLibrary("retroshare-service"); }
public static void start(
Context ctx, int jsonApiPort, String jsonApiBindAddress )
{
Log.d(TAG, "start");
Intent intent = new Intent(ctx, RetroShareServiceAndroid.class);
intent.putExtra(JSON_API_PORT_KEY, jsonApiPort);
intent.putExtra(JSON_API_BIND_ADDRESS_KEY, jsonApiBindAddress);
ctx.startService(intent);
}
public static void stop(Context ctx)
{
Log.d(TAG, "stop");
ctx.stopService(new Intent(ctx, RetroShareServiceAndroid.class));
}
public static boolean isRunning(Context ctx)
{
Log.d(TAG, "isRunning");
ActivityManager manager =
(ActivityManager) ctx.getSystemService(Context.ACTIVITY_SERVICE);
for( ActivityManager.RunningServiceInfo service :
manager.getRunningServices(Integer.MAX_VALUE) )
if( RetroShareServiceAndroid.class.getName()
.equals(service.service.getClassName()) )
return true;
return false;
}
public static Context getServiceContext()
{
if(sServiceContext == null)
{
Log.e(TAG, "getServiceContext() called before onCreate");
throw new NullPointerException();
}
return sServiceContext;
}
@Override
public int onStartCommand(
Intent intent, int flags, int startId )
{
if(intent == null)
{
Log.i(TAG, "onStartCommand called without intent");
return Service.START_REDELIVER_INTENT;
}
int jsonApiPort = DEFAULT_JSON_API_PORT;
String jsonApiBindAddress = DEFAULT_JSON_API_BINDING_ADDRESS;
Bundle args = intent.getExtras();
if(args.containsKey(JSON_API_PORT_KEY))
jsonApiPort = args.getInt(JSON_API_PORT_KEY);
if(args.containsKey(JSON_API_BIND_ADDRESS_KEY))
jsonApiBindAddress =
args.getString(JSON_API_BIND_ADDRESS_KEY);
ErrorConditionWrap ec = nativeStart(jsonApiPort, jsonApiBindAddress);
if(ec.toBool()) Log.e(TAG, "onStartCommand(...) " + ec.toString());
return super.onStartCommand(intent, flags, startId);
}
@Override
public void onCreate ()
{
super.onCreate();
sServiceContext = this;
}
@Override
public void onDestroy()
{
ErrorConditionWrap ec = nativeStop();
if(ec.toBool()) Log.e(TAG, "onDestroy() " + ec.toString());
sServiceContext = null;
super.onDestroy();
}
@Override
public IBinder onBind(Intent arg0) { return null; }
private static final String JSON_API_PORT_KEY =
RetroShareServiceAndroid.class.getCanonicalName() +
"/JSON_API_PORT_KEY";
private static final String JSON_API_BIND_ADDRESS_KEY =
RetroShareServiceAndroid.class.getCanonicalName() +
"/JSON_API_BIND_ADDRESS_KEY" ;
private static final String TAG = "RetroShareServiceAndroid.java";
private static Context sServiceContext;
protected static native ErrorConditionWrap nativeStart(
int jsonApiPort, String jsonApiBindAddress );
protected static native ErrorConditionWrap nativeStop();
}

103
libretroshare/src/rs_android/retroshareserviceandroid.cpp Normal file

@ -0,0 +1,103 @@
/*
* RetroShare Service Android
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
#include <limits>
#include <cstdint>
#include "util/stacktrace.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsiface.h"
#include "util/rsdebug.h"
#include "rs_android/retroshareserviceandroid.hpp"
#include "rs_android/rsjni.hpp"
/*static*/ std::unique_ptr<AndroidCoutCerrCatcher>
RetroShareServiceAndroid::sAndroidCoutCerrCatcher = nullptr;
using ErrorConditionWrap = RsJni::ErrorConditionWrap;
/*static*/ jni::Local<jni::Object<ErrorConditionWrap>>
RetroShareServiceAndroid::start(
JNIEnv& env, jni::Class<RetroShareServiceAndroid>&,
jni::jint jsonApiPort, const jni::String& jsonApiBindAddress )
{
if(jsonApiPort < 0 || jsonApiPort > std::numeric_limits<uint16_t>::max())
{
RS_ERR("Got invalid JSON API port: ", jsonApiPort);
return jni::Make<ErrorConditionWrap>(env, std::errc::invalid_argument);
}
RsInfo() << "\n" <<
"+================================================================+\n"
"| o---o o |\n"
"| \\ / - Retroshare Service Android - / \\ |\n"
"| o o---o |\n"
"+================================================================+"
<< std::endl << std::endl;
sAndroidCoutCerrCatcher = std::make_unique<AndroidCoutCerrCatcher>();
RsInit::InitRsConfig();
RsControl::earlyInitNotificationSystem();
RsConfigOptions conf;
conf.jsonApiPort = static_cast<uint16_t>(jsonApiPort);
conf.jsonApiBindAddress = jni::Make<std::string>(env, jsonApiBindAddress);
// Dirty workaround plugins not supported on Android ATM
conf.main_executable_path = " ";
int initResult = RsInit::InitRetroShare(conf);
if(initResult != RS_INIT_OK)
{
RS_ERR("Retroshare core initalization failed with: ", initResult);
return jni::Make<ErrorConditionWrap>(env, std::errc::no_child_process);
}
return jni::Make<ErrorConditionWrap>(env, std::error_condition());
}
jni::Local<jni::Object<ErrorConditionWrap>> RetroShareServiceAndroid::stop(
JNIEnv& env, jni::Class<RetroShareServiceAndroid>& )
{
if(RsControl::instance()->isReady())
{
RsControl::instance()->rsGlobalShutDown();
return jni::Make<ErrorConditionWrap>(env, std::error_condition());
}
sAndroidCoutCerrCatcher.reset();
return jni::Make<ErrorConditionWrap>(env, std::errc::no_such_process);
}
jni::Local<jni::Object<RetroShareServiceAndroid::Context> >
RetroShareServiceAndroid::getAndroidContext(JNIEnv& env)
{
auto& clazz = jni::Class<RetroShareServiceAndroid>::Singleton(env);
static auto method =
clazz.GetStaticMethod<jni::Object<RetroShareServiceAndroid::Context>()>(
env, "getServiceContext" );
return clazz.Call(env, method);
}

84
libretroshare/src/rs_android/retroshareserviceandroid.hpp Normal file

@ -0,0 +1,84 @@
/*
* RetroShare Service Android
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
#pragma once
#include <system_error>
#include <memory>
#include <jni/jni.hpp>
#include "rs_android/rsjni.hpp"
#include "rs_android/androidcoutcerrcatcher.hpp"
#include "util/stacktrace.h"
/** Provide native methods that are registered into corresponding Java class
* to start/stop RetroShare with reasonable comfort on Android platform */
struct RetroShareServiceAndroid
{
static constexpr auto Name()
{ return "org/retroshare/service/RetroShareServiceAndroid"; }
using ErrorConditionWrap = RsJni::ErrorConditionWrap;
/**
* Called from RetroShareServiceAndroid Java to init libretroshare
* @param[in] env the usual JNI parafernalia
* @param[in] jclass the usual JNI parafernalia
* @param[in] jsonApiPort port on which JSON API server will listen
* @param[in] jsonApiBindAddress binding address of the JSON API server
* @note Yeah you read it well we use a full 32 bit signed integer for JSON
* API port. This is because Java lack even the minimum decency to implement
* unsigned integral types so we need to wrap the port (16 bit unsigned
* integer everywhere reasonable) into a full integer and then check at
* runtime the value.
*/
static jni::Local<jni::Object<ErrorConditionWrap>> start(
JNIEnv& env, jni::Class<RetroShareServiceAndroid>& jclass,
jni::jint jsonApiPort, const jni::String& jsonApiBindAddress );
/**
* Called from RetroShareServiceAndroid Java to shutdown libretroshare
* @param[in] env the usual JNI parafernalia
* @param[in] jclass the usual JNI parafernalia
*/
static jni::Local<jni::Object<ErrorConditionWrap>> stop(
JNIEnv& env, jni::Class<RetroShareServiceAndroid>& );
struct Context
{
/// JNI parafernalia
static constexpr auto Name() { return "android/content/Context"; }
};
/// Return RetroShare Service Android Context
static jni::Local<jni::Object<Context>> getAndroidContext(JNIEnv& env);
private:
/** Doesn't involve complex liftime handling stuff better let the runtime
* handle costruction (ASAP)/destruction for us */
static CrashStackTrace CrashStackTrace;
/** Involve threads, file descriptors etc. better handle lifetime
* explicitely */
static std::unique_ptr<AndroidCoutCerrCatcher> sAndroidCoutCerrCatcher;
};

70
libretroshare/src/rs_android/rsjni.cpp Normal file

@ -0,0 +1,70 @@
/*******************************************************************************
* RetroShare JNI utilities *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "rs_android/rsjni.hpp"
#include "rs_android/retroshareserviceandroid.hpp"
rs_view_ptr<JavaVM> RsJni::mJvm = nullptr;
extern "C" JNIEXPORT jint JNICALL JNI_OnLoad_retroshare(JavaVM* vm, void*)
{
RS_DBG(vm);
RsJni::mJvm = vm;
jni::JNIEnv& env { jni::GetEnv(*vm) };
/** Ensure singleton refereces to our own Java classes are inizialized here
* because default Java class loader which is the one accessible by native
* threads which is not main even if attached, is not capable to find them.
* https://stackoverflow.com/questions/20752352/classnotfoundexception-when-finding-a-class-in-jni-background-thread
* https://groups.google.com/g/android-ndk/c/2gkr1mXKn_E */
jni::Class<RsJni::AssetHelper>::Singleton(env);
jni::Class<RsJni::ErrorConditionWrap>::Singleton(env);
jni::RegisterNatives(
env, *jni::Class<RetroShareServiceAndroid>::Singleton(env),
jni::MakeNativeMethod<
decltype(&RetroShareServiceAndroid::start),
&RetroShareServiceAndroid::start >("nativeStart"),
jni::MakeNativeMethod<
decltype(&RetroShareServiceAndroid::stop),
&RetroShareServiceAndroid::stop >("nativeStop")
);
return jni::Unwrap(jni::jni_version_1_2);
}
#ifdef RS_LIBRETROSHARE_EXPORT_JNI_ONLOAD
/** If libretroshare is linked statically to other components which already
* export JNI_OnLoad then a symbol clash may happen
* if RS_LIBRETROSHARE_EXPORT_JNI_ONLOAD is defined.
* @see JNI_OnLoad_retroshare should instead be called from the exported
* JNI_OnLoad */
extern "C" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* _reserved)
{
RS_DBG(vm);
return JNI_OnLoad_retroshare(vm, _reserved);
}
#endif // def RS_LIBRETROSHARE_EXPORT_JNI_ONLOAD

90
libretroshare/src/rs_android/rsjni.hpp Normal file

@ -0,0 +1,90 @@
/*******************************************************************************
* RetroShare JNI utilities *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <system_error>
#include <cstdlib>
#include <jni/jni.hpp>
#include "util/rsmemory.h"
#include "util/cxx23retrocompat.h"
/** Store JVM pointer safely and register native methods */
extern "C" JNIEXPORT jint JNICALL JNI_OnLoad_retroshare(JavaVM* vm, void*);
/** Provide library wide JVM access with some safe measures
* The JVM pointer is set properly by @see JNI_OnLoad_retroshare
*/
class RsJni
{
public:
static inline JavaVM& getVM()
{
if(!mJvm) // [[unlikely]]
{
RS_FATAL( "Attempt to access JVM before JNI_OnLoad_retroshare ",
std::errc::bad_address );
print_stacktrace();
std::exit(std::to_underlying(std::errc::bad_address));
}
return *mJvm;
}
friend jint JNI_OnLoad_retroshare(JavaVM* vm, void*);
/** Provide a comfortable way to access Android package assets like
* bdboot.txt from C++ */
struct AssetHelper
{
static constexpr auto Name()
{ return "org/retroshare/service/AssetHelper"; }
};
/** Provide a comfortable way to propagate C++ error_conditions to Java
* callers */
struct ErrorConditionWrap
{
static constexpr auto Name()
{ return "org/retroshare/service/ErrorConditionWrap"; }
};
private:
static rs_view_ptr<JavaVM> mJvm;
};
namespace jni
{
/** Provides idiomatic way of creating instances via
@code{.cpp}
jni::Make<ErrorConditionWrap>(env, std::error_condition());
@endcode */
jni::Local<jni::Object<RsJni::ErrorConditionWrap>>
MakeAnything(
jni::ThingToMake<RsJni::ErrorConditionWrap>, JNIEnv& env,
const std::error_condition& ec );
}

8
libretroshare/src/rsitems/itempriorities.h

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2011-2018 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -21,7 +23,9 @@
*******************************************************************************/
#pragma once
#include <stdint.h>
#include <cstdint>
using RsItemPriority = uint8_t;
// This file centralises QoS priorities for all transfer RsItems
//

62
libretroshare/src/rsitems/rsgxsforumitems.h

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_GXS_FORUM_ITEMS_H
#define RS_GXS_FORUM_ITEMS_H
#pragma once
#include <map>
@ -31,7 +32,18 @@
#include "retroshare/rsgxsforums.h"
enum class RsGxsForumsItems : uint8_t
{
GROUP_ITEM = 0x02,
MESSAGE_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
class RsGxsForumGroupItem : public RsGxsGrpItem
@ -61,6 +73,48 @@ public:
RsGxsForumMsg mMsg;
};
struct RsGxsForumsSearchRequest : RsSerializable
{
RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsForumsSearchRequest() override = default;
};
struct RsGxsForumsSearchReply : RsSerializable
{
RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsForumsSearchReply() override = default;
};
class RsGxsForumSerialiser : public RsServiceSerializer
{
public:
@ -69,5 +123,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
};
#endif /* RS_GXS_FORUM_ITEMS_H */

42
libretroshare/src/rsitems/rsitem.h

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -28,6 +29,9 @@
#include "serialiser/rsserializer.h"
#include "serialiser/rsserializable.h"
#include "util/stacktrace.h"
#include "rsitems/itempriorities.h"
#include "rsitems/rsserviceids.h"
#include <typeinfo>
@ -42,8 +46,13 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual ~RsItem();
/// TODO: Do this make sense with the new serialization system?
virtual void clear() = 0;
/** TODO: Does the existence of this method make sense with the new
* serialization system? **/
virtual void clear()
{
RS_ERR("Called without being overridden, report to developers");
print_stacktrace();
}
/// @deprecated use << ostream operator instead
RS_DEPRECATED_FOR("<< ostream operator")
@ -70,14 +79,21 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
uint8_t PacketType();
uint8_t PacketSubType() const;
/** For Service Packets, @deprecated use the costructor with priority
* paramether instead */
RS_DEPRECATED RsItem(uint8_t ver, uint16_t service, uint8_t subtype);
/// For Service Packets
RsItem(uint8_t ver, uint16_t service, uint8_t subtype);
RsItem( uint8_t ver, RsServiceType service, uint8_t subtype,
RsItemPriority prio );
uint16_t PacketService() const; /* combined Packet class/type (mid 16bits) */
void setPacketService(uint16_t service);
inline uint8_t priority_level() const { return _priority_level ;}
inline void setPriorityLevel(uint8_t l) { _priority_level = l ;}
#ifdef RS_DEAD_CODE
/*
* TODO: This default implementation should be removed and childs structs
* implement ::serial_process(...) as soon as all the codebase is ported to
@ -86,16 +102,16 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0;
{
std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
<< "new serialization classes, but not derived! Class is "
<< typeid(*this).name() << std::endl;
RS_ERR( "called by an item using new serialization system without "
"overriding Class is: ", typeid(*this).name() );
print_stacktrace();
}
#endif //def RS_DEAD_CODE
protected:
uint32_t type;
RsPeerId peerId;
uint8_t _priority_level;
RsItemPriority _priority_level;
};
/// TODO: Do this make sense with the new serialization system?
@ -109,9 +125,17 @@ public:
uint32_t getRawLength() { return len; }
void * getRawData() { return data; }
virtual void clear() {}
// virtual void clear() override {}
virtual std::ostream &print(std::ostream &out, uint16_t indent = 0);
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&) override
{
RS_ERR( "called by an item using new serialization system ",
typeid(*this).name() );
print_stacktrace();
}
private:
void *data;
uint32_t len;

6
libretroshare/src/rsitems/rsnxsitems.cc

@ -64,6 +64,12 @@ RsItem *RsNxsSerialiser::create_item(uint16_t service_id,uint8_t item_subtype) c
if(service_id != SERVICE_TYPE)
return NULL ;
switch(static_cast<RsNxsSubtype>(item_subtype))
{
case RsNxsSubtype::PULL_REQUEST:
return new RsNxsPullRequestItem(static_cast<RsServiceType>(service_id));
}
switch(item_subtype)
{
case RS_PKT_SUBTYPE_NXS_SYNC_GRP_REQ_ITEM: return new RsNxsSyncGrpReqItem(SERVICE_TYPE) ;

62
libretroshare/src/rsitems/rsnxsitems.h

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012 Christopher Evi-Parker,Robert Fernie<retroshare@lunamutt.com>*
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +22,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSITEMS_H
#define RSNXSITEMS_H
#pragma once
#include <map>
#include <openssl/ssl.h>
@ -33,8 +35,13 @@
#include "serialiser/rstlvkeys.h"
#include "gxs/rsgxsdata.h"
// These items have "flag type" numbers, but this is not used.
enum class RsNxsSubtype : uint8_t
{
PULL_REQUEST = 0x90 /// @see RsNxsPullRequestItem
};
// These items have "flag type" numbers, but this is not used.
// TODO: refactor as C++11 enum class
const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_GRP_REQ_ITEM = 0x01;
const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_GRP_ITEM = 0x02;
const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_GRP_STATS_ITEM = 0x03;
@ -47,14 +54,15 @@ const uint8_t RS_PKT_SUBTYPE_NXS_MSG_ITEM = 0x20;
const uint8_t RS_PKT_SUBTYPE_NXS_TRANSAC_ITEM = 0x40;
const uint8_t RS_PKT_SUBTYPE_NXS_GRP_PUBLISH_KEY_ITEM = 0x80;
// possibility create second service to deal with this functionality
#ifdef RS_DEAD_CODE
// possibility create second service to deal with this functionality
const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_GRP = 0x0001;
const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_MSG = 0x0002;
const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_GRP = 0x0004;
const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_MSG = 0x0008;
const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_REQ = 0x0010;
#endif // def RS_DEAD_CODE
/*!
* Base class for Network exchange service
@ -65,17 +73,14 @@ const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_REQ = 0x0010;
*/
class RsNxsItem : public RsItem
{
public:
RsNxsItem(uint16_t servtype, uint8_t subtype) : RsItem(RS_PKT_VERSION_SERVICE, servtype, subtype), transactionNumber(0)
{
setPriorityLevel(QOS_PRIORITY_RS_GXS_NET);
return;
}
virtual ~RsNxsItem(){}
virtual void clear() = 0;
RsNxsItem(uint16_t servtype, uint8_t subtype):
RsItem(RS_PKT_VERSION_SERVICE, servtype, subtype), transactionNumber(0)
{ setPriorityLevel(QOS_PRIORITY_RS_GXS_NET); }
uint32_t transactionNumber; // set to zero if this is not a transaction item
virtual ~RsNxsItem() = default;
uint32_t transactionNumber; // set to zero if this is not a transaction item
};
@ -362,6 +367,22 @@ public:
};
/*!
* Used to request to a peer pull updates from us ASAP without waiting GXS sync
* timer */
struct RsNxsPullRequestItem: RsItem
{
explicit RsNxsPullRequestItem(RsServiceType servtype):
RsItem( RS_PKT_VERSION_SERVICE,
servtype,
static_cast<uint8_t>(RsNxsSubtype::PULL_REQUEST),
QOS_PRIORITY_RS_GXS_NET ) {}
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext& ) override {}
};
/*!
* Used to respond to a RsGrpMsgsReq
@ -401,6 +422,7 @@ struct RsNxsMsg : RsNxsItem
RsGxsMsgMetaData* metaData;
};
#ifdef RS_DEAD_CODE
/*!
* Used to request a search of user data
*/
@ -422,7 +444,7 @@ public:
RsTlvBinaryData serviceSearchItem; // service aware of item class
uint32_t expiration; // expiration date
};
#endif //def RS_DEAD_CODE
#ifdef UNUSED_CODE
@ -503,14 +525,12 @@ class RsNxsSerialiser : public RsServiceSerializer
{
public:
explicit RsNxsSerialiser(uint16_t servtype) : RsServiceSerializer(servtype), SERVICE_TYPE(servtype) {}
virtual ~RsNxsSerialiser() {}
explicit RsNxsSerialiser(uint16_t servtype):
RsServiceSerializer(servtype), SERVICE_TYPE(servtype) {}
virtual ~RsNxsSerialiser() = default;
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
protected:
const uint16_t SERVICE_TYPE;
};
#endif // RSNXSITEMS_H

14
libretroshare/src/rsitems/rsserviceids.h

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t
{
NONE = 0, /// To detect non-initialized reads
/// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012,
MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028,
BANLIST = 0x0101,
STATUS = 0x0102,
NXS = 0x0200,
@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216,
CHANNELS = 0x0217,
GXSCIRCLE = 0x0218,
/// not gxs, but used with identities.
REPUTATION = 0x0219,
GXS_RECOGN = 0x0220,
@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB,

6
libretroshare/src/rsserver/p3face.h

@ -43,7 +43,7 @@
class p3heartbeat;
class p3discovery2;
class p3I2pBob;
class p3I2pSam3;
/* GXS Classes - just declare the classes.
so we don't have to totally recompile to switch */
@ -161,8 +161,8 @@ public:
p3ChatService *chatSrv;
p3StatusService *mStatusSrv;
p3GxsTunnelService *mGxsTunnels;
#ifdef RS_USE_I2P_BOB
p3I2pBob *mI2pBob;
#ifdef RS_USE_I2P_SAM3
p3I2pSam3 *mI2pSam3;
#endif
// This list contains all threaded services. It will be used to shut them down properly.

4
libretroshare/src/rsserver/p3msgs.cc

@ -530,9 +530,9 @@ bool p3Msgs::acceptLobbyInvite(const ChatLobbyId& id,const RsGxsId& gxs_id)
{
return mChatSrv->acceptLobbyInvite(id,gxs_id) ;
}
void p3Msgs::denyLobbyInvite(const ChatLobbyId& id)
bool p3Msgs::denyLobbyInvite(const ChatLobbyId& id)
{
mChatSrv->denyLobbyInvite(id) ;
return mChatSrv->denyLobbyInvite(id) ;
}
void p3Msgs::getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites)
{

110
libretroshare/src/rsserver/p3msgs.h

@ -64,66 +64,66 @@ public:
/*!
* @param msgList ref to list summarising client's msgs
*/
virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList);
virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg);
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox);
virtual bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList)override ;
virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg)override ;
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox)override ;
RS_DEPRECATED_FOR(sendMail)
virtual bool MessageSend(Rs::Msgs::MessageInfo &info);
virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag);
virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId);
virtual bool MessageToTrash(const std::string &mid, bool bTrash);
virtual bool MessageDelete(const std::string &mid);
virtual bool MessageRead(const std::string &mid, bool unreadByUser);
virtual bool MessageReplied(const std::string &mid, bool replied);
virtual bool MessageForwarded(const std::string &mid, bool forwarded);
virtual bool MessageStar(const std::string &mid, bool star);
virtual bool MessageJunk(const std::string &mid, bool junk);
virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load);
virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId);
virtual bool MessageSend(Rs::Msgs::MessageInfo &info)override ;
virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag)override ;
virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId)override ;
virtual bool MessageToTrash(const std::string &mid, bool bTrash)override ;
virtual bool MessageDelete(const std::string &mid)override ;
virtual bool MessageRead(const std::string &mid, bool unreadByUser)override ;
virtual bool MessageReplied(const std::string &mid, bool replied)override ;
virtual bool MessageForwarded(const std::string &mid, bool forwarded)override ;
virtual bool MessageStar(const std::string &mid, bool star)override ;
virtual bool MessageJunk(const std::string &mid, bool junk)override ;
virtual bool MessageLoadEmbeddedImages(const std::string &mid, bool load)override ;
virtual bool getMsgParentId(const std::string &msgId, std::string &msgParentId)override ;
virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags);
virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color);
virtual bool removeMessageTagType(uint32_t tagId);
virtual bool getMessageTagTypes(Rs::Msgs::MsgTagType& tags)override ;
virtual bool setMessageTagType(uint32_t tagId, std::string& text, uint32_t rgb_color)override ;
virtual bool removeMessageTagType(uint32_t tagId)override ;
virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info);
virtual bool getMessageTag(const std::string &msgId, Rs::Msgs::MsgTagInfo& info)override ;
/* set == false && tagId == 0 --> remove all */
virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set);
virtual bool setMessageTag(const std::string &msgId, uint32_t tagId, bool set)override ;
virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags);
virtual bool resetMessageStandardTagTypes(Rs::Msgs::MsgTagType& tags)override ;
virtual uint32_t getDistantMessagingPermissionFlags() ;
virtual void setDistantMessagingPermissionFlags(uint32_t flags) ;
virtual uint32_t getDistantMessagingPermissionFlags() override ;
virtual void setDistantMessagingPermissionFlags(uint32_t flags) override ;
/*!
* gets avatar from peer, image data in jpeg format
*/
virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size);
virtual void getAvatarData(const RsPeerId& pid,unsigned char *& data,int& size)override ;
/*!
* sets clients avatar, image data should be in jpeg format
*/
virtual void setOwnAvatarData(const unsigned char *data,int size);
virtual void setOwnAvatarData(const unsigned char *data,int size)override ;
/*!
* retrieve clients avatar, image data in jpeg format
*/
virtual void getOwnAvatarData(unsigned char *& data,int& size);
virtual void getOwnAvatarData(unsigned char *& data,int& size)override ;
/*!
* sets clients custom status (e.g. "i'm tired")
*/
virtual void setCustomStateString(const std::string& status_string) ;
virtual void setCustomStateString(const std::string& status_string) override ;
/*!
* retrieves client's custom status
*/
virtual std::string getCustomStateString() ;
virtual std::string getCustomStateString() override ;
/*!
* retrieves peer's custom status
*/
virtual std::string getCustomStateString(const RsPeerId& peer_id) ;
virtual std::string getCustomStateString(const RsPeerId& peer_id) override ;
/*!
@ -132,56 +132,56 @@ public:
* @param msg the message
* @see ChatId
*/
virtual bool sendChat(ChatId destination, std::string msg) ;
virtual bool sendChat(ChatId destination, std::string msg) override ;
/*!
* Return the max message size for security forwarding
*/
virtual uint32_t getMaxMessageSecuritySize(int type);
virtual uint32_t getMaxMessageSecuritySize(int type)override ;
/*!
* sends immediate status string to a specific peer, e.g. in a private chat
* @param chat_id chat id to send status string to
* @param status_string immediate status to send
*/
virtual void sendStatusString(const ChatId& id, const std::string& status_string) ;
virtual void sendStatusString(const ChatId& id, const std::string& status_string) override ;
/**
* @brief clearChatLobby: Signal chat was cleared by GUI.
* @param id: Chat id cleared.
*/
virtual void clearChatLobby(const ChatId &id);
virtual void clearChatLobby(const ChatId &id)override ;
/****************************************/
virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) ;
virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) ;
virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) ;
virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) ;
virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) ;
virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) ;
virtual void denyLobbyInvite(const ChatLobbyId& id) ;
virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) ;
virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) ;
virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id);
virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) ;
virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) ;
virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) ;
virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) ;
virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe);
virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id);
virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) ;
virtual bool joinVisibleChatLobby(const ChatLobbyId& id, const RsGxsId &own_id) override ;
virtual void getListOfNearbyChatLobbies(std::vector<VisibleChatLobbyRecord>& public_lobbies) override ;
virtual void getChatLobbyList(std::list<ChatLobbyId>& cl_list) override ;
virtual bool getChatLobbyInfo(const ChatLobbyId& id,ChatLobbyInfo& info) override ;
virtual void invitePeerToLobby(const ChatLobbyId&, const RsPeerId&) override ;
virtual bool acceptLobbyInvite(const ChatLobbyId& id, const RsGxsId &gxs_id) override ;
virtual bool denyLobbyInvite(const ChatLobbyId& id) override ;
virtual void getPendingChatLobbyInvites(std::list<ChatLobbyInvite>& invites) override ;
virtual void unsubscribeChatLobby(const ChatLobbyId& lobby_id) override ;
virtual void sendLobbyStatusPeerLeaving(const ChatLobbyId& lobby_id)override ;
virtual bool setIdentityForChatLobby(const ChatLobbyId& lobby_id,const RsGxsId&) override ;
virtual bool getIdentityForChatLobby(const ChatLobbyId&,RsGxsId& nick) override ;
virtual bool setDefaultIdentityForChatLobby(const RsGxsId&) override ;
virtual void getDefaultIdentityForChatLobby(RsGxsId& nick) override ;
virtual void setLobbyAutoSubscribe(const ChatLobbyId& lobby_id, const bool autoSubscribe)override ;
virtual bool getLobbyAutoSubscribe(const ChatLobbyId& lobby_id)override ;
virtual ChatLobbyId createChatLobby(const std::string& lobby_name,const RsGxsId& lobby_identity,const std::string& lobby_topic,const std::set<RsPeerId>& invited_friends,ChatLobbyFlags privacy_type) override ;
virtual bool initiateDistantChatConnexion(
const RsGxsId& to_gxs_id, const RsGxsId& from_gxs_id,
DistantChatPeerId &pid, uint32_t& error_code,
bool notify = true );
bool notify = true )override ;
virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info);
virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) ;
virtual bool getDistantChatStatus(const DistantChatPeerId& gxs_id,DistantChatPeerInfo& info)override ;
virtual bool closeDistantChatConnexion(const DistantChatPeerId &pid) override ;
virtual uint32_t getDistantChatPermissionFlags() ;
virtual bool setDistantChatPermissionFlags(uint32_t flags) ;
virtual uint32_t getDistantChatPermissionFlags() override ;
virtual bool setDistantChatPermissionFlags(uint32_t flags) override ;
private:

63
libretroshare/src/rsserver/p3peers.cc

@ -373,7 +373,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec loc", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
for(it = ps.ipAddrs.mExt.mAddrs.begin(); it != ps.ipAddrs.mExt.mAddrs.end(); ++it)
@ -381,7 +381,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec ext", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
}
@ -859,9 +859,15 @@ void p3Peers::getIPServersList(std::list<std::string>& ip_servers)
{
mNetMgr->getIPServersList(ip_servers) ;
}
void p3Peers::getCurrentExtIPList(std::list<std::string>& ip_list)
{
mNetMgr->getCurrentExtIPList(ip_list) ;
}
bool p3Peers::resetOwnExternalAddressList()
{
return mPeerMgr->resetOwnExternalAddressList();
//TODO Phenom 2021-10-30: Need to call something like mNetMgr->netReset();
// to update this addresslist.
return mPeerMgr->resetOwnExternalAddressList();
}
void p3Peers::allowServerIPDetermination(bool b)
{
@ -1284,7 +1290,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
#else
sockaddr_storage tLocal;
if(sockaddr_storage_inet_pton(tLocal, tDetails.localAddr) && sockaddr_storage_isValidNet(tLocal) && sockaddr_storage_ipv6_to_ipv4(tLocal) && tDetails.localPort )
bool validLoc = sockaddr_storage_inet_pton(tLocal, tDetails.localAddr)
&& sockaddr_storage_isValidNet(tLocal)
&& tDetails.localPort;
bool isLocIpv4 = sockaddr_storage_ipv6_to_ipv4(tLocal);
if(validLoc && isLocIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tLocal).sin_addr.s_addr;
@ -1302,7 +1312,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
sockaddr_storage tExt;
if(sockaddr_storage_inet_pton(tExt, tDetails.extAddr) && sockaddr_storage_isValidNet(tExt) && sockaddr_storage_ipv6_to_ipv4(tExt) && tDetails.extPort )
bool validExt = sockaddr_storage_inet_pton(tExt, tDetails.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& tDetails.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if(validExt && isExtIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tExt).sin_addr.s_addr;
@ -1318,6 +1332,17 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
offset += 4+2;
}
else if(validExt && !isExtIpv4)
{
// External address is IPv6, save it on LOCATOR
sockaddr_storage_setport(tExt,tDetails.extPort);
std::string tLocator = sockaddr_storage_tostring(tExt);
addPacketHeader(RsShortInviteFieldType::LOCATOR, tLocator.size(),buf,offset,buf_size);
memcpy(&buf[offset],tLocator.c_str(),tLocator.size());
offset += tLocator.size();
}
#endif
}
@ -1592,13 +1617,31 @@ std::string p3Peers::GetRetroshareInvite( const RsPeerId& sslId, RetroshareInvit
if (getPeerDetails(ssl_id, detail))
{
if(!(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY) || detail.isHiddenNode)
detail.ipAddressList.clear();
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
|| detail.isHiddenNode)
detail.ipAddressList.clear();
//Check if external address is IPv6, then move it to ipAddressList as RsCertificate only allow 4 numbers.
sockaddr_storage tExt;
bool validExt = sockaddr_storage_inet_pton(tExt, detail.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& detail.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
&& !detail.isHiddenNode
&& validExt && !isExtIpv4)
{
sockaddr_storage_setport(tExt,detail.extPort);
detail.ipAddressList.push_front(sockaddr_storage_tostring(tExt) + " "); // Space needed to later parse.
detail.extAddr = ""; //Clear it to not trigg error.
detail.extPort = 0;
}
unsigned char *mem_block = nullptr;
size_t mem_block_size = 0;
if(!AuthPGP::exportPublicKey( RsPgpId(detail.gpg_id), mem_block, mem_block_size, false, !!(invite_flags & RetroshareInviteFlags::PGP_SIGNATURES) ))
if(!AuthPGP::exportPublicKey( RsPgpId(detail.gpg_id), mem_block, mem_block_size, false, !!(invite_flags & RetroshareInviteFlags::PGP_SIGNATURES) ))
{
std::cerr << "Cannot output certificate for id \"" << detail.gpg_id
<< "\". Sorry." << std::endl;
@ -1710,11 +1753,11 @@ bool p3Peers::loadDetailsFromStringCert( const std::string &certstr,
return true;
}
bool p3Peers::cleanCertificate(const std::string &certstr, std::string &cleanCert,bool& is_short_format,uint32_t& error_code)
bool p3Peers::cleanCertificate(const std::string &certstr, std::string &cleanCert,bool& is_short_format,uint32_t& error_code,RsPeerDetails& details)
{
RsCertificate::Format format ;
bool res = RsCertificate::cleanCertificate(certstr,cleanCert,format,error_code,true) ;
bool res = RsCertificate::cleanCertificate(certstr,cleanCert,format,error_code,true,details) ;
if(format == RsCertificate::RS_CERTIFICATE_RADIX)
is_short_format = false;

Some files were not shown because too many files have changed in this diff Show More